Repository: AI-SDC/SACRO-ML (language: python)

Issue:
Yoem's bounded loss function
In [Yoem et al 2018](https://arxiv.org/abs/1709.01604v5) they show an attack vector they call the "bounded loss function". In it's simplest form (with 0, 1) loss, this is equivalent to predicting that:
1. Examples where the target classifier is correct belong to the training set
2. Example where the target classifier is wrong do not belong to the training set

Where a model has been overfitted, this can be quite successful.

Suggestion here is that we modify the code such that we can provide the performance of this kind of attack into the metrics, as something of a baseline.


Discussion / hints:
good plan

Fix patch:
diff --git a/aisdc/attacks/report.py b/aisdc/attacks/report.py
index 12b38870..515c709c 100644
--- a/aisdc/attacks/report.py
+++ b/aisdc/attacks/report.py
@@ -1,4 +1,5 @@
 """Code for automatic report generation"""
+import abc
 import json
 
 import numpy as np
@@ -83,6 +84,8 @@ def default(self, o):
             return int(o)
         if isinstance(o, np.int32):
             return int(o)
+        if isinstance(o, abc.ABCMeta):
+            return str(o)
         return json.JSONEncoder.default(self, o)
 
 
diff --git a/aisdc/attacks/worst_case_attack.py b/aisdc/attacks/worst_case_attack.py
index 28e65128..361822c0 100644
--- a/aisdc/attacks/worst_case_attack.py
+++ b/aisdc/attacks/worst_case_attack.py
@@ -14,6 +14,7 @@
 import numpy as np
 import sklearn
 from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import confusion_matrix
 from sklearn.model_selection import train_test_split
 
 from aisdc.attacks import metrics, report
@@ -40,7 +41,14 @@ def __init__(self, **kwargs):
         self.__dict__["in_sample_filename"] = None
         self.__dict__["out_sample_filename"] = None
         self.__dict__["report_name"] = None
+        self.__dict__["include_model_correct_feature"] = False
         self.__dict__["sort_probs"] = True
+        self.__dict__["mia_attack_model"] = RandomForestClassifier
+        self.__dict__["mia_attack_model_hyp"] = {
+            "min_samples_split": 20,
+            "min_samples_leaf": 10,
+            "max_depth": 5,
+        }
         self.__dict__.update(kwargs)
 
     def __str__(self):
@@ -83,7 +91,20 @@ def attack(self, dataset: Data, target_model: sklearn.base.BaseEstimator) -> Non
         """
         train_preds = target_model.predict_proba(dataset.x_train)
         test_preds = target_model.predict_proba(dataset.x_test)
-        self.attack_from_preds(train_preds, test_preds)
+        train_correct = None
+        test_correct = None
+        if self.args.include_model_correct_feature:
+            train_correct = 1 * (
+                dataset.y_train == target_model.predict(dataset.x_train)
+            )
+            test_correct = 1 * (dataset.y_test == target_model.predict(dataset.x_test))
+
+        self.attack_from_preds(
+            train_preds,
+            test_preds,
+            train_correct=train_correct,
+            test_correct=test_correct,
+        )
 
     def attack_from_prediction_files(self):
         """Start an attack from saved prediction files
@@ -98,7 +119,11 @@ def attack_from_prediction_files(self):
         self.attack_from_preds(train_preds, test_preds)
 
     def attack_from_preds(  # pylint: disable=too-many-locals
-        self, train_preds: np.ndarray, test_preds: np.ndarray
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
     ) -> None:
         """
         Runs the attack based upon the predictions in train_preds and test_preds, and the params
@@ -115,7 +140,13 @@ def attack_from_preds(  # pylint: disable=too-many-locals
         """
         logger = logging.getLogger("attack-from-preds")
         logger.info("Running main attack repetitions")
-        self.attack_metrics = self.run_attack_reps(train_preds, test_preds)
+        self.attack_metrics = self.run_attack_reps(
+            train_preds,
+            test_preds,
+            train_correct=train_correct,
+            test_correct=test_correct,
+        )
+
         if self.args.n_dummy_reps > 0:
             logger.info("Running dummy attack reps")
             self.dummy_attack_metrics = []
@@ -130,7 +161,11 @@ def attack_from_preds(  # pylint: disable=too-many-locals
         logger.info("Finished running attacks")
 
     def _prepare_attack_data(
-        self, train_preds: np.ndarray, test_preds: np.ndarray
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
     ) -> tuple[np.ndarray, np.ndarray]:
         """Prepare training data and labels for attack model
         Combines the train and test preds into a single numpy array (optionally) sorting each
@@ -143,12 +178,23 @@ def _prepare_attack_data(
             test_preds = -np.sort(-test_preds, axis=1)
 
         logger.info("Creating MIA data")
+
+        if self.args.include_model_correct_feature and train_correct is not None:
+            train_preds = np.hstack((train_preds, train_correct[:, None]))
+            test_preds = np.hstack((test_preds, test_correct[:, None]))
+
         mi_x = np.vstack((train_preds, test_preds))
         mi_y = np.hstack((np.ones(len(train_preds)), np.zeros(len(test_preds))))
 
         return (mi_x, mi_y)
 
-    def run_attack_reps(self, train_preds: np.ndarray, test_preds: np.ndarray) -> list:
+    def run_attack_reps(  # pylint: disable = too-many-locals
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
+    ) -> list:
         """
         Run actual attack reps from train and test predictions
 
@@ -167,8 +213,9 @@ def run_attack_reps(self, train_preds: np.ndarray, test_preds: np.ndarray) -> li
         self.args.set_param("n_rows_in", len(train_preds))
         self.args.set_param("n_rows_out", len(test_preds))
         logger = logging.getLogger("attack-reps")
-
-        mi_x, mi_y = self._prepare_attack_data(train_preds, test_preds)
+        mi_x, mi_y = self._prepare_attack_data(
+            train_preds, test_preds, train_correct, test_correct
+        )
 
         mia_metrics = []
         for rep in range(self.args.n_reps):
@@ -176,13 +223,25 @@ def run_attack_reps(self, train_preds: np.ndarray, test_preds: np.ndarray) -> li
             mi_train_x, mi_test_x, mi_train_y, mi_test_y = train_test_split(
                 mi_x, mi_y, test_size=self.args.test_prop, stratify=mi_y
             )
-            attack_classifier = RandomForestClassifier()
+            attack_classifier = self.args.mia_attack_model(
+                **self.args.mia_attack_model_hyp
+            )
             attack_classifier.fit(mi_train_x, mi_train_y)
 
             mia_metrics.append(
                 metrics.get_metrics(attack_classifier, mi_test_x, mi_test_y)
             )
 
+            if self.args.include_model_correct_feature and train_correct is not None:
+                # Compute the Yeom TPR and FPR
+                yeom_preds = mi_test_x[:, -1]
+                tn, fp, fn, tp = confusion_matrix(mi_test_y, yeom_preds).ravel()
+                mia_metrics[-1]["yeom_tpr"] = tp / (tp + fn)
+                mia_metrics[-1]["yeom_fpr"] = fp / (fp + tn)
+                mia_metrics[-1]["yeom_advantage"] = (
+                    mia_metrics[-1]["yeom_tpr"] - mia_metrics[-1]["yeom_fpr"]
+                )
+
         logger.info("Finished simulating attacks")
 
         return mia_metrics
@@ -554,6 +613,21 @@ def main():
         help=("P-value threshold for significance testing. Default = %(default)f"),
     )
 
+    # Not currently possible from the command line as we cannot compute the correctness
+    # of predictions. Possibly to be added in the future
+    # attack_parser.add_argument(
+    #     "--include-correct",
+    #     action="store",
+    #     type=bool,
+    #     required=False,
+    #     default=False,
+    #     dest='include_model_correct_feature',
+    #     help=(
+    #         "Whether or not to include an additional feature into the MIA attack model that "
+    #         "holds whether or not the target model made a correct predicion for each example."
+    #     ),
+    # )
+
     attack_parser.add_argument(
         "--sort-probs",
         action="store",


Test patch:
diff --git a/tests/test_worst_case_attack.py b/tests/test_worst_case_attack.py
index 02cdb732..cdf11e62 100644
--- a/tests/test_worst_case_attack.py
+++ b/tests/test_worst_case_attack.py
@@ -6,6 +6,7 @@
 from unittest.mock import patch
 
 import numpy as np
+import pytest
 from sklearn.datasets import load_breast_cancer
 from sklearn.model_selection import train_test_split
 from sklearn.svm import SVC
@@ -65,6 +66,39 @@ def test_report_worstcase():
     _ = attack_obj.make_report()
 
 
+def test_attack_with_correct_feature():
+    """Test the attack when the model correctness feature is used"""
+    X, y = load_breast_cancer(return_X_y=True, as_frame=False)
+    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3)
+    dataset_obj = dataset.Data()
+    dataset_obj.add_processed_data(train_X, train_y, test_X, test_y)
+
+    target_model = SVC(gamma=0.1, probability=True)
+    target_model.fit(train_X, train_y)
+
+    args = worst_case_attack.WorstCaseAttackArgs(
+        # How many attacks to run -- in each the attack model is trained on a different
+        # subset of the data
+        n_reps=1,
+        n_dummy_reps=1,
+        p_thresh=0.05,
+        in_sample_filename=None,
+        out_sample_filename=None,
+        test_prop=0.5,
+        report_name="test-1rep",
+        include_model_correct_feature=True,
+    )
+
+    # with multiple reps
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    attack_obj.attack(dataset_obj, target_model)
+
+    # Check that attack_metrics has the Yeom metrics
+    assert "yeom_tpr" in attack_obj.attack_metrics[0]
+    assert "yeom_fpr" in attack_obj.attack_metrics[0]
+    assert "yeom_advantage" in attack_obj.attack_metrics[0]
+
+
 def test_attack_from_predictions():
     """checks code that runs attacks from predictions"""
 
@@ -177,6 +211,67 @@ def test_attack_data_prep():
     np.testing.assert_array_equal(mi_x, np.array([[1, 0], [0, 1], [2, 0], [0, 2]]))
 
 
+def test_attack_data_prep_with_correct_feature():
+    """test the method that prepares the attack data.
+    This time, testing that the model correctness values are added, are always
+    the final feature, and are not included in the sorting"""
+    args = worst_case_attack.WorstCaseAttackArgs(include_model_correct_feature=True)
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    train_preds = np.array([[1, 0], [0, 1]], int)
+    test_preds = np.array([[2, 0], [0, 2]], int)
+    train_correct = np.array([1, 0], int)
+    test_correct = np.array([0, 1], int)
+
+    mi_x, mi_y = attack_obj._prepare_attack_data(  # pylint: disable=protected-access
+        train_preds, test_preds, train_correct=train_correct, test_correct=test_correct
+    )
+    np.testing.assert_array_equal(mi_y, np.array([1, 1, 0, 0], np.int))
+    # Test the x data produced. Each row should be sorted in descending order
+    np.testing.assert_array_equal(
+        mi_x, np.array([[1, 0, 1], [1, 0, 0], [2, 0, 0], [2, 0, 1]])
+    )
+
+    # With sort_probs = False, the rows of x should not be sorted
+    args = worst_case_attack.WorstCaseAttackArgs(
+        sort_probs=False, include_model_correct_feature=True
+    )
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    mi_x, mi_y = attack_obj._prepare_attack_data(  # pylint: disable=protected-access
+        train_preds, test_preds, train_correct=train_correct, test_correct=test_correct
+    )
+    np.testing.assert_array_equal(mi_y, np.array([1, 1, 0, 0], np.int))
+    np.testing.assert_array_equal(
+        mi_x, np.array([[1, 0, 1], [0, 1, 0], [2, 0, 0], [0, 2, 1]])
+    )
+
+
+def test_non_rf_mia():
+    """Tests that it is possible to set the attack model via the args
+    In this case, we set as a SVC. But we set probability to false. If the code does
+    indeed try and use the SVC (as we want) it will fail as it will try and access
+    the predict_proba which won't work if probability=False. Hence, if the code throws
+    an AttributeError we now it *is* trying to use the SVC"""
+
+    args = worst_case_attack.WorstCaseAttackArgs(
+        mia_attack_model=SVC,
+        mia_attack_model_hyp={"kernel": "rbf", "probability": False},
+    )
+
+    X, y = load_breast_cancer(return_X_y=True, as_frame=False)
+    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3)
+    dataset_obj = dataset.Data()
+    dataset_obj.add_processed_data(train_X, train_y, test_X, test_y)
+
+    target_model = SVC(gamma=0.1, probability=True)
+    target_model.fit(train_X, train_y)
+    ytr_pred = target_model.predict_proba(train_X)
+    yte_pred = target_model.predict_proba(test_X)
+
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    with pytest.raises(AttributeError):
+        attack_obj.attack_from_preds(ytr_pred, yte_pred)
+
+
 def test_main():
     """test invocation via command line"""