Repository: AI-SDC/AI-SDC

Issue:
Add option to include target model error into attacks as a feature
Whether or not the target model classifies an example correctly provides some signal that could be of use to an attacker. We currently do not use this in the attacks, but should include an option to allow it.

The 0/1 loss between the predicted class and the actual class (binary feature) should be added for the worst case attack.

Discussion:

Patch:
diff --git a/aisdc/attacks/report.py b/aisdc/attacks/report.py
index 12b3887..515c709 100644
--- a/aisdc/attacks/report.py
+++ b/aisdc/attacks/report.py
@@ -1,4 +1,5 @@
 """Code for automatic report generation"""
+import abc
 import json
 
 import numpy as np
@@ -83,6 +84,8 @@ class NumpyArrayEncoder(json.JSONEncoder):
             return int(o)
         if isinstance(o, np.int32):
             return int(o)
+        if isinstance(o, abc.ABCMeta):
+            return str(o)
         return json.JSONEncoder.default(self, o)
 
 
diff --git a/aisdc/attacks/worst_case_attack.py b/aisdc/attacks/worst_case_attack.py
index 28e6512..361822c 100644
--- a/aisdc/attacks/worst_case_attack.py
+++ b/aisdc/attacks/worst_case_attack.py
@@ -14,6 +14,7 @@ from typing import Any
 import numpy as np
 import sklearn
 from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import confusion_matrix
 from sklearn.model_selection import train_test_split
 
 from aisdc.attacks import metrics, report
@@ -40,7 +41,14 @@ class WorstCaseAttackArgs:
         self.__dict__["in_sample_filename"] = None
         self.__dict__["out_sample_filename"] = None
         self.__dict__["report_name"] = None
+        self.__dict__["include_model_correct_feature"] = False
         self.__dict__["sort_probs"] = True
+        self.__dict__["mia_attack_model"] = RandomForestClassifier
+        self.__dict__["mia_attack_model_hyp"] = {
+            "min_samples_split": 20,
+            "min_samples_leaf": 10,
+            "max_depth": 5,
+        }
         self.__dict__.update(kwargs)
 
     def __str__(self):
@@ -83,7 +91,20 @@ class WorstCaseAttack(Attack):
         """
         train_preds = target_model.predict_proba(dataset.x_train)
         test_preds = target_model.predict_proba(dataset.x_test)
-        self.attack_from_preds(train_preds, test_preds)
+        train_correct = None
+        test_correct = None
+        if self.args.include_model_correct_feature:
+            train_correct = 1 * (
+                dataset.y_train == target_model.predict(dataset.x_train)
+            )
+            test_correct = 1 * (dataset.y_test == target_model.predict(dataset.x_test))
+
+        self.attack_from_preds(
+            train_preds,
+            test_preds,
+            train_correct=train_correct,
+            test_correct=test_correct,
+        )
 
     def attack_from_prediction_files(self):
         """Start an attack from saved prediction files
@@ -98,7 +119,11 @@ class WorstCaseAttack(Attack):
         self.attack_from_preds(train_preds, test_preds)
 
     def attack_from_preds(  # pylint: disable=too-many-locals
-        self, train_preds: np.ndarray, test_preds: np.ndarray
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
     ) -> None:
         """
         Runs the attack based upon the predictions in train_preds and test_preds, and the params
@@ -115,7 +140,13 @@ class WorstCaseAttack(Attack):
         """
         logger = logging.getLogger("attack-from-preds")
         logger.info("Running main attack repetitions")
-        self.attack_metrics = self.run_attack_reps(train_preds, test_preds)
+        self.attack_metrics = self.run_attack_reps(
+            train_preds,
+            test_preds,
+            train_correct=train_correct,
+            test_correct=test_correct,
+        )
+
         if self.args.n_dummy_reps > 0:
             logger.info("Running dummy attack reps")
             self.dummy_attack_metrics = []
@@ -130,7 +161,11 @@ class WorstCaseAttack(Attack):
         logger.info("Finished running attacks")
 
     def _prepare_attack_data(
-        self, train_preds: np.ndarray, test_preds: np.ndarray
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
     ) -> tuple[np.ndarray, np.ndarray]:
         """Prepare training data and labels for attack model
         Combines the train and test preds into a single numpy array (optionally) sorting each
@@ -143,12 +178,23 @@ class WorstCaseAttack(Attack):
             test_preds = -np.sort(-test_preds, axis=1)
 
         logger.info("Creating MIA data")
+
+        if self.args.include_model_correct_feature and train_correct is not None:
+            train_preds = np.hstack((train_preds, train_correct[:, None]))
+            test_preds = np.hstack((test_preds, test_correct[:, None]))
+
         mi_x = np.vstack((train_preds, test_preds))
         mi_y = np.hstack((np.ones(len(train_preds)), np.zeros(len(test_preds))))
 
         return (mi_x, mi_y)
 
-    def run_attack_reps(self, train_preds: np.ndarray, test_preds: np.ndarray) -> list:
+    def run_attack_reps(  # pylint: disable = too-many-locals
+        self,
+        train_preds: np.ndarray,
+        test_preds: np.ndarray,
+        train_correct: np.ndarray = None,
+        test_correct: np.ndarray = None,
+    ) -> list:
         """
         Run actual attack reps from train and test predictions
 
@@ -167,8 +213,9 @@ class WorstCaseAttack(Attack):
         self.args.set_param("n_rows_in", len(train_preds))
         self.args.set_param("n_rows_out", len(test_preds))
         logger = logging.getLogger("attack-reps")
-
-        mi_x, mi_y = self._prepare_attack_data(train_preds, test_preds)
+        mi_x, mi_y = self._prepare_attack_data(
+            train_preds, test_preds, train_correct, test_correct
+        )
 
         mia_metrics = []
         for rep in range(self.args.n_reps):
@@ -176,13 +223,25 @@ class WorstCaseAttack(Attack):
             mi_train_x, mi_test_x, mi_train_y, mi_test_y = train_test_split(
                 mi_x, mi_y, test_size=self.args.test_prop, stratify=mi_y
             )
-            attack_classifier = RandomForestClassifier()
+            attack_classifier = self.args.mia_attack_model(
+                **self.args.mia_attack_model_hyp
+            )
             attack_classifier.fit(mi_train_x, mi_train_y)
 
             mia_metrics.append(
                 metrics.get_metrics(attack_classifier, mi_test_x, mi_test_y)
             )
 
+            if self.args.include_model_correct_feature and train_correct is not None:
+                # Compute the Yeom TPR and FPR
+                yeom_preds = mi_test_x[:, -1]
+                tn, fp, fn, tp = confusion_matrix(mi_test_y, yeom_preds).ravel()
+                mia_metrics[-1]["yeom_tpr"] = tp / (tp + fn)
+                mia_metrics[-1]["yeom_fpr"] = fp / (fp + tn)
+                mia_metrics[-1]["yeom_advantage"] = (
+                    mia_metrics[-1]["yeom_tpr"] - mia_metrics[-1]["yeom_fpr"]
+                )
+
         logger.info("Finished simulating attacks")
 
         return mia_metrics
@@ -554,6 +613,21 @@ def main():
         help=("P-value threshold for significance testing. Default = %(default)f"),
     )
 
+    # Not currently possible from the command line as we cannot compute the correctness
+    # of predictions. Possibly to be added in the future
+    # attack_parser.add_argument(
+    #     "--include-correct",
+    #     action="store",
+    #     type=bool,
+    #     required=False,
+    #     default=False,
+    #     dest='include_model_correct_feature',
+    #     help=(
+    #         "Whether or not to include an additional feature into the MIA attack model that "
+    #         "holds whether or not the target model made a correct predicion for each example."
+    #     ),
+    # )
+
     attack_parser.add_argument(
         "--sort-probs",
         action="store",


Test patch:
diff --git a/tests/test_worst_case_attack.py b/tests/test_worst_case_attack.py
index 02cdb73..cdf11e6 100644
--- a/tests/test_worst_case_attack.py
+++ b/tests/test_worst_case_attack.py
@@ -6,6 +6,7 @@ import sys
 from unittest.mock import patch
 
 import numpy as np
+import pytest
 from sklearn.datasets import load_breast_cancer
 from sklearn.model_selection import train_test_split
 from sklearn.svm import SVC
@@ -65,6 +66,39 @@ def test_report_worstcase():
     _ = attack_obj.make_report()
 
 
+def test_attack_with_correct_feature():
+    """Test the attack when the model correctness feature is used"""
+    X, y = load_breast_cancer(return_X_y=True, as_frame=False)
+    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3)
+    dataset_obj = dataset.Data()
+    dataset_obj.add_processed_data(train_X, train_y, test_X, test_y)
+
+    target_model = SVC(gamma=0.1, probability=True)
+    target_model.fit(train_X, train_y)
+
+    args = worst_case_attack.WorstCaseAttackArgs(
+        # How many attacks to run -- in each the attack model is trained on a different
+        # subset of the data
+        n_reps=1,
+        n_dummy_reps=1,
+        p_thresh=0.05,
+        in_sample_filename=None,
+        out_sample_filename=None,
+        test_prop=0.5,
+        report_name="test-1rep",
+        include_model_correct_feature=True,
+    )
+
+    # with multiple reps
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    attack_obj.attack(dataset_obj, target_model)
+
+    # Check that attack_metrics has the Yeom metrics
+    assert "yeom_tpr" in attack_obj.attack_metrics[0]
+    assert "yeom_fpr" in attack_obj.attack_metrics[0]
+    assert "yeom_advantage" in attack_obj.attack_metrics[0]
+
+
 def test_attack_from_predictions():
     """checks code that runs attacks from predictions"""
 
@@ -177,6 +211,67 @@ def test_attack_data_prep():
     np.testing.assert_array_equal(mi_x, np.array([[1, 0], [0, 1], [2, 0], [0, 2]]))
 
 
+def test_attack_data_prep_with_correct_feature():
+    """test the method that prepares the attack data.
+    This time, testing that the model correctness values are added, are always
+    the final feature, and are not included in the sorting"""
+    args = worst_case_attack.WorstCaseAttackArgs(include_model_correct_feature=True)
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    train_preds = np.array([[1, 0], [0, 1]], int)
+    test_preds = np.array([[2, 0], [0, 2]], int)
+    train_correct = np.array([1, 0], int)
+    test_correct = np.array([0, 1], int)
+
+    mi_x, mi_y = attack_obj._prepare_attack_data(  # pylint: disable=protected-access
+        train_preds, test_preds, train_correct=train_correct, test_correct=test_correct
+    )
+    np.testing.assert_array_equal(mi_y, np.array([1, 1, 0, 0], np.int))
+    # Test the x data produced. Each row should be sorted in descending order
+    np.testing.assert_array_equal(
+        mi_x, np.array([[1, 0, 1], [1, 0, 0], [2, 0, 0], [2, 0, 1]])
+    )
+
+    # With sort_probs = False, the rows of x should not be sorted
+    args = worst_case_attack.WorstCaseAttackArgs(
+        sort_probs=False, include_model_correct_feature=True
+    )
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    mi_x, mi_y = attack_obj._prepare_attack_data(  # pylint: disable=protected-access
+        train_preds, test_preds, train_correct=train_correct, test_correct=test_correct
+    )
+    np.testing.assert_array_equal(mi_y, np.array([1, 1, 0, 0], np.int))
+    np.testing.assert_array_equal(
+        mi_x, np.array([[1, 0, 1], [0, 1, 0], [2, 0, 0], [0, 2, 1]])
+    )
+
+
+def test_non_rf_mia():
+    """Tests that it is possible to set the attack model via the args
+    In this case, we set as a SVC. But we set probability to false. If the code does
+    indeed try and use the SVC (as we want) it will fail as it will try and access
+    the predict_proba which won't work if probability=False. Hence, if the code throws
+    an AttributeError we now it *is* trying to use the SVC"""
+
+    args = worst_case_attack.WorstCaseAttackArgs(
+        mia_attack_model=SVC,
+        mia_attack_model_hyp={"kernel": "rbf", "probability": False},
+    )
+
+    X, y = load_breast_cancer(return_X_y=True, as_frame=False)
+    train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3)
+    dataset_obj = dataset.Data()
+    dataset_obj.add_processed_data(train_X, train_y, test_X, test_y)
+
+    target_model = SVC(gamma=0.1, probability=True)
+    target_model.fit(train_X, train_y)
+    ytr_pred = target_model.predict_proba(train_X)
+    yte_pred = target_model.predict_proba(test_X)
+
+    attack_obj = worst_case_attack.WorstCaseAttack(args)
+    with pytest.raises(AttributeError):
+        attack_obj.attack_from_preds(ytr_pred, yte_pred)
+
+
 def test_main():
     """test invocation via command line"""