Repository: ICB-DCM/pyPESTO

Issue:
Add more tests to validation intervals
**Feature description**
It would be nice to check, if the prediction for `x_opt` leads to the significance `1` and if the significance approaches zero for  larger and larger deviations. (I had some problems with that for one application of mine, but I would not exclude that the issue there is outside pyPESTO) 


Comments:

Patch:
diff --git a/pypesto/profile/validation_intervals.py b/pypesto/profile/validation_intervals.py
index 15de409..c60a437 100644
--- a/pypesto/profile/validation_intervals.py
+++ b/pypesto/profile/validation_intervals.py
@@ -125,13 +125,13 @@ def validation_profile_significance(
     # => survival function chi.sf
     if return_significance:
         if lsq_objective:
-            return chi2.sf(nllh_new-nllh_old, 1)
+            return chi2.sf(nllh_old-nllh_new, 1)
         else:
-            return chi2.sf(2*(nllh_new-nllh_old), 1)
+            return chi2.sf(2*(nllh_old-nllh_new), 1)
     # compute the probability, that the validation data set is inside the CI
     # => cumulative density function chi.cdf
     else:
         if lsq_objective:
-            return chi2.cdf(nllh_new-nllh_old, 1)
+            return chi2.cdf(nllh_old-nllh_new, 1)
         else:
-            return chi2.cdf(2*(nllh_new-nllh_old), 1)
+            return chi2.cdf(2*(nllh_old-nllh_new), 1)


Test patch:
diff --git a/test/profile/test_validation_intervals.py b/test/profile/test_validation_intervals.py
index 623153c..a129b19 100644
--- a/test/profile/test_validation_intervals.py
+++ b/test/profile/test_validation_intervals.py
@@ -16,18 +16,18 @@ class ValidationIntervalTest(unittest.TestCase):
     @classmethod
     def setUp(cls):
 
-        lb = np.array([-1])
-        ub = np.array([5])
+        cls.lb = np.array([-1])
+        cls.ub = np.array([5])
 
         cls.problem_training_data = pypesto.Problem(
             lsq_residual_objective(0),
-            lb, ub)
+            cls.lb, cls.ub)
 
         cls.problem_all_data = pypesto.Problem(
             pypesto.objective.AggregatedObjective(
                 [lsq_residual_objective(0),
                  lsq_residual_objective(2)]),
-            lb, ub)
+            cls.lb, cls.ub)
 
         # optimum f(0)=0
         cls.result_training_data = optimize.minimize(cls.problem_training_data,
@@ -45,8 +45,58 @@ class ValidationIntervalTest(unittest.TestCase):
                                                 self.result_all_data)
 
         # fit with refitting inside function
-        profile.validation_profile_significance(self.problem_all_data,
-                                                self.result_training_data)
+        res_with_significance_true = profile.validation_profile_significance(
+            self.problem_all_data,
+            self.result_training_data)
+
+        # test return_significance = False leads the result to 1-significance
+        res_with_significance_false = profile.validation_profile_significance(
+            self.problem_all_data,
+            self.result_training_data,
+            return_significance=False)
+
+        self.assertAlmostEqual(1-res_with_significance_true,
+                               res_with_significance_false,
+                               places=4)
+
+        # test if significance=1, if the validation experiment coincides with
+        # the Max Likelihood prediction
+
+        problem_val = pypesto.Problem(pypesto.objective.AggregatedObjective(
+                [lsq_residual_objective(0),
+                 lsq_residual_objective(0)]),
+            self.lb, self.ub)
+
+        significance = profile.validation_profile_significance(
+            problem_val,
+            self.result_training_data)
+
+        self.assertAlmostEqual(significance, 1)
+
+        # Test if the significance approaches zero (monotonously decreasing)
+        # if the discrepancy between Max. Likelihood prediction and observed
+        # validation interval is increased
+
+        for d_val in [0.1, 1, 10, 100]:
+
+            # problem including d_val
+            problem_val = pypesto.Problem(
+                pypesto.objective.AggregatedObjective(
+                    [lsq_residual_objective(0),
+                     lsq_residual_objective(d_val)]),
+                self.lb, self.ub)
+
+            sig_d_val = profile.validation_profile_significance(
+                problem_val,
+                self.result_training_data)
+
+            # test, if the more extreme measurement leads to a more
+            # significant result
+            self.assertLessEqual(sig_d_val, significance)
+            significance = sig_d_val
+
+        # test if the significance approaches 0 for d_val=100
+        self.assertAlmostEqual(significance, 0)
 
 
 def lsq_residual_objective(d: float):