Repository: educationaltestingservice/rsmtool
Issue: Make check in RSMTool more explicit

There's a currently [a place in RSMTool](https://github.com/EducationalTestingService/rsmtool/blob/master/rsmtool/rsmtool.py#L286) where we raise a `ValueError` if a coefficients file already exists in the output directory. There's no documentation as to why this is being checked.

From @aloukina, this is why we do this:

> This is what I think it’s doing in a roundabout way: it uses the existence of coefficients file as a shortcut to establish that we are dealing with a linear model.  If you first run a LR experiment and then followed it with SVR using the same id and the same output_directory, you’ll get this error. The coefficients file will be there but the modeler will not be able to scale them because [these lines](https://github.com/EducationalTestingService/rsmtool/blob/0eb9a96e236828483123e46a4b1e2f1376c59827/rsmtool/modeler.py#L1422) will fail. 

We should replace this roundabout check with a much more explicit check, e.g., simply testingwhether `predconfig.get_coefficients()` fails? That would make it much more readable.


Changed files:
rsmtool/rsmtool.py, rsmtool/modeler.py, tests/test_experiment_rsmtool_4.py

diff --git a/rsmtool/modeler.py b/rsmtool/modeler.py
index 75b0418d..0e2c30a5 100644
--- a/rsmtool/modeler.py
+++ b/rsmtool/modeler.py
@@ -1409,6 +1409,11 @@ class Modeler:
             A data_container object containing `coefficients_scaled`
             This DataFrame contains the scaled coefficients
             and the feature names, along with the intercept.
+
+        Raises
+        ------
+        RuntimeError
+            If the model is non-linear and no coefficients are available.
         """
 
         Analyzer.check_param_names(configuration, ['train_predictions_mean',
@@ -1421,7 +1426,12 @@ class Modeler:
 
         feature_names = self.get_feature_names()
 
-        coefficients = self.get_coefficients()
+        # try to get the model coefficients, if available
+        try:
+            coefficients = self.get_coefficients()
+        except AttributeError:
+            raise RuntimeError("no coefficients available for this model.")
+
         intercept = self.get_intercept()
 
         # scale the coefficients and the intercept
diff --git a/rsmtool/rsmtool.py b/rsmtool/rsmtool.py
index b505e53a..86ef23e4 100644
--- a/rsmtool/rsmtool.py
+++ b/rsmtool/rsmtool.py
@@ -269,27 +269,35 @@ def run_experiment(config_file_or_obj_or_dict,
     original_coef_file = join(csvdir, '{}_coefficients.{}'.format(pred_config['experiment_id'],
                                                                   file_format))
 
-    # If coefficients file exists, then generate
-    # scaled coefficients and save to file
+    # If coefficients file exists, then try to generate the scaled
+    # coefficients and save them to a file
     if exists(original_coef_file):
         logger.info('Scaling the coefficients and saving them to disk')
         try:
 
-            # Scale coefficients, and return DataContainer w/ scaled coefficients
+            # scale coefficients, and return DataContainer w/ scaled coefficients
             scaled_data_container = modeler.scale_coefficients(pred_config)
 
-            # Write out files to disk
+        # raise an error if the coefficient file exists but the
+        # coefficients are not available for the current model
+        # which can happen if the user is re-running the same experiment
+        # with the same ID but with a non-linear model whereas the previous
+        # run of the same ID was with a linear model and the user has not
+        # cleared the directory
+        except RuntimeError:
+            raise ValueError("It appears you previously ran an experiment with the "
+                             "same ID using a linear model and saved its output to "
+                             "the same directory. That output is interfering with "
+                             "the current experiment. Either clear the contents "
+                             "of the output directory or re-run the current "
+                             "experiment using a different experiment ID.")
+        else:
+
+            # Write out scaled coefficients to disk
             writer.write_experiment_output(csvdir,
                                            scaled_data_container,
                                            file_format=file_format)
 
-        except AttributeError:
-            raise ValueError("It appears you are trying to save two different "
-                             "experiments to the same directory using the same "
-                             "ID. Please clear the content of the directory and "
-                             "rerun both experiments using different "
-                             "experiment IDs.")
-
     # Add processed data_container frames to pred_data_container
     new_pred_data_container = pred_data_container + processed_container
 
diff --git a/tests/test_experiment_rsmtool_4.py b/tests/test_experiment_rsmtool_4.py
index f95ed3b3..e15e9495 100644
--- a/tests/test_experiment_rsmtool_4.py
+++ b/tests/test_experiment_rsmtool_4.py
@@ -1,3 +1,4 @@
+import json
 import os
 
 from os.path import join
@@ -6,6 +7,8 @@ from nose.tools import raises, assert_equal
 from parameterized import param, parameterized
 from sklearn.exceptions import ConvergenceWarning
 
+from rsmtool import run_experiment
+from rsmtool.configuration_parser import Configuration
 from rsmtool.test_utils import (check_run_experiment,
                                 collect_warning_messages_from_report,
                                 do_run_experiment)
@@ -119,3 +122,18 @@ def test_run_experiment_with_warnings():
     assert_equal(len(unicode_warnings), 1)
     assert_equal(len(runtime_warnings), 1)
     assert_equal(len(user_warnings), 1)
+
+
+@raises(ValueError)
+def test_same_id_linear_then_non_linear_raises_error():
+
+    experiment_path = join(rsmtool_test_dir, "data", "experiments", "lr")
+    configpath = join(experiment_path, "lr.json")
+    configdict = json.load(open(configpath, "r"))
+
+    output_dir = "test_outputs/same-id-different-model"
+    config = Configuration(configdict, configdir=experiment_path)
+    run_experiment(config, output_dir, overwrite_output=True)
+
+    config['model'] = "SVC"
+    run_experiment(config, output_dir, overwrite_output=True)