Repository: OSeMOSYS/otoole

Issue:
`otoole results cbc csv {} {} --input_datafile` fails if datafile contains ResultsPath (or other parameter)
```python
DEBUG:otoole.read_strategies:Extracting data for ResultsPath
Traceback (most recent call last):
  File "/Users/wusher/miniconda3/envs/snakemake/bin/otoole", line 8, in <module>
    sys.exit(main())
  File "/Users/wusher/miniconda3/envs/snakemake/lib/python3.8/site-packages/otoole/cli.py", line 299, in main
    args.func(args)
  File "/Users/wusher/miniconda3/envs/snakemake/lib/python3.8/site-packages/otoole/cli.py", line 98, in result_matrix
    convert_cbc_to_csv(
  File "/Users/wusher/miniconda3/envs/snakemake/lib/python3.8/site-packages/otoole/results/convert.py", line 287, in convert_cbc_to_csv
    input_data, _ = ReadDatafile().read(input_data_path)
  File "/Users/wusher/miniconda3/envs/snakemake/lib/python3.8/site-packages/otoole/read_strategies.py", line 201, in read
    inputs = self._convert_amply_to_dataframe(amply_datafile, config)
  File "/Users/wusher/miniconda3/envs/snakemake/lib/python3.8/site-packages/otoole/read_strategies.py", line 265, in _convert_amply_to_dataframe
    if config[name]["type"] == "param":
KeyError: 'ResultsPath'
```

Discussion:

Patch:
diff --git a/src/otoole/read_strategies.py b/src/otoole/read_strategies.py
index 8e2900c..36143f3 100644
--- a/src/otoole/read_strategies.py
+++ b/src/otoole/read_strategies.py
@@ -258,39 +258,56 @@ class ReadDatafile(ReadStrategy):
         dict of pandas.DataFrame
         """
 
-        dict_of_dataframes = {}
+        dict_of_dataframes = {}  # type: Dict[str, pd.DataFrame]
 
         for name in datafile_parser.symbols.keys():
             logger.debug("Extracting data for %s", name)
-            if config[name]["type"] == "param":
-                indices = config[name]["indices"].copy()
-                indices_dtypes = [config[index]["dtype"] for index in indices]
-                indices.append("VALUE")
-                indices_dtypes.append("float")
-
-                raw_data = datafile_parser[name].data
-                data = self._convert_amply_data_to_list(raw_data)
-                df = pd.DataFrame(data=data, columns=indices)
-                try:
-                    dict_of_dataframes[name] = check_datatypes(df, config, name)
-                except ValueError as ex:
-                    msg = "Validation error when checking datatype of {}: {}".format(
-                        name, str(ex)
-                    )
-                    raise ValueError(msg)
-            elif config[name]["type"] == "set":
-                data = datafile_parser[name].data
-                logger.debug(data)
-
-                indices = ["VALUE"]
-                df = pd.DataFrame(
-                    data=data, columns=indices, dtype=config[name]["dtype"]
+            if name in config and config[name]["type"] == "param":
+                dict_of_dataframes[name] = self.extract_param(
+                    config, name, datafile_parser, dict_of_dataframes
+                )
+            elif name in config and config[name]["type"] == "set":
+                dict_of_dataframes[name] = self.extract_set(
+                    datafile_parser, name, config, dict_of_dataframes
+                )
+                logger.debug("\n%s\n", dict_of_dataframes[name])
+            else:
+                logger.warning(
+                    "Parameter {} could not be found in the configuration.".format(name)
                 )
-                dict_of_dataframes[name] = check_set_datatype(df, config, name)
-            logger.debug("\n%s\n", dict_of_dataframes[name])
 
         return dict_of_dataframes
 
+    def extract_set(
+        self, datafile_parser, name, config, dict_of_dataframes
+    ) -> pd.DataFrame:
+        data = datafile_parser[name].data
+        logger.debug(data)
+
+        indices = ["VALUE"]
+        df = pd.DataFrame(data=data, columns=indices, dtype=config[name]["dtype"])
+
+        return check_set_datatype(df, config, name)
+
+    def extract_param(
+        self, config, name, datafile_parser, dict_of_dataframes
+    ) -> pd.DataFrame:
+        indices = config[name]["indices"].copy()
+        indices_dtypes = [config[index]["dtype"] for index in indices]
+        indices.append("VALUE")
+        indices_dtypes.append("float")
+
+        raw_data = datafile_parser[name].data
+        data = self._convert_amply_data_to_list(raw_data)
+        df = pd.DataFrame(data=data, columns=indices)
+        try:
+            return check_datatypes(df, config, name)
+        except ValueError as ex:
+            msg = "Validation error when checking datatype of {}: {}".format(
+                name, str(ex)
+            )
+            raise ValueError(msg)
+
     def _convert_amply_data_to_list(self, amply_data: Dict) -> List[List]:
         """Flattens a dictionary into a list of lists
 


Test patch:
diff --git a/tests/test_read_strategies.py b/tests/test_read_strategies.py
index f6581a7..9f1c5c2 100644
--- a/tests/test_read_strategies.py
+++ b/tests/test_read_strategies.py
@@ -155,3 +155,23 @@ class TestReadDatafile:
         actual = read._load_parameter_definitions(config)
         expected = "set TestSet;\n"
         assert actual == expected
+
+    def test_catch_error_no_parameter(self, caplog):
+        """Fix for https://github.com/OSeMOSYS/otoole/issues/70 where parameter in
+        datafile but not in config causes error.  Instead, throw warning (and advise
+        that user should use a custom configuration).
+        """
+        read = ReadDatafile()
+        config = read.config
+        amply_datafile = amply = Amply(
+            """set REGION;
+            set TECHNOLOGY;
+            set MODE_OF_OPERATION;
+            set YEAR;"""
+        )
+        amply.load_string("""param ResultsPath := 'test_path';""")
+        read._convert_amply_to_dataframe(amply_datafile, config)
+        assert (
+            "Parameter ResultsPath could not be found in the configuration."
+            in caplog.text
+        )