Repository: OSeMOSYS/otoole

Issue:
Converting OSeMBE datafile (.txt) to excel (.xlsx) using Otoole - Windows 10
Hi,

I am using the modified version of otoole adapted for OSeMBE. My otoole version is v0.10.1b3. When I try to convert the OSeMBE datafile (in .txt format) to excel (.xlsx), using the following input line, it gives an error.

```otoole convert datafile excel osembe_data_ref.txt osembe_data_ref.xlsx``` 

See error message in the attached file. 
[Otoole debug.txt](https://github.com/OSeMOSYS/otoole/files/6576915/Otoole.debug.txt)

Operating system used: Windows

What is the best way to address this issue?

Discussion:

Patch:
diff --git a/.gitignore b/.gitignore
index c42d6b2..5754bed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,9 +13,22 @@ __pycache__/*
 .*.swp
 */.ipynb_checkpoints/*
 .DS_Store
-.vscode
+*.feather
+*.parquet
+*.kgrind
+*.sol
+*.lp
+*.txt
+*.csv
+datapackage.json
+*.ipynb
 
 # Project files
+*.srctrlbm
+*.srctrldb
+*.srctrlprj
+.vscode
+*.code-workspace
 .ropeproject
 .project
 .pydevproject
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a6ffaef..007b09a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@ exclude: '^docs/conf.py'
 
 repos:
 - repo: https://github.com/psf/black
-  rev: stable
+  rev: 21.5b1
   hooks:
     - id: black
       language_version: python3.8
@@ -26,11 +26,11 @@ repos:
     args: ['--max-line-length=88','--extend-ignore=E203']  # default of Black
 
 - repo: https://github.com/pre-commit/mirrors-isort
-  rev: v4.3.4
+  rev: v5.8.0
   hooks:
   - id: isort
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: master  # Use the sha / tag you want to point at
+    rev: v0.812  # Use the sha / tag you want to point at
     hooks:
     -   id: mypy
diff --git a/src/otoole/cli.py b/src/otoole/cli.py
index 9ee25ec..3308232 100644
--- a/src/otoole/cli.py
+++ b/src/otoole/cli.py
@@ -41,6 +41,7 @@ Ask for help on the command line::
 """
 import argparse
 import logging
+import os
 import sys
 
 from otoole import (
@@ -58,7 +59,7 @@ from otoole import (
     __version__,
 )
 from otoole.input import Context
-from otoole.utils import read_packaged_file
+from otoole.utils import _read_file, read_packaged_file
 from otoole.validate import main as validate
 from otoole.visualise import create_res
 
@@ -141,14 +142,22 @@ def conversion_matrix(args):
     read_strategy = None
     write_strategy = None
 
+    config = None
+    if args.config:
+        for filename in args.config:
+            _, ending = os.path.splitext(filename)
+            with open(filename, "r") as config_file:
+                config = _read_file(config_file, ending)
+            logger.info("Reading config from {}".format(filename))
+
     if args.from_format == "datafile":
-        read_strategy = ReadDatafile()
+        read_strategy = ReadDatafile(user_config=config)
     elif args.from_format == "datapackage":
-        read_strategy = ReadDatapackage()
+        read_strategy = ReadDatapackage(user_config=config)
     elif args.from_format == "csv":
-        read_strategy = ReadCsv()
+        read_strategy = ReadCsv(user_config=config)
     elif args.from_format == "excel":
-        read_strategy = ReadExcel()
+        read_strategy = ReadExcel(user_config=config)
 
     if args.to_format == "datapackage":
         write_strategy = WriteDatapackage()
@@ -232,6 +241,9 @@ def get_parser():
         "from_path", help="Path to file or folder to convert from"
     )
     convert_parser.add_argument("to_path", help="Path to file or folder to convert to")
+    convert_parser.add_argument(
+        "-c", "--config", action="append", help="Path to config YAML files"
+    )
     convert_parser.set_defaults(func=conversion_matrix)
 
     # Parser for validation
@@ -279,7 +291,7 @@ def main():
         if args.verbose:
             debug_hook(exception_type, exception, traceback)
         else:
-            print("{}: {}".format(exception_type.__name__, exception.message))
+            print("{}: {}".format(exception_type.__name__, str(exception)))
 
     sys.excepthook = exception_handler
 
diff --git a/src/otoole/input.py b/src/otoole/input.py
index 01bd354..2b34c99 100644
--- a/src/otoole/input.py
+++ b/src/otoole/input.py
@@ -42,7 +42,7 @@ from __future__ import annotations
 
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional, TextIO, Tuple, Union
+from typing import Any, Dict, List, Optional, TextIO, Tuple, Union
 
 import pandas as pd
 
@@ -138,18 +138,17 @@ class Strategy(ABC):
     def __init__(
         self, user_config: Optional[Dict] = None, results_config: Optional[Dict] = None
     ):
-        self._input_config = {}
+        self._input_config = {}  # type: Dict[str, Dict[str, Union[str, List[str]]]]
         self._results_config = {}
 
         if user_config:
-            self._input_config = user_config
+            self.input_config = user_config
         else:
-            self._input_config = self._read_config()
+            self.input_config = self._read_config()
         if results_config:
             self._results_config = results_config
         else:
             self._results_config = self._read_results_config()
-        self._input_config = self._add_dtypes(self._input_config)
 
     def _add_dtypes(self, config: Dict):
         for name, details in config.items():
@@ -170,9 +169,13 @@ class Strategy(ABC):
         return read_packaged_file("config.yaml", "otoole.results")
 
     @property
-    def input_config(self):
+    def input_config(self) -> Dict:
         return self._input_config
 
+    @input_config.setter
+    def input_config(self, value: Dict):
+        self._input_config = self._add_dtypes(value)
+
     @property
     def results_config(self):
         return self._results_config
@@ -310,7 +313,7 @@ class ReadStrategy(Strategy):
                 logger.debug(
                     "Column dtypes identified: {}".format(details["index_dtypes"])
                 )
-
+                logger.debug(df.head())
                 # Drop empty rows
                 df = (
                     df.dropna(axis=0, how="all")
diff --git a/src/otoole/preprocess/datapackage.json b/src/otoole/preprocess/datapackage.json
index 8f297e6..e6aabba 100644
--- a/src/otoole/preprocess/datapackage.json
+++ b/src/otoole/preprocess/datapackage.json
@@ -294,7 +294,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1354,7 +1354,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1616,7 +1616,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1860,7 +1860,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1926,7 +1926,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2206,7 +2206,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2315,7 +2315,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2493,7 +2493,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
diff --git a/src/otoole/preprocess/longify_data.py b/src/otoole/preprocess/longify_data.py
index 16a5d21..fd2e241 100644
--- a/src/otoole/preprocess/longify_data.py
+++ b/src/otoole/preprocess/longify_data.py
@@ -1,9 +1,9 @@
 """Read in a folder of irregular wide-format csv files and write them out as narrow csvs
 """
 import logging
-from typing import Dict
-
+import numpy as np
 import pandas as pd
+from typing import Dict
 
 logger = logging.getLogger()
 
@@ -30,13 +30,13 @@ def check_set_datatype(
 
 
 def check_datatypes(
-    narrow: pd.DataFrame, config_details: Dict, parameter: str
+    df: pd.DataFrame, config_details: Dict, parameter: str
 ) -> pd.DataFrame:
     """Checks a parameters datatypes
 
     Arguments
     ---------
-    narrow : pandas.DataFrame
+    df : pandas.DataFrame
         The parameter data
     config_details : dict
         The configuration dictionary
@@ -44,16 +44,18 @@ def check_datatypes(
         The name of the parameter
     """
     logger.info("Checking datatypes for %s", parameter)
+    logger.debug(df.columns)
     dtypes = {}
 
-    for column in narrow.columns:
+    for column in df.columns:
         if column == "VALUE":
             datatype = config_details[parameter]["dtype"]
             dtypes["VALUE"] = datatype
         else:
             datatype = config_details[column]["dtype"]
             dtypes[column] = datatype
-        if narrow[column].dtype != datatype:
+            logger.debug(f"Found {datatype} for column {column}")
+        if df[column].dtype != datatype:
             logger.info(
                 "dtype of column %s does not match %s for parameter %s",
                 column,
@@ -63,14 +65,15 @@ def check_datatypes(
             if datatype == "int":
                 dtypes[column] = "int64"
                 try:
-                    narrow[column] = narrow[column].apply(_cast_to_int)
+                    df[column] = df[column].apply(_cast_to_int)
                 except ValueError as ex:
                     msg = "Unable to apply datatype for column {}: {}".format(
                         column, str(ex)
                     )
                     raise ValueError(msg)
-    return narrow.astype(dtypes)
+
+    return df.astype(dtypes)
 
 
 def _cast_to_int(value):
-    return int(float(value))
+    return np.int64(float(value))
diff --git a/src/otoole/read_strategies.py b/src/otoole/read_strategies.py
index 21ac88f..12eea6c 100644
--- a/src/otoole/read_strategies.py
+++ b/src/otoole/read_strategies.py
@@ -9,6 +9,7 @@ from pandas_datapackage_reader import read_datapackage
 
 from otoole.input import ReadStrategy
 from otoole.preprocess.longify_data import check_datatypes, check_set_datatype
+from otoole.utils import read_datapackage_schema_into_config
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +67,6 @@ class _ReadTabular(ReadStrategy):
         logger.debug("Expected headers for %s: %s", name, expected_headers)
 
         if "REGION" in expected_headers and "REGION" not in actual_headers:
-            logger.error("No 'REGION' column provided for %s", name)
             raise ValueError("No REGION column provided for %s", name)
 
         if "MODEOFOPERATION" in actual_headers:
@@ -198,6 +198,9 @@ class ReadDatapackage(ReadStrategy):
         inputs = read_datapackage(filepath)
         default_resource = inputs.pop("default_values").set_index("name").to_dict()
         default_values = default_resource["default_value"]
+        self.input_config = read_datapackage_schema_into_config(
+            filepath, default_values
+        )
         inputs = self._check_index(inputs)
         return inputs, default_values
 
diff --git a/src/otoole/results/config.yaml b/src/otoole/results/config.yaml
index 4abe610..c4ba2f8 100644
--- a/src/otoole/results/config.yaml
+++ b/src/otoole/results/config.yaml
@@ -172,3 +172,6 @@ UseByTechnology:
     dtype: float
     default: 0
     calculated: False
+_REGION:
+    type: set
+    dtype: str
diff --git a/src/otoole/results/results.py b/src/otoole/results/results.py
index c181944..1e8054c 100644
--- a/src/otoole/results/results.py
+++ b/src/otoole/results/results.py
@@ -1,8 +1,8 @@
 import logging
-from abc import abstractmethod
-from typing import Any, Dict, List, Optional, Set, TextIO, Tuple, Union
-
 import pandas as pd
+from abc import abstractmethod
+from io import StringIO
+from typing import Any, Dict, List, Set, TextIO, Tuple, Union
 
 from otoole.input import ReadStrategy
 from otoole.preprocess.longify_data import check_datatypes
@@ -35,30 +35,62 @@ class ReadResults(ReadStrategy):
         else:
             input_data = None
 
-        cbc = self._convert_to_dataframe(filepath)
-        results = self._convert_dataframe_to_csv(cbc, input_data)
-        default_values = self._read_default_values(self.results_config)
+        available_results = self.get_results_from_file(
+            filepath, input_data
+        )  # type: Dict[str, pd.DataFrame]
+        results = self.calculate_results(
+            available_results, input_data
+        )  # type: Dict[str, pd.DataFrame]
+        default_values = self._read_default_values(self.results_config)  # type: Dict
         return results, default_values
 
     @abstractmethod
-    def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
+    def get_results_from_file(self, filepath, input_data):
         raise NotImplementedError()
 
-    def _convert_dataframe_to_csv(
-        self, data: pd.DataFrame, input_data: Optional[Dict[str, pd.DataFrame]] = None
+    def calculate_results(
+        self,
+        available_results: Dict[str, pd.DataFrame],
+        input_data: Dict[str, pd.DataFrame],
     ) -> Dict[str, pd.DataFrame]:
-        """Convert from dataframe to long format
+        """Populates the results with calculated values using input data"""
+
+        results = {}
+        results_package = ResultsPackage(available_results, input_data)
+
+        for name in sorted(self.results_config.keys()):
+
+            LOGGER.info("Looking for %s", name)
+
+            try:
+                results[name] = results_package[name]
+            except KeyError as ex:
+                LOGGER.info("No calculation method available for %s", name)
+                LOGGER.debug("Error calculating %s: %s", name, str(ex))
+
+        return results
+
+
+class ReadResultsCBC(ReadResults):
+    def get_results_from_file(self, filepath, input_data):
+        cbc = self._convert_to_dataframe(filepath)
+        available_results = self._convert_wide_to_long(cbc)
+        return available_results
+
+    @abstractmethod
+    def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
+        raise NotImplementedError()
+
+    def _convert_wide_to_long(self, data: pd.DataFrame) -> Dict[str, pd.DataFrame]:
+        """Convert from wide to long format
 
         Converts a pandas DataFrame containing all CBC results to reformatted
-        dictionary of pandas DataFrames in long format ready to write out as
-        csv files
+        dictionary of pandas DataFrames in long format ready to write out
 
         Arguments
         ---------
         data : pandas.DataFrame
             CBC results stored in a dataframe
-        input_data_path : str, default=None
-            Path to the OSeMOSYS data file containing input data
 
         Example
         -------
@@ -71,16 +103,13 @@ class ReadResults(ReadStrategy):
                                     0  SIMPLICITY  2015  187.01576
                                     1  SIMPLICITY  2016  183.30788}
         """
-        input_config = self.input_config
-        results_config = self.results_config
 
-        sets = {x: y for x, y in input_config.items() if y["type"] == "set"}
+        sets = {x: y for x, y in self.input_config.items() if y["type"] == "set"}
 
         results = {}  # type: Dict[str, pd.DataFrame]
-
         not_found = []
 
-        for name, details in sorted(results_config.items()):
+        for name, details in sorted(self.results_config.items()):
             df = data[data["Variable"] == name]
 
             if not df.empty:
@@ -102,115 +131,138 @@ class ReadResults(ReadStrategy):
                 df = df[columns]
 
                 index = details["indices"].copy()
-                # catches pandas error when there are duplicate column indices
-                if check_duplicate_index(index):
-                    index = rename_duplicate_column(index)
-                    LOGGER.debug("Original column names: %s", columns)
-                    renamed_columns = rename_duplicate_column(columns)
-                    LOGGER.debug("New column names: %s", renamed_columns)
-                    df.columns = renamed_columns
+                df, index = check_duplicate_index(df, columns, index)
                 results[name] = df.set_index(index)
             else:
                 not_found.append(name)
 
         LOGGER.debug("Unable to find result variables for: %s", ", ".join(not_found))
 
-        results_package = ResultsPackage(results, input_data)
+        return results
 
-        for name in not_found:
 
-            LOGGER.info("Looking for %s", name)
-            details = results_config[name]
+def check_duplicate_index(df: pd.DataFrame, columns: List, index: List) -> pd.DataFrame:
+    """Catches pandas error when there are duplicate column indices"""
+    if check_for_duplicates(index):
+        index = rename_duplicate_column(index)
+        LOGGER.debug("Original column names: %s", columns)
+        renamed_columns = rename_duplicate_column(columns)
+        LOGGER.debug("New column names: %s", renamed_columns)
+        df.columns = renamed_columns
+    return df, index
 
-            try:
-                df = results_package[name]
-            except KeyError as ex:
-                LOGGER.info("No calculation method available for %s", name)
-                LOGGER.debug("Error calculating %s: %s", name, str(ex))
-                df = pd.DataFrame()
 
-            if not df.empty:
-                results[name] = df
-            else:
-                LOGGER.warning(
-                    "Calculation returned empty dataframe for parameter '%s'", name
-                )
+def check_for_duplicates(index: List) -> bool:
+    return len(set(index)) != len(index)
 
-        return results
 
+def identify_duplicate(index: List) -> Union[int, bool]:
+    elements = set()  # type: Set
+    for counter, elem in enumerate(index):
+        if elem in elements:
+            return counter
+        else:
+            elements.add(elem)
+    return False
 
-class ReadCplex(ReadStrategy):
-    """
-    """
 
-    def read(
-        self, filepath: Union[str, TextIO], **kwargs
-    ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
-        data = {}  # type: Dict
+def rename_duplicate_column(index: List) -> List:
+    column = index.copy()
+    location = identify_duplicate(column)
+    if location:
+        column[location] = "_" + column[location]
+    return column
 
-        if "input_data" in kwargs:
-            input_data = kwargs["input_data"]
+
+class ReadCplex(ReadResults):
+    """ """
+
+    def get_results_from_file(
+        self, filepath: Union[str, TextIO], input_data
+    ) -> Dict[str, pd.DataFrame]:
+
+        if input_data:
             years = input_data["YEAR"].values  # type: List
             start_year = int(years[0])
             end_year = int(years[-1])
         else:
             raise RuntimeError("To process CPLEX results please provide the input file")
 
-        for linenum, line in enumerate(filepath):
+        if isinstance(filepath, str):
+            with open(filepath, "r") as sol_file:
+                data = self.extract_rows(sol_file, start_year, end_year)
+        elif isinstance(filepath, StringIO):
+            data = self.extract_rows(filepath, start_year, end_year)
+        else:
+            raise TypeError("Argument filepath type must be a string or an open file")
+
+        results = {}
+
+        for name in data.keys():
+            results[name] = self.convert_df(data[name], name, start_year, end_year)
+
+        return results
+
+    def extract_rows(
+        self, sol_file: TextIO, start_year: int, end_year: int
+    ) -> Dict[str, List[List[str]]]:
+        """ """
+        data = {}  # type: Dict[str, List[List[str]]]
+        for linenum, line in enumerate(sol_file):
+            line = line.replace("\n", "")
             try:
-                row_as_list = line.split("\t")
-                name, df = self.convert_df(row_as_list, start_year, end_year,)
+                row_as_list = line.split("\t")  # type: List[str]
+                name = row_as_list[0]  # type: str
 
-                if name in data:
-                    data[name] = data[name].append(df)
+                if name in data.keys():
+                    data[name].append(row_as_list)
                 else:
-                    data[name] = [df]
-
+                    data[name] = [row_as_list]
             except ValueError as ex:
                 msg = "Error caused at line {}: {}. {}"
                 raise ValueError(msg.format(linenum, line, ex))
-
-        results = {}
-
-        for name, contents in data.items():
-            results[name] = pd.concat(contents)
-
-        return results, self._read_default_values(self.results_config)
+        return data
 
     def extract_variable_dimensions_values(self, data: List) -> Tuple[str, Tuple, List]:
-        """Extracts useful information from a line of a results file
-        """
+        """Extracts useful information from a line of a results file"""
         variable = data[0]
-        number = len(self.results_config[variable]["indices"])
+        try:
+            number = len(self.results_config[variable]["indices"])
+        except KeyError as ex:
+            print(data)
+            raise KeyError(ex)
         dimensions = tuple(data[1:(number)])
         values = data[(number):]
         return (variable, dimensions, values)
 
     def convert_df(
-        self, row_as_list: List, start_year: int, end_year: int
-    ) -> Tuple[str, pd.DataFrame]:
-        """Read the cplex line into a pandas DataFrame
-
-        """
-        variable, dimensions, values = self.extract_variable_dimensions_values(
-            row_as_list
-        )
+        self, data: List[List[str]], variable: str, start_year: int, end_year: int
+    ) -> pd.DataFrame:
+        """Read the cplex lines into a pandas DataFrame"""
         index = self.results_config[variable]["indices"]
-        columns = index[:-1] + list(range(start_year, end_year + 1, 1))
-        df = pd.DataFrame(data=[list(dimensions) + values], columns=columns).set_index(
-            index[:-1]
+        columns = ["variable"] + index[:-1] + list(range(start_year, end_year + 1, 1))
+        df = pd.DataFrame(data=data, columns=columns)
+        df, index = check_duplicate_index(df, columns, index)
+        df = df.drop(columns="variable")
+
+        LOGGER.debug(
+            f"Attempting to set index for {variable} with columns {index[:-1]}"
         )
+        try:
+            df = df.set_index(index[:-1])
+        except NotImplementedError as ex:
+            LOGGER.error(f"Error setting index for {df.head()}")
+            raise NotImplementedError(ex)
         df = df.melt(var_name="YEAR", value_name="VALUE", ignore_index=False)
         df = df.reset_index()
         df = check_datatypes(df, {**self.input_config, **self.results_config}, variable)
         df = df.set_index(index)
         df = df[(df != 0).any(axis=1)]
-        return (variable, df)
+        return df
 
 
-class ReadGurobi(ReadResults):
-    """Read a Gurobi solution file into memory
-    """
+class ReadGurobi(ReadResultsCBC):
+    """Read a Gurobi solution file into memory"""
 
     def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
         """Reads a Gurobi solution file into a pandas DataFrame
@@ -220,7 +272,11 @@ class ReadGurobi(ReadResults):
         file_path : str
         """
         df = pd.read_csv(
-            file_path, header=None, sep=" ", names=["Variable", "Value"], skiprows=2,
+            file_path,
+            header=None,
+            sep=" ",
+            names=["Variable", "Value"],
+            skiprows=2,
         )  # type: pd.DataFrame
         df[["Variable", "Index"]] = df["Variable"].str.split("(", expand=True)
         df["Index"] = df["Index"].str.replace(")", "")
@@ -229,7 +285,7 @@ class ReadGurobi(ReadResults):
         return df[["Variable", "Index", "Value"]].astype({"Value": float})
 
 
-class ReadCbc(ReadResults):
+class ReadCbc(ReadResultsCBC):
     """Read a CBC solution file into memory
 
     Arguments
@@ -267,25 +323,3 @@ class ReadCbc(ReadResults):
         df["Index"] = df["Index"].str.replace(")", "")
         df = df.drop(columns=["indexvalue"])
         return df[["Variable", "Index", "Value"]].astype({"Value": float})
-
-
-def check_duplicate_index(index: List) -> bool:
-    return len(set(index)) != len(index)
-
-
-def identify_duplicate(index: List) -> Union[int, bool]:
-    elements = set()  # type: Set
-    for counter, elem in enumerate(index):
-        if elem in elements:
-            return counter
-        else:
-            elements.add(elem)
-    return False
-
-
-def rename_duplicate_column(index: List) -> List:
-    column = index.copy()
-    location = identify_duplicate(column)
-    if location:
-        column[location] = "_" + column[location]
-    return column
diff --git a/src/otoole/utils.py b/src/otoole/utils.py
index efb4908..99b8f14 100644
--- a/src/otoole/utils.py
+++ b/src/otoole/utils.py
@@ -1,4 +1,6 @@
+import json
 import os
+from typing import Dict, List, Union
 
 from datapackage import Package
 from sqlalchemy import create_engine
@@ -14,6 +16,8 @@ except ImportError:
 def _read_file(open_file, ending):
     if ending == ".yaml" or ending == ".yml":
         contents = load(open_file, Loader=SafeLoader)
+    elif ending == ".json":
+        contents = json.load(open_file)
     else:
         contents = open_file.readlines()
     return contents
@@ -49,3 +53,47 @@ def read_datapackage(filepath: str, sql: bool = False):
         package = Package(filepath)
 
     return package
+
+
+def read_datapackage_schema_into_config(
+    filepath: str, default_values: Dict
+) -> Dict[str, Dict[str, Union[str, List]]]:
+    with open(filepath, "r") as json_file:
+        _, ending = os.path.splitext(filepath)
+        schema = _read_file(json_file, ending)
+    return extract_config(schema, default_values)
+
+
+def extract_config(
+    schema: Dict, default_values: Dict
+) -> Dict[str, Dict[str, Union[str, List[str]]]]:
+
+    config = {}  # type: Dict[str, Dict[str, Union[str, List[str]]]]
+    for resource in schema["resources"]:
+
+        name = resource["name"]
+        if name == "default_values":
+            continue
+
+        dtype_mapping = {
+            "number": "float",
+            "string": "str",
+            "float": "float",
+            "integer": "int",
+        }
+
+        fields = resource["schema"]["fields"]
+        dtype = [x["type"] for x in fields if x["name"] == "VALUE"][0]
+        if (len(fields) == 1) & (fields[0]["name"] == "VALUE"):
+            element_type = "set"
+            config[name] = {"dtype": dtype_mapping[dtype], "type": element_type}
+        else:
+            element_type = "param"
+            indices = [x["name"] for x in fields if x["name"] != "VALUE"]
+            config[name] = {
+                "type": element_type,
+                "indices": indices,
+                "dtype": dtype_mapping[dtype],
+                "default": default_values[name],
+            }
+    return config
diff --git a/src/otoole/write_strategies.py b/src/otoole/write_strategies.py
index c11a26f..79bdb17 100644
--- a/src/otoole/write_strategies.py
+++ b/src/otoole/write_strategies.py
@@ -1,8 +1,8 @@
 import logging
 import os
-from typing import Any, TextIO
-
 import pandas as pd
+from json import dump
+from typing import Any, TextIO
 
 from otoole.input import WriteStrategy
 from otoole.read_strategies import CSV_TO_EXCEL
@@ -33,34 +33,34 @@ class WriteExcel(WriteStrategy):
 
         if not df.empty:
 
-            names = df.columns.to_list()
-            if len(names) > 2:
+            index_names = df.index.names
+            column_names = df.columns.to_list()
+            if index_names[0]:
+                names = index_names + column_names
+            else:
+                names = column_names
+            logger.debug(f"Identified {len(names)} names: {names}")
+
+            total_columns = len(names)
+
+            if total_columns > 3:
                 logger.debug(
-                    "More than 2 columns for {}: {}".format(parameter_name, names)
+                    "More than 3 columns for {}: {}".format(parameter_name, names)
                 )
                 rows = names[0:-2]
                 columns = names[-2]
                 values = names[-1]
-                logger.debug("Rows: %s; columns: %s; values: %s", rows, columns, values)
+                logger.debug(f"Rows: {rows}; columns: {columns}; values: {values}")
                 logger.debug("dtypes: {}".format(df.dtypes))
-                pivot = pd.pivot_table(
-                    df, index=rows, columns=columns, values=values, fill_value=default
-                )
-            elif len(names) == 2:
-                logger.debug("Two columns for {}: {}".format(parameter_name, names))
-                rows = names[0]
-                values = names[1]
-                logger.debug("Rows: %s; values: %s", rows, values)
-                pivot = pd.pivot_table(
-                    df, index=rows, values=values, fill_value=default
+                pivot = df.reset_index().pivot(
+                    index=rows, columns=columns, values=values
                 )
             else:
-                logger.debug("One column for {}: {}".format(parameter_name, names))
+                logger.debug(f"One column for {parameter_name}: {names}")
                 pivot = df.copy()
-                pivot = pivot.reset_index(drop=True)
 
         else:
-            logger.debug("Dataframe {} is empty".format(parameter_name))
+            logger.debug(f"Dataframe {parameter_name} is empty")
             pivot = df.copy()
 
         return pivot
@@ -77,9 +77,10 @@ class WriteExcel(WriteStrategy):
         except KeyError:
             name = parameter_name
         df = self._form_parameter(df, parameter_name, default)
-        df.to_excel(handle, sheet_name=name, merge_cells=False)
+        df.to_excel(handle, sheet_name=name, merge_cells=False, index=True)
 
     def _write_set(self, df: pd.DataFrame, set_name, handle: pd.ExcelWriter):
+        df = df.reset_index()
         df.to_excel(handle, sheet_name=set_name, merge_cells=False, index=False)
 
     def _footer(self, handle=pd.ExcelWriter):
@@ -95,6 +96,7 @@ class WriteDatafile(WriteStrategy):
 
     def _form_parameter(self, df: pd.DataFrame, default: float):
 
+        # Don't write out values equal to the default value
         df = df[df.VALUE != default]
         return df
 
@@ -113,7 +115,9 @@ class WriteDatafile(WriteStrategy):
         """
         df = self._form_parameter(df, default)
         handle.write("param default {} : {} :=\n".format(default, parameter_name))
-        df.to_csv(path_or_buf=handle, sep=" ", header=False, index=True)
+        df.to_csv(
+            path_or_buf=handle, sep=" ", header=False, index=True, float_format="%g"
+        )
         handle.write(";\n")
 
     def _write_set(self, df: pd.DataFrame, set_name, handle: TextIO):
@@ -126,7 +130,9 @@ class WriteDatafile(WriteStrategy):
         handle: TextIO
         """
         handle.write("set {} :=\n".format(set_name))
-        df.to_csv(path_or_buf=handle, sep=" ", header=False, index=False)
+        df.to_csv(
+            path_or_buf=handle, sep=" ", header=False, index=False, float_format="%g"
+        )
         handle.write(";\n")
 
     def _footer(self, handle: TextIO):
@@ -187,7 +193,7 @@ class WriteDatapackage(WriteCsv):
         os.makedirs(os.path.join(self.filepath, "data"), exist_ok=True)
         return None
 
-    def _write_out_dataframe(self, folder, parameter, df):
+    def _write_out_dataframe(self, folder, parameter, df, index=False):
         """Writes out a dataframe as a csv into the data subfolder of a datapackage
 
         Arguments
@@ -202,13 +208,13 @@ class WriteDatapackage(WriteCsv):
             logger.info(
                 "Writing %s rows into narrow file for %s", df.shape[0], parameter
             )
-            df.to_csv(csvfile, index=True)
+            df.to_csv(csvfile, index=index)
 
     def _footer(self, handle: TextIO):
         datapackage = read_packaged_file("datapackage.json", "otoole.preprocess")
         filepath = os.path.join(self.filepath, "datapackage.json")
         with open(filepath, "w", newline="") as destination:
-            destination.writelines(datapackage)
+            dump(datapackage, destination)
         self._write_default_values()
 
     def _write_default_values(self):


Test patch:
diff --git a/tests/test_convert.py b/tests/test_convert.py
index ab051d4..14a494f 100644
--- a/tests/test_convert.py
+++ b/tests/test_convert.py
@@ -52,7 +52,7 @@ class TestConvert:
 
         assert actual[-1] == b"end;\n"
         assert actual[0] == b"# Model file written by *otoole*\n"
-        assert actual[2] == b"09_ROK d_bld_2_coal_products 2017 20.892132\n"
+        assert actual[2] == b"09_ROK d_bld_2_coal_products 2017 20.8921\n"
         assert actual[8996] == b"param default 1 : DepreciationMethod :=\n"
 
     def test_convert_excel_to_csv(self):
diff --git a/tests/test_read_strategies.py b/tests/test_read_strategies.py
index f368c08..d1cb4ab 100644
--- a/tests/test_read_strategies.py
+++ b/tests/test_read_strategies.py
@@ -1,19 +1,19 @@
-import os
-from io import StringIO
-from textwrap import dedent
-from typing import List
-
 from pytest import mark
 
+import os
 import pandas as pd
 from amply import Amply
+from io import StringIO
+from textwrap import dedent
+from typing import List
 
 from otoole import ReadDatafile, ReadExcel, ReadMemory
+from otoole.preprocess.longify_data import check_datatypes
 from otoole.results.results import (
     ReadCbc,
     ReadCplex,
     ReadGurobi,
-    check_duplicate_index,
+    check_for_duplicates,
     identify_duplicate,
     rename_duplicate_column,
 )
@@ -25,43 +25,46 @@ class TestReadCplex:
     cplex_short = "AnnualFixedOperatingCost	REGION	CDBACKSTOP	0.0	0.0	137958.8400384134	305945.38410619126	626159.9611543404	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0"
     cplex_long = "RateOfActivity	REGION	S1D1	CGLFRCFURX	1	0.0	0.0	0.0	0.0	0.0	0.3284446367303371	0.3451714779880536	0.3366163200621617	0.3394945166233896	0.3137488154250392	0.28605725055560716	0.2572505015401749	0.06757558148965725	0.0558936625751148	0.04330608461292407	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0"
 
-    cplex_mid_short = (
-        "AnnualFixedOperatingCost",
+    cplex_mid_empty = (
         pd.DataFrame(
-            data=[
-                ["REGION", "CDBACKSTOP", 2017, 137958.8400384134],
-                ["REGION", "CDBACKSTOP", 2018, 305945.38410619126],
-                ["REGION", "CDBACKSTOP", 2019, 626159.9611543404],
-            ],
+            data=[],
             columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
-        ).set_index(["REGION", "TECHNOLOGY", "YEAR"]),
+        )
+        .astype({"VALUE": float})
+        .set_index(["REGION", "TECHNOLOGY", "YEAR"])
     )
 
-    cplex_mid_long = (
-        "RateOfActivity",
-        pd.DataFrame(
-            data=[
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2020, 0.3284446367303371],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2021, 0.3451714779880536],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2022, 0.3366163200621617],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2023, 0.3394945166233896],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2024, 0.3137488154250392],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2025, 0.28605725055560716],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2026, 0.2572505015401749],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2027, 0.06757558148965725],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2028, 0.0558936625751148],
-                ["REGION", "S1D1", "CGLFRCFURX", 1, 2029, 0.04330608461292407],
-            ],
-            columns=[
-                "REGION",
-                "TIMESLICE",
-                "TECHNOLOGY",
-                "MODE_OF_OPERATION",
-                "YEAR",
-                "VALUE",
-            ],
-        ).set_index(["REGION", "TIMESLICE", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR"]),
-    )
+    cplex_mid_short = pd.DataFrame(
+        data=[
+            ["REGION", "CDBACKSTOP", 2017, 137958.8400384134],
+            ["REGION", "CDBACKSTOP", 2018, 305945.38410619126],
+            ["REGION", "CDBACKSTOP", 2019, 626159.9611543404],
+        ],
+        columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
+    ).set_index(["REGION", "TECHNOLOGY", "YEAR"])
+
+    cplex_mid_long = pd.DataFrame(
+        data=[
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2020, 0.3284446367303371],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2021, 0.3451714779880536],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2022, 0.3366163200621617],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2023, 0.3394945166233896],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2024, 0.3137488154250392],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2025, 0.28605725055560716],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2026, 0.2572505015401749],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2027, 0.06757558148965725],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2028, 0.0558936625751148],
+            ["REGION", "S1D1", "CGLFRCFURX", 1, 2029, 0.04330608461292407],
+        ],
+        columns=[
+            "REGION",
+            "TIMESLICE",
+            "TECHNOLOGY",
+            "MODE_OF_OPERATION",
+            "YEAR",
+            "VALUE",
+        ],
+    ).set_index(["REGION", "TIMESLICE", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR"])
 
     dataframe_short = {
         "AnnualFixedOperatingCost": pd.DataFrame(
@@ -132,22 +135,90 @@ class TestReadCplex:
             data, _ = cplex_reader.read(file_buffer, input_data=input_data)
         assert "AnnualFixedOperatingCost" in data
         expected = (
-            pd.DataFrame(columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],)
-            .astype({"VALUE": float, "YEAR": int})
+            pd.DataFrame(
+                [],
+                columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
+            )
+            .astype({"REGION": str, "VALUE": float, "YEAR": int})
             .set_index(["REGION", "TECHNOLOGY", "YEAR"])
         )
         actual = data["AnnualFixedOperatingCost"]
-        pd.testing.assert_frame_equal(actual, expected)
+        pd.testing.assert_frame_equal(actual, expected, check_index_type=False)
 
-    test_data_mid = [(cplex_short, cplex_mid_short), (cplex_long, cplex_mid_long)]
+    test_data_to_cplex = [
+        (cplex_empty, cplex_mid_empty),
+        (cplex_short, cplex_mid_short),
+        (cplex_long, cplex_mid_long),
+    ]
 
-    @mark.parametrize("cplex_input,expected", test_data_mid, ids=["short", "long"])
+    @mark.parametrize(
+        "cplex_input,expected", test_data_to_cplex, ids=["empty", "short", "long"]
+    )
     def test_convert_cplex_to_df(self, cplex_input, expected):
 
         data = cplex_input.split("\t")
+        variable = data[0]
         cplex_reader = ReadCplex()
-        actual = cplex_reader.convert_df(data, 2015, 2070)
-        pd.testing.assert_frame_equal(actual[1], expected[1])
+        actual = cplex_reader.convert_df([data], variable, 2015, 2070)
+        pd.testing.assert_frame_equal(actual, expected, check_index_type=False)
+
+    def test_convert_lines_to_df_empty(self):
+
+        data = [
+            [
+                "AnnualFixedOperatingCost",
+                "REGION",
+                "AOBACKSTOP",
+                "0",
+                "0",
+                "0",
+                "0",
+                "0",
+                "0",
+                "0",
+                "0",
+                "0",
+            ]
+        ]
+        variable = "AnnualFixedOperatingCost"
+        cplex_reader = ReadCplex()
+        actual = cplex_reader.convert_df(data, variable, 2015, 2023)
+        pd.testing.assert_frame_equal(
+            actual,
+            pd.DataFrame(
+                data=[],
+                columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
+            )
+            .astype({"REGION": str, "TECHNOLOGY": str, "YEAR": int, "VALUE": float})
+            .set_index(["REGION", "TECHNOLOGY", "YEAR"]),
+            check_index_type=False,
+        )
+
+    def test_check_datatypes_with_empty(self):
+
+        df = pd.DataFrame(data=[], columns=["REGION", "FUEL", "YEAR", "VALUE"])
+
+        parameter = "AccumulatedAnnualDemand"
+
+        config_dict = {
+            "AccumulatedAnnualDemand": {
+                "indices": ["REGION", "FUEL", "YEAR"],
+                "type": "param",
+                "dtype": float,
+                "default": 0,
+            },
+            "REGION": {"dtype": "str", "type": "set"},
+            "FUEL": {"dtype": "str", "type": "set"},
+            "YEAR": {"dtype": "int", "type": "set"},
+        }
+
+        actual = check_datatypes(df, config_dict, parameter)
+
+        expected = pd.DataFrame(
+            data=[], columns=["REGION", "FUEL", "YEAR", "VALUE"]
+        ).astype({"REGION": str, "FUEL": str, "YEAR": int, "VALUE": float})
+
+        pd.testing.assert_frame_equal(actual, expected, check_index_type=False)
 
 
 class TestReadGurobi:
@@ -258,7 +329,12 @@ class TestReadCbc:
         columns=["REGION", "_REGION", "TIMESLICE", "FUEL", "YEAR", "VALUE"],
     ).set_index(["REGION", "_REGION", "TIMESLICE", "FUEL", "YEAR"])
 
-    test_data = [(cbc_data, otoole_data,)]
+    test_data = [
+        (
+            cbc_data,
+            otoole_data,
+        )
+    ]
 
     @mark.parametrize("cbc_input,expected", test_data)
     def test_read_cbc_to_otoole_dataframe(self, cbc_input, expected):
@@ -278,7 +354,7 @@ class TestReadCbc:
             ],
             columns=["Variable", "Index", "Value"],
         )
-        actual = ReadCbc()._convert_dataframe_to_csv(prelim_data, {})["Trade"]
+        actual = ReadCbc()._convert_wide_to_long(prelim_data)["Trade"]
         pd.testing.assert_frame_equal(actual, self.otoole_data)
 
     test_data_4 = [
@@ -289,7 +365,7 @@ class TestReadCbc:
 
     @mark.parametrize("data,expected", test_data_4)
     def test_handle_duplicate_indices(self, data, expected):
-        assert check_duplicate_index(data) is expected
+        assert check_for_duplicates(data) is expected
 
     test_data_5 = [
         (["REGION", "REGION", "TIMESLICE", "FUEL", "YEAR"], 1),
@@ -358,7 +434,7 @@ class TestReadCbc:
 
     test_data_2 = [
         # First case
-        (total_cost_cbc_mid, {}, total_cost_otoole_df),
+        (total_cost_cbc_mid, total_cost_otoole_df),
         # Second case
         (
             pd.DataFrame(
@@ -369,7 +445,6 @@ class TestReadCbc:
                 ],
                 columns=["Variable", "Index", "Value"],
             ),
-            {},
             {
                 "AnnualEmissions": pd.DataFrame(
                     data=[
@@ -384,13 +459,13 @@ class TestReadCbc:
     ]  # type: List
 
     @mark.parametrize(
-        "results,cbc_input,expected",
+        "results,expected",
         test_data_2,
         ids=["TotalDiscountedCost", "AnnualEmissions1"],
     )
-    def test_convert_cbc_to_csv_long(self, results, cbc_input, expected):
+    def test_convert_cbc_to_csv_long(self, results, expected):
         cbc_reader = ReadCbc()
-        actual = cbc_reader._convert_dataframe_to_csv(results, cbc_input)
+        actual = cbc_reader._convert_wide_to_long(results)
         assert isinstance(actual, dict)
         for name, df in actual.items():
             pd.testing.assert_frame_equal(df, expected[name])
@@ -398,7 +473,9 @@ class TestReadCbc:
     test_data_3 = [(total_cost_cbc, {}, total_cost_otoole_df)]  # type: List
 
     @mark.parametrize(
-        "cbc_solution,input_data,expected", test_data_3, ids=["TotalDiscountedCost"],
+        "cbc_solution,input_data,expected",
+        test_data_3,
+        ids=["TotalDiscountedCost"],
     )
     def test_convert_cbc_to_csv_long_read(self, cbc_solution, input_data, expected):
         cbc_reader = ReadCbc()
@@ -409,24 +486,35 @@ class TestReadCbc:
         assert isinstance(actual, pd.DataFrame)
         pd.testing.assert_frame_equal(actual, expected["TotalDiscountedCost"])
 
-    def test_convert_cbc_to_csv_short(self):
-        cbc_results = pd.DataFrame(
-            data=[
-                ["RateOfActivity", "SIMPLICITY,ID,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,IN,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,SD,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,SN,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,WD,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,WN,GAS_EXTRACTION,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,ID,DUMMY,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,IN,DUMMY,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,SD,DUMMY,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,SN,DUMMY,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,WD,DUMMY,1,2014", 1],
-                ["RateOfActivity", "SIMPLICITY,WN,DUMMY,1,2014", 1],
-            ],
-            columns=["Variable", "Index", "Value"],
-        )
+    def test_calculate_results(self):
+        cbc_results = {
+            "RateOfActivity": pd.DataFrame(
+                data=[
+                    ["SIMPLICITY", "GAS_EXTRACTION", "ID", 1, 2014, 1],
+                    ["SIMPLICITY", "GAS_EXTRACTION", "IN", 1, 2014, 1],
+                    ["SIMPLICITY", "GAS_EXTRACTION", "SD", 1, 2014, 1],
+                    ["SIMPLICITY", "GAS_EXTRACTION", "SN", 1, 2014, 1],
+                    ["SIMPLICITY", "GAS_EXTRACTION", "WD", 1, 2014, 1],
+                    ["SIMPLICITY", "GAS_EXTRACTION", "WN", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "ID", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "IN", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "SD", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "SN", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "WD", 1, 2014, 1],
+                    ["SIMPLICITY", "DUMMY", "WN", 1, 2014, 1],
+                ],
+                columns=[
+                    "REGION",
+                    "TECHNOLOGY",
+                    "TIMESLICE",
+                    "MODE_OF_OPERATION",
+                    "YEAR",
+                    "VALUE",
+                ],
+            ).set_index(
+                ["REGION", "TECHNOLOGY", "TIMESLICE", "MODE_OF_OPERATION", "YEAR"]
+            )
+        }
         input_data = {
             "EmissionActivityRatio": pd.DataFrame(
                 data=[["SIMPLICITY", "GAS_EXTRACTION", "CO2", 1, 2014, 1.0]],
@@ -460,7 +548,7 @@ class TestReadCbc:
         ).set_index(["REGION", "EMISSION", "YEAR"])
 
         cbc_reader = ReadCbc()
-        actual = cbc_reader._convert_dataframe_to_csv(cbc_results, input_data)
+        actual = cbc_reader.calculate_results(cbc_results, input_data)
         assert isinstance(actual, dict)
         pd.testing.assert_frame_equal(actual["AnnualEmissions"], expected)
 
@@ -511,8 +599,7 @@ class TestReadCbc:
 
 
 class TestCleanOnRead:
-    """Tests that a datapackage is cleaned and indexed upon reading
-    """
+    """Tests that a datapackage is cleaned and indexed upon reading"""
 
     def test_index_dtypes_available(self):
         reader = ReadMemory({})
@@ -763,8 +850,7 @@ class TestReadDatafile:
 
 class TestReadExcel:
     def test_read_excel_yearsplit(self):
-        """
-        """
+        """ """
         spreadsheet = os.path.join("tests", "fixtures", "combined_inputs.xlsx")
         reader = ReadExcel()
         actual, _ = reader.read(spreadsheet)
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 0000000..cf42ad6
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,19 @@
+import pytest
+
+from unittest.mock import MagicMock
+
+from otoole.utils import extract_config, read_packaged_file
+
+
+class TestDataPackageSchema:
+    @pytest.mark.xfail
+    def test_read_datapackage_schema_into_config(self):
+
+        schema = read_packaged_file("datapackage.json", "otoole.preprocess")
+        mock = MagicMock()
+        mock.__getitem__.return_value = 0
+
+        actual = extract_config(schema, mock)
+
+        expected = read_packaged_file("config.yaml", "otoole.preprocess")
+        assert actual == expected
diff --git a/tests/test_write_strategies.py b/tests/test_write_strategies.py
index 3ad201c..71ec736 100644
--- a/tests/test_write_strategies.py
+++ b/tests/test_write_strategies.py
@@ -1,5 +1,4 @@
 import io
-
 import pandas as pd
 
 from otoole.write_strategies import WriteDatafile, WriteExcel
@@ -12,18 +11,24 @@ class TestWriteExcel:
 
         data = []
 
-        df = pd.DataFrame(data=data, columns=["REGION", "FUEL", "VALUE"])
+        df = pd.DataFrame(data=data, columns=["REGION", "FUEL", "VALUE"]).set_index(
+            ["REGION", "FUEL"]
+        )
         actual = convert._form_parameter(df, "test_parameter", 0)
-        expected = pd.DataFrame(data=data, columns=["REGION", "FUEL", "VALUE"])
+        expected = pd.DataFrame(
+            data=data, columns=["REGION", "FUEL", "VALUE"]
+        ).set_index(["REGION", "FUEL"])
         pd.testing.assert_frame_equal(actual, expected)
 
     def test_form_empty_two_index_param_with_defaults(self):
 
         convert = WriteExcel()  # typing: WriteExcel
 
-        df = pd.DataFrame(data=[], columns=["REGION", "VALUE"])
+        df = pd.DataFrame(data=[], columns=["REGION", "VALUE"]).set_index("REGION")
         actual = convert._form_parameter(df, "test_parameter", 0)
-        expected = pd.DataFrame(data=[], columns=["REGION", "VALUE"])
+        expected = pd.DataFrame(data=[], columns=["REGION", "VALUE"]).set_index(
+            "REGION"
+        )
         pd.testing.assert_frame_equal(actual, expected)
 
     def test_form_two_index_param(self):
@@ -32,10 +37,11 @@ class TestWriteExcel:
 
         df = pd.DataFrame(
             data=[["SIMPLICITY", 0.10], ["UTOPIA", 0.20]], columns=["REGION", "VALUE"]
-        )
+        ).set_index(["REGION"])
         actual = convert._form_parameter(df, "test_parameter", 0)
         index = pd.Index(data=["SIMPLICITY", "UTOPIA"], name="REGION")
         expected = pd.DataFrame(data=[[0.10], [0.20]], columns=["VALUE"], index=index)
+        print(actual, expected)
         pd.testing.assert_frame_equal(actual, expected)
 
     def test_form_one_columns(self):
@@ -47,6 +53,7 @@ class TestWriteExcel:
         df = pd.DataFrame(data=data, columns=["FUEL"])
         actual = convert._form_parameter(df, "test_set", 0)
         expected = pd.DataFrame(data=["A", "B", "C"], columns=["FUEL"])
+        print(actual, expected)
         pd.testing.assert_frame_equal(actual, expected)
 
     def test_form_three_columns(self):
@@ -55,7 +62,9 @@ class TestWriteExcel:
 
         data = [["SIMPLICITY", "COAL", 2015, 41], ["SIMPLICITY", "COAL", 2016, 42]]
 
-        df = pd.DataFrame(data=data, columns=["REGION", "FUEL", "YEAR", "VALUE"])
+        df = pd.DataFrame(
+            data=data, columns=["REGION", "FUEL", "YEAR", "VALUE"]
+        ).set_index(["REGION", "FUEL", "YEAR"])
         actual = convert._form_parameter(df, "test_parameter", 0)
 
         expected_data = [[41, 42]]