Repository: OSeMOSYS/otoole

Issue:
[Bug]: otoole.results calculation of CRF fails if no data in discount_rate parameter
### The Issue

Calculation of the capital recovery factor fails if no data is entered in the discount rate parameter file.

### Expected Behavior

The capital recovery factor should be calculated using the default value for the discount factor, for any region/technology combination not defined in the `discount_rate_idv` parameter failed or region not defined in the `discount_rate` parameter.

### Steps To Reproduce

This test reproduces the issue, with a new fixture representing an empty discount rate dataframe::

```python
@fixture
def discount_rate_empty():
    df = pd.DataFrame(
        data=[],
        columns=["REGION", "VALUE"],
    ).set_index(["REGION"])
    return df

  def test_crf_empty_discount_rate(self, region, discount_rate_empty, operational_life):

      technologies = ["GAS_EXTRACTION", "DUMMY"]
      regions = region["VALUE"].to_list()
      actual = capital_recovery_factor(
          regions, technologies, discount_rate_empty, operational_life
      )

      expected = pd.DataFrame(
          data=[
              ["SIMPLICITY", "GAS_EXTRACTION", 0.5121951219512197],
              ["SIMPLICITY", "DUMMY", 0.34972244250594786],
          ],
          columns=["REGION", "TECHNOLOGY", "VALUE"],
      ).set_index(["REGION", "TECHNOLOGY"])

      assert_frame_equal(actual, expected)
```

### Log output

```shell
src/otoole/results/result_package.py:798: in capital_recovery_factor
    crf[:] = values
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/frame.py:4074: in __setitem__
    return self._setitem_slice(slc, value)
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/frame.py:4098: in _setitem_slice
    self.iloc[key] = value
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/indexing.py:885: in __setitem__
    iloc._setitem_with_indexer(indexer, value, self.name)
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/indexing.py:1895: in _setitem_with_indexer
    self._setitem_single_block(indexer, value, name)
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/indexing.py:2138: in _setitem_single_block
    self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/internals/managers.py:399: in setitem
    return self.apply("setitem", indexer=indexer, value=value)
../../miniconda3/envs/otoole/lib/python3.9/site-packages/pandas/core/internals/managers.py:354: in apply
    applied = getattr(b, f)(**kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = NumpyBlock: slice(0, 1, 1), 1 x 2, dtype: object
indexer = slice(None, None, None)
value = Series([], Name: VALUE, dtype: object), using_cow = False

    def setitem(self, indexer, value, using_cow: bool = False) -> Block:
        """
        Attempt self.values[indexer] = value, possibly creating a new array.
    
        Parameters
        ----------
        indexer : tuple, list-like, array-like, slice, int
            The subset of self.values to set
        value : object
            The value being set
        using_cow: bool, default False
            Signaling if CoW is used.
    
        Returns
        -------
        Block
    
        Notes
        -----
        `indexer` is a direct slice/positional indexer. `value` must
        be a compatible shape.
        """
    
        value = self._standardize_fill_value(value)
    
        values = cast(np.ndarray, self.values)
        if self.ndim == 2:
            values = values.T
    
        # length checking
        check_setitem_lengths(indexer, value, values)
    
        if self.dtype != _dtype_obj:
            # GH48933: extract_array would convert a pd.Series value to np.ndarray
            value = extract_array(value, extract_numpy=True)
        try:
            casted = np_can_hold_element(values.dtype, value)
        except LossySetitemError:
            # current dtype cannot store value, coerce to common dtype
            nb = self.coerce_to_target_dtype(value, warn_on_upcast=True)
            return nb.setitem(indexer, value)
        else:
            if self.dtype == _dtype_obj:
                # TODO: avoid having to construct values[indexer]
                vi = values[indexer]
                if lib.is_list_like(vi):
                    # checking lib.is_scalar here fails on
                    #  test_iloc_setitem_custom_object
                    casted = setitem_datetimelike_compat(values, len(vi), casted)
    
            self = self._maybe_copy(using_cow, inplace=True)
            values = cast(np.ndarray, self.values.T)
            if isinstance(casted, np.ndarray) and casted.ndim == 1 and len(casted) == 1:
                # NumPy 1.25 deprecation: https://github.com/numpy/numpy/pull/10615
                casted = casted[0, ...]
>           values[indexer] = casted
E           ValueError: could not broadcast input array from shape (0,) into shape (2,1)
```


### Operating System

MacOS

### What version of otoole are you running?

1.1.2.post1.dev15+g15bdc0b.d20240201

### Possible Solution

Yes, we'll need to always ensure that the discount_rate or discount_rate_idv parameter is padded out with default values before passing it to the crf calculation.

### Anything else?

_No response_

Discussion:

Patch:
diff --git a/src/otoole/results/result_package.py b/src/otoole/results/result_package.py
index a63de92..c961d5e 100644
--- a/src/otoole/results/result_package.py
+++ b/src/otoole/results/result_package.py
@@ -775,6 +775,9 @@ def capital_recovery_factor(
 
         return numerator / denominator
 
+    if discount_rate_idv.empty or operational_life.empty:
+        raise ValueError("Cannot calculate PV Annuity due to missing data")
+
     if not regions and not technologies:
         return pd.DataFrame(
             data=[],
@@ -823,6 +826,10 @@ def pv_annuity(
         param PvAnnuity{r in REGION, t in TECHNOLOGY} :=
                 (1 - (1 + DiscountRate[r])^(-(OperationalLife[r,t]))) * (1 + DiscountRate[r]) / DiscountRate[r];
     """
+
+    if discount_rate.empty or operational_life.empty:
+        raise ValueError("Cannot calculate PV Annuity due to missing data")
+
     if regions and technologies:
         index = pd.MultiIndex.from_product(
             [regions, technologies], names=["REGION", "TECHNOLOGY"]
@@ -873,6 +880,11 @@ def discount_factor(
                 (1 + DiscountRate[r]) ^ (y - min{yy in YEAR} min(yy) + 0.5);
     """
 
+    if discount_rate.empty:
+        raise ValueError(
+            "Cannot calculate discount factor due to missing discount rate"
+        )
+
     if regions and years:
         discount_rate["YEAR"] = [years]
         discount_factor = discount_rate.explode("YEAR").reset_index(level="REGION")
@@ -917,6 +929,11 @@ def discount_factor_storage(
                 (1 + DiscountRateStorage[r,s]) ^ (y - min{yy in YEAR} min(yy) + 0.0);
     """
 
+    if discount_rate_storage.empty:
+        raise ValueError(
+            "Cannot calculate discount_factor_storage due to missing discount rate"
+        )
+
     if regions and years:
         index = pd.MultiIndex.from_product(
             [regions, storages, years], names=["REGION", "STORAGE", "YEAR"]
diff --git a/src/otoole/results/results.py b/src/otoole/results/results.py
index 982c11c..a5be439 100644
--- a/src/otoole/results/results.py
+++ b/src/otoole/results/results.py
@@ -35,6 +35,7 @@ class ReadResults(ReadStrategy):
             param_default_values = self._read_default_values(self.input_config)
         else:
             input_data = {}
+            param_default_values = {}
 
         available_results = self.get_results_from_file(
             filepath, input_data
@@ -42,15 +43,7 @@ class ReadResults(ReadStrategy):
 
         default_values = self._read_default_values(self.results_config)  # type: Dict
 
-        # need to expand discount rate for results processing
-        if "DiscountRate" in input_data:
-            input_data["DiscountRate"] = self._expand_dataframe(
-                "DiscountRate", input_data, param_default_values
-            )
-        if "DiscountRateIdv" in input_data:
-            input_data["DiscountRateIdv"] = self._expand_dataframe(
-                "DiscountRateIdv", input_data, param_default_values
-            )
+        input_data = self._expand_required_params(input_data, param_default_values)
 
         results = self.calculate_results(
             available_results, input_data
@@ -87,6 +80,24 @@ class ReadResults(ReadStrategy):
 
         return results
 
+    def _expand_required_params(
+        self,
+        input_data: dict[str, pd.DataFrame],
+        param_defaults: dict[str, Any],
+    ) -> dict[str, pd.DataFrame]:
+        """Expands required default values for results processing"""
+
+        if "DiscountRate" in input_data:
+            input_data["DiscountRate"] = self._expand_dataframe(
+                "DiscountRate", input_data, param_defaults
+            )
+        if "DiscountRateIdv" in input_data:
+            input_data["DiscountRateIdv"] = self._expand_dataframe(
+                "DiscountRateIdv", input_data, param_defaults
+            )
+
+        return input_data
+
 
 class ReadWideResults(ReadResults):
     def get_results_from_file(self, filepath, input_data):


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index bd41e71..d22edb5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -83,6 +83,33 @@ def discount_rate_storage():
     return df
 
 
+@fixture
+def discount_rate_empty():
+    df = pd.DataFrame(
+        data=[],
+        columns=["REGION", "VALUE"],
+    ).set_index(["REGION"])
+    return df
+
+
+@fixture
+def discount_rate_idv_empty():
+    df = pd.DataFrame(
+        data=[],
+        columns=["REGION", "TECHNOLOGY", "VALUE"],
+    ).set_index(["REGION", "TECHNOLOGY"])
+    return df
+
+
+@fixture
+def discount_rate_storage_empty():
+    df = pd.DataFrame(
+        data=[],
+        columns=["REGION", "STORAGE", "VALUE"],
+    ).set_index(["REGION", "STORAGE"])
+    return df
+
+
 @fixture
 def emission_activity_ratio():
     df = pd.DataFrame(
diff --git a/tests/results/test_results_package.py b/tests/results/test_results_package.py
index 33c784b..418599c 100644
--- a/tests/results/test_results_package.py
+++ b/tests/results/test_results_package.py
@@ -669,6 +669,17 @@ class TestCapitalRecoveryFactor:
 
         assert_frame_equal(actual, expected)
 
+    def test_crf_empty_discount_rate(
+        self, region, discount_rate_empty, operational_life
+    ):
+        technologies = ["GAS_EXTRACTION", "DUMMY"]
+        regions = region["VALUE"].to_list()
+
+        with raises(ValueError):
+            capital_recovery_factor(
+                regions, technologies, discount_rate_empty, operational_life
+            )
+
 
 class TestPvAnnuity:
     def test_pva(self, region, discount_rate, operational_life):
@@ -687,7 +698,7 @@ class TestPvAnnuity:
 
         assert_frame_equal(actual, expected)
 
-    def test_pva_null(self, discount_rate):
+    def test_pva_null(self, discount_rate, operational_life):
 
         actual = pv_annuity([], [], discount_rate, operational_life)
 
@@ -698,6 +709,15 @@ class TestPvAnnuity:
 
         assert_frame_equal(actual, expected)
 
+    def test_pva_empty_discount_rate(
+        self, region, discount_rate_empty, operational_life
+    ):
+        technologies = ["GAS_EXTRACTION", "DUMMY"]
+        regions = region["VALUE"].to_list()
+
+        with raises(ValueError):
+            pv_annuity(regions, technologies, discount_rate_empty, operational_life)
+
 
 class TestDiscountFactor:
     def test_df_start(self, region, year, discount_rate):
@@ -774,6 +794,13 @@ class TestDiscountFactor:
 
         assert_frame_equal(actual, expected)
 
+    def test_df_empty_discount_rate(self, region, year, discount_rate_empty):
+        regions = region["VALUE"].to_list()
+        years = year["VALUE"].to_list()
+
+        with raises(ValueError):
+            discount_factor(regions, years, discount_rate_empty, 1.0)
+
 
 class TestDiscountFactorStorage:
     def test_dfs_start(self, region, year, discount_rate_storage):
@@ -859,6 +886,18 @@ class TestDiscountFactorStorage:
 
         assert_frame_equal(actual, expected)
 
+    def test_df_storage_empty_discount_rate(
+        self, region, year, discount_rate_storage_empty
+    ):
+        storages = ["DAM"]
+        regions = region["VALUE"].to_list()
+        years = year["VALUE"].to_list()
+
+        with raises(ValueError):
+            discount_factor_storage(
+                regions, storages, years, discount_rate_storage_empty, 1.0
+            )
+
 
 class TestResultsPackage:
     def test_results_package_init(self):
diff --git a/tests/test_read_strategies.py b/tests/test_read_strategies.py
index 7b4670c..494a5b7 100644
--- a/tests/test_read_strategies.py
+++ b/tests/test_read_strategies.py
@@ -14,6 +14,7 @@ from otoole.results.results import (
     ReadCplex,
     ReadGlpk,
     ReadGurobi,
+    ReadResults,
     check_for_duplicates,
     identify_duplicate,
     rename_duplicate_column,
@@ -21,6 +22,12 @@ from otoole.results.results import (
 from otoole.utils import _read_file
 
 
+# To instantiate abstract class ReadResults
+class DummyReadResults(ReadResults):
+    def get_results_from_file(self, filepath, input_data):
+        raise NotImplementedError()
+
+
 class TestReadCplex:
 
     cplex_data = """<?xml version = "1.0" encoding="UTF-8" standalone="yes"?>
@@ -1275,3 +1282,66 @@ class TestLongifyData:
 
         with raises(ValueError):
             check_datatypes(df, user_config, "AvailabilityFactor")
+
+
+class TestExpandRequiredParameters:
+    """Tests the expansion of required parameters for results processing"""
+
+    region = pd.DataFrame(data=["SIMPLICITY"], columns=["VALUE"])
+
+    technology = pd.DataFrame(data=["NGCC"], columns=["VALUE"])
+
+    def test_no_expansion(self):
+
+        user_config = {
+            "REGION": {
+                "dtype": "str",
+                "type": "set",
+            },
+        }
+
+        reader = DummyReadResults(user_config=user_config)
+        defaults = {}
+        input_data = {}
+
+        actual = reader._expand_required_params(input_data, defaults)
+
+        assert not actual
+
+    def test_expansion(self, user_config, discount_rate_empty, discount_rate_idv_empty):
+
+        user_config["DiscountRateIdv"] = {
+            "indices": ["REGION", "TECHNOLOGY"],
+            "type": "param",
+            "dtype": "float",
+            "default": 0.10,
+        }
+
+        reader = DummyReadResults(user_config=user_config)
+        defaults = reader._read_default_values(user_config)
+        input_data = {
+            "REGION": self.region,
+            "TECHNOLOGY": self.technology,
+            "DiscountRate": discount_rate_empty,
+            "DiscountRateIdv": discount_rate_idv_empty,
+        }
+
+        actual = reader._expand_required_params(input_data, defaults)
+
+        actual_dr = actual["DiscountRate"]
+
+        expected_dr = pd.DataFrame(
+            data=[["SIMPLICITY", 0.05]],
+            columns=["REGION", "VALUE"],
+        ).set_index(["REGION"])
+
+        pd.testing.assert_frame_equal(actual_dr, expected_dr)
+
+        actual_dr_idv = actual["DiscountRateIdv"]
+
+        expected_dr_idv = pd.DataFrame(
+            data=[["SIMPLICITY", "NGCC", 0.10]],
+            columns=["REGION", "TECHNOLOGY", "VALUE"],
+        ).set_index(["REGION", "TECHNOLOGY"])
+
+        pd.testing.assert_frame_equal(actual_dr_idv, expected_dr_idv)