Repository: electricitymaps/electricitymaps-contrib

Issue:
New keys causing problems in US-SPP
US-SPP is missing from the map due to changes in their data keys.

Error
```
Traceback (most recent call last):
  File "/home/feeder/lib/fetch_data.py", line 131, in launch_parsers
    **parser_kwargs)
  File "/home/contrib/parsers/US_SPP.py", line 122, in fetch_production
    processed_data = data_processor(raw_data, logger)
  File "/home/contrib/parsers/US_SPP.py", line 71, in data_processor
    production['coal'] = production['Coal Market'] + production['Coal Self']
KeyError: 'Coal Market'
```
Warning
```
New column 'Coal' present in US-SPP data source.
```


Discussion:

Resolving patch:
diff --git a/parsers/US_SPP.py b/parsers/US_SPP.py
--- a/parsers/US_SPP.py
+++ b/parsers/US_SPP.py
@@ -18,7 +18,8 @@
            'Solar': 'solar',
            'Natural Gas': 'gas',
            'Diesel Fuel Oil': 'oil',
-           'Waste Disposal Services': 'biomass'
+           'Waste Disposal Services': 'biomass',
+           'Coal': 'coal'
             }
 
 TIE_MAPPING = {'US-MISO->US-SPP': ['AMRN', 'DPC', 'GRE', 'MDU', 'MEC', 'NSP', 'OTP']}
@@ -50,8 +51,7 @@ def data_processor(df, logger):
     # Remove leading whitespace in column headers.
     df.columns = df.columns.str.strip()
 
-    keys_to_remove = {'Coal Market', 'Coal Self', 'GMT MKT Interval', 'Average Actual Load',
-                      'Other', 'Waste Heat'}
+    keys_to_remove = {'GMT MKT Interval', 'Average Actual Load', 'Other', 'Waste Heat'}
 
     # Check for new generation columns.
     known_keys = MAPPING.keys() | keys_to_remove
@@ -68,7 +68,6 @@ def data_processor(df, logger):
     processed_data = []
     for index, row in df.iterrows():
         production = row.to_dict()
-        production['coal'] = production['Coal Market'] + production['Coal Self']
 
         extra_unknowns = sum([production[k] for k in unknown_keys])
         production['unknown'] = production['Other'] + production['Waste Heat'] + extra_unknowns


Test patch:
diff --git a/parsers/test/mocks/US_SPP_Gen_Mix.pkl b/parsers/test/mocks/US_SPP_Gen_Mix.pkl
Binary files a/parsers/test/mocks/US_SPP_Gen_Mix.pkl and b/parsers/test/mocks/US_SPP_Gen_Mix.pkl differ
diff --git a/parsers/test/test_US_SPP.py b/parsers/test/test_US_SPP.py
--- a/parsers/test/test_US_SPP.py
+++ b/parsers/test/test_US_SPP.py
@@ -32,16 +32,12 @@ def test_fetch_production(self):
         with self.subTest():
             self.assertEqual(len(data), 23)
 
-        # Multiple coal keys in data source, ensure they are summed correctly.
-        with self.subTest():
-            self.assertEqual(round(datapoint['production']['coal'], 2), 15963.60)
-
         # Unknown keys must be assigned and summed.
         with self.subTest():
-            self.assertEqual(round(datapoint['production']['unknown'], 2), 63.67)
+            self.assertEqual(round(datapoint['production']['unknown'], 2), 33.1)
 
         with self.subTest():
-            expected_dt = get(datetime(2018, 2, 23, 12, 25), 'UTC').datetime
+            expected_dt = get(datetime(2018, 7, 27, 11, 45), 'UTC').datetime
             self.assertEqual(datapoint['datetime'], expected_dt)
 
         with self.subTest():