Repository: electricitymaps/electricitymaps-contrib

Issue:
TW production parser down
## Description

This is an automatic error report generated for Taiwan (TW).

Issues:
- No recent data found for `production` parser

## Suggestions
- Try running the parser locally using the command `poetry run test_parser TW production`
- <a href="https://kibana.electricitymap.org/app/kibana#/discover/10af54f0-0c4a-11e9-85c1-1d63df8c862c?_g=(refreshInterval:('$$hashKey':'object:232',display:'5%20minutes',pause:!f,section:2,value:300000),time:(from:now-24h,mode:quick,to:now))&_a=(columns:!(message,extra.key,level),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!t,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:level,negate:!f,params:(query:ERROR,type:phrase),type:phrase,value:ERROR),query:(match:(level:(query:ERROR,type:phrase)))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'96f67170-0c49-11e9-85c1-1d63df8c862c',key:extra.key,negate:!f,params:(query:TW,type:phrase),type:phrase,value:TW),query:(match:(extra.key:(query:TW,type:phrase))))),index:'96f67170-0c49-11e9-85c1-1d63df8c862c',interval:auto,query:(language:lucene,query:''),sort:!('@timestamp',desc))">Explore the runtime logs</a>

You can see an overview of all parser issues [here](https://github.com/tmrowco/electricitymap-contrib/wiki/Parser-issues).



Discussion:
```
  File "/home/contrib/parsers/TW.py", line 26, in fetch_production
    'additional']
  File "/usr/local/lib/python3.7/site-packages/pandas/core/generic.py", line 4389, in __setattr__
    return object.__setattr__(self, name, value)
  File "pandas/_libs/properties.pyx", line 69, in pandas._libs.properties.AxisProperty.__set__
  File "/usr/local/lib/python3.7/site-packages/pandas/core/generic.py", line 646, in _set_axis
    self._data.set_axis(axis, labels)
  File "/usr/local/lib/python3.7/site-packages/pandas/core/internals.py", line 3323, in set_axis
    'values have {new} elements'.format(old=old_len, new=new_len))
ValueError: Length mismatch: Expected axis has 7 elements, new values have 6 elements
```

Resolving patch:
diff --git a/parsers/TW.py b/parsers/TW.py
--- a/parsers/TW.py
+++ b/parsers/TW.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 import arrow
-import requests
-import pandas
 import dateutil
+import pandas as pd
+import requests
 
 
 def fetch_production(zone_key='TW', session=None, target_datetime=None, logger=None) -> dict:
@@ -20,21 +20,27 @@ def fetch_production(zone_key='TW', session=None, target_datetime=None, logger=N
     tz = 'Asia/Taipei'
     dumpDate = arrow.get(dumpDate, 'YYYY-MM-DD HH:mm').replace(tzinfo=dateutil.tz.gettz(tz))
 
-    objData = pandas.DataFrame(prodData)
+    objData = pd.DataFrame(prodData)
 
-    objData.columns = ['fueltype', 'name', 'capacity', 'output', 'percentage',
-                       'additional']
+    columns = ['fueltype', 'additional_1', 'name', 'capacity', 'output', 'percentage', 'additional_2']
+    assert len(objData.iloc[0]) == len(columns), "number of input columns changed"
+    objData.columns = columns
 
     objData['fueltype'] = objData.fueltype.str.split('(').str[1]
     objData['fueltype'] = objData.fueltype.str.split(')').str[0]
-    objData.drop('additional', axis=1, inplace=True)
-    objData.drop('percentage', axis=1, inplace=True)
+    objData.loc[:,['capacity', 'output']] = objData[['capacity', 'output']].apply(pd.to_numeric, errors='coerce')
+    assert not objData.capacity.isna().all(), "capacity data is entirely NaN - input column order may have changed"
+    assert not objData.output.isna().all(), "output data is entirely NaN - input column order may have changed"
 
-    objData['capacity'] = pandas.to_numeric(objData['capacity'], errors='coerce')
-    objData['output'] = pandas.to_numeric(objData['output'], errors='coerce')
-    production = pandas.DataFrame(objData.groupby('fueltype').sum())
+    objData.drop(columns=['additional_1', 'name', 'additional_2', 'percentage'], axis=1, inplace=True)
+    # summing because items in returned object are for each power plant and operational units
+    production = pd.DataFrame(objData.groupby('fueltype').sum())
     production.columns = ['capacity', 'output']
 
+    # check output values coincide with total capacity by fuel type
+    check_values = production.output <= production.capacity
+    assert check_values.loc[~check_values.index.isin(["Co-Gen"])].all(), "output > capacity" # HACK: Co-Gen capacity is underestimated
+
     coal_capacity = production.loc['Coal'].capacity + production.loc['IPP-Coal'].capacity
     gas_capacity = production.loc['LNG'].capacity + production.loc['IPP-LNG'].capacity
     oil_capacity = production.loc['Oil'].capacity + production.loc['Diesel'].capacity


Test patch: