Repository: electricitymaps/electricitymaps-contrib

Issue:
ENTSOE parser cleanup
EE oil shale fix by @brunolajoie didn't work because the mapping
https://github.com/tmrowco/electricitymap-contrib/blob/master/parsers/ENTSOE.py#L52-L64

is not used.

We should make sure those lines use the mapping directly, instead of using functions that do not depend on the mapping:

https://github.com/tmrowco/electricitymap-contrib/blob/master/parsers/ENTSOE.py#L863-L875

Good catch @alixunderplatz !


Discussion:
- EE oil shale quick fix in PR #1743.
- Refacto as suggested above is still needed

Resolving patch:
diff --git a/parsers/ENTSOE.py b/parsers/ENTSOE.py
--- a/parsers/ENTSOE.py
+++ b/parsers/ENTSOE.py
@@ -50,17 +50,21 @@
 }
 ENTSOE_PARAMETER_BY_DESC = {v: k for k, v in ENTSOE_PARAMETER_DESC.items()}
 ENTSOE_PARAMETER_GROUPS = {
-    'biomass': ['B01', 'B17'],
-    'coal': ['B02', 'B05', 'B07', 'B08'],
-    'gas': ['B03', 'B04'],
-    'geothermal': ['B09'],
-    'hydro': ['B11', 'B12'],
-    'hydro storage': ['B10'],
-    'nuclear': ['B14'],
-    'oil': ['B06'],
-    'solar': ['B16'],
-    'wind': ['B18', 'B19'],
-    'other': ['B20', 'B13', 'B15']
+    'production': {
+        'biomass': ['B01', 'B17'],
+        'coal': ['B02', 'B05', 'B07', 'B08'],
+        'gas': ['B03', 'B04'],
+        'geothermal': ['B09'],
+        'hydro': ['B11', 'B12'],
+        'nuclear': ['B14'],
+        'oil': ['B06'],
+        'solar': ['B16'],
+        'wind': ['B18', 'B19'],
+        'other': ['B20', 'B13', 'B15']
+    },
+    'storage': {
+        'hydro storage': ['B10']
+    }
 }
 ENTSOE_PARAMETER_BY_GROUP = {v: k for k, g in ENTSOE_PARAMETER_GROUPS.items() for v in g}
 # Define all ENTSOE zone_key <-> domain mapping
@@ -740,66 +744,11 @@ def validate_production(datapoint, logger):
     return True
 
 
-def get_biomass(values):
-    if 'Biomass' in values or 'Waste' in values:
-        return (values.get('Biomass', 0)
-                + values.get('Waste', 0))
-
-
-def get_coal(values):
-    if 'Fossil Brown coal/Lignite' in values \
-        or 'Fossil Peat' in values \
-        or 'Fossil Oil shale' in values \
-        or 'Fossil Hard coal' in values:
-        return (values.get('Fossil Brown coal/Lignite', 0)
-                + values.get('Fossil Peat', 0)
-                + values.get('Fossil Oil shale', 0)
-                + values.get('Fossil Hard coal', 0))
-
-
-def get_gas(values):
-    if 'Fossil Coal-derived gas' in values or 'Fossil Gas' in values:
-        return values.get('Fossil Coal-derived gas', 0) + \
-               values.get('Fossil Gas', 0)
-
-
-def get_hydro(values):
-    if ('Hydro Run-of-river and poundage' in values or
-        'Hydro Water Reservoir' in values):
-        return values.get('Hydro Run-of-river and poundage', 0) + \
-               values.get('Hydro Water Reservoir', 0)
-
-
-def get_hydro_storage(storage_values):
-    if 'Hydro Pumped Storage' in storage_values:
-        return -1 * storage_values.get('Hydro Pumped Storage', 0)
-
-
-def get_oil(values):
-    if 'Fossil Oil' in values:
-        value = values.get('Fossil Oil', 0)
-        return value if value != -1.0 else None
-
-
 def get_wind(values):
     if 'Wind Onshore' in values or 'Wind Offshore' in values:
         return values.get('Wind Onshore', 0) + values.get('Wind Offshore', 0)
 
 
-def get_geothermal(values):
-    if 'Geothermal' in values:
-        return values.get('Geothermal', 0)
-
-
-def get_unknown(values):
-    if ('Marine' in values or
-        'Other renewable' in values or
-        'Other' in values):
-        return (values.get('Marine', 0) +
-                values.get('Other renewable', 0) +
-                values.get('Other', 0))
-
-
 def fetch_consumption(zone_key, session=None, target_datetime=None,
                       logger=logging.getLogger(__name__)):
     """Gets consumption for a specified zone, returns a dictionary."""
@@ -857,27 +806,30 @@ def fetch_production(zone_key, session=None, target_datetime=None,
 
     data = []
     for i in range(len(production_dates)):
-        production_values = {ENTSOE_PARAMETER_DESC[k]: v for k, v in
-                             productions[i].items()}
+        production_values = {k: v for k, v in productions[i].items()}
         production_date = production_dates[i]
 
+        production_types = {'production': {}, 'storage': {}}
+        for key in ['production', 'storage']:
+            parameter_groups = ENTSOE_PARAMETER_GROUPS[key]
+            multiplier = -1 if key == 'storage' else 1
+
+            for fuel, groups in parameter_groups.items():
+                has_value = any([production_values.get(grp) is not None for grp in groups])
+                if has_value:
+                    value = sum([production_values.get(grp, 0) for grp in groups])
+                    value *= multiplier
+                else:
+                    value = None
+
+                production_types[key][fuel] = value
+
         data.append({
             'zoneKey': zone_key,
             'datetime': production_date.datetime,
-            'production': {
-                'biomass': get_biomass(production_values),
-                'coal': get_coal(production_values),
-                'gas': get_gas(production_values),
-                'hydro': get_hydro(production_values),
-                'nuclear': production_values.get('Nuclear', None),
-                'oil': get_oil(production_values),
-                'solar': production_values.get('Solar', None),
-                'wind': get_wind(production_values),
-                'geothermal': get_geothermal(production_values),
-                'unknown': get_unknown(production_values)
-            },
+            'production': production_types['production'],
             'storage': {
-                'hydro': get_hydro_storage(production_values),
+                'hydro': production_types['storage']['hydro storage'],
             },
             'source': 'entsoe.eu'
         })


Test patch: