Repository: electricitymaps/electricitymaps-contrib

Issue:
GE production parser down
## Description

This is an automatic error report generated for Georgia (GE).

Issues:
- No recent data found for `production` parser

## Suggestions
- Try running the parser locally using the command `poetry run test_parser GE production`
- <a href="https://storage.googleapis.com/electricitymap-parser-logs/GE.html">Explore the runtime logs</a>

You can see an overview of all parser issues [here](https://github.com/tmrowco/electricitymap-contrib/wiki/Parser-issues).



Discussion:
This appears to be an SSL issue. Inspired by #2462 (GB-ORK) and #2504, disabling certificate verification with `verify=False` in the GET requests seems to fix it. The call to [`pandas.read_excel`](https://github.com/electricitymap/electricitymap-contrib/blob/809f33788039e13f2336cabb1697d3b03632b341/parsers/GE.py#L54) will require a bit of extra care, as that function doesn't seem to expose an option to ignore certificates.

I can work on this, if this is the route we'd like to take!
> This appears to be an SSL issue. Inspired by #2462 (GB-ORK) and #2504, disabling certificate verification with `verify=False` in the GET requests seems to fix it. The call to [`pandas.read_excel`](https://github.com/electricitymap/electricitymap-contrib/blob/809f33788039e13f2336cabb1697d3b03632b341/parsers/GE.py#L54) will require a bit of extra care, as that function doesn't seem to expose an option to ignore certificates.
> 
> I can work on this, if this is the route we'd like to take!

That would be great! :)

Resolving patch:
diff --git a/parsers/GE.py b/parsers/GE.py
--- a/parsers/GE.py
+++ b/parsers/GE.py
@@ -1,139 +1,149 @@
-import logging
+"""Fetch the status of the Georgian electricity grid."""
+
+# Standard library imports
 import datetime
-import pandas as pd
+import logging
+import urllib.parse
 
+# Third-party library imports
 import arrow
 import requests
+import pandas
 
-def fetch_production(zone_key='GE', session=None, target_datetime: datetime.datetime=None,
-                     logger: logging.Logger=None) -> dict:
-    """Requests the last known production mix (in MW) of a given country."""
-    r = session or requests.session()
-    
-    if target_datetime is None:
-        prod_map = {'hydroData':'hydro','solarData':'solar',
-           'thermalData':'gas', 'windPowerData':'wind'}
-    
-    
-        url = 'http://www.gse.com.ge/apps/gsebackend/rest/map'
-        response = r.get(url)
-        obj = response.json()
+# Local library imports
+from parsers.lib import config, validation
 
-        obj['typeSum']['timestamp']=arrow.now('Asia/Tbilisi').floor('minute').datetime
+MINIMUM_PRODUCTION_THRESHOLD = 10 # MW
+TIMEZONE = 'Asia/Tbilisi'
+URL = urllib.parse.urlsplit('https://gse.com.ge/apps/gsebackend/rest')
+URL_STRING = URL.geturl()
 
-        data = {
-        'zoneKey': 'GE',
-        'production': {},
-        'storage': {},
-        'source': 'gse.com.ge',
-        }
 
-        for key, value in obj['typeSum'].items():
-            # All production values should be >= 0
-            # Should be a floating point value
-            if key in list(prod_map.keys()):
-                if value>=0:
-                    data['production'][prod_map[key]] = value
-                else:
-                    data['production'][prod_map[key]] = None
-                
-        data['datetime'] = obj['typeSum']['timestamp']
-    
-        return data
+@config.refetch_frequency(datetime.timedelta(hours=1))
+def fetch_production(zone_key='GE',
+                     session=None,
+                     target_datetime=None,
+                     logger=logging.getLogger(__name__)) -> dict:
+    """Request the last known production mix (in MW) of a given country."""
+    session = session or requests.session()
+    if target_datetime is None: # Get the current production mix.
+        # TODO: remove `verify=False` ASAP.
+        production_mix = session.get(f'{URL_STRING}/map', verify=False) \
+                                .json()['typeSum']
+        return validation.validate(
+            {
+                'datetime': arrow.now(TIMEZONE).floor('minute').datetime,
+                'production': {
+                    'gas': production_mix['thermalData'],
+                    'hydro': production_mix['hydroData'],
+                    'solar': production_mix['solarData'],
+                    'wind': production_mix['windPowerData'],
+                },
+                'source': URL.netloc,
+                'zoneKey': 'GE',
+            },
+            logger,
+            remove_negative=True,
+            floor=MINIMUM_PRODUCTION_THRESHOLD)
     else:
-        # Format url
-        date = arrow.get(target_datetime).to('Asia/Tbilisi')
-        formatted_from = date.format('YYYY-MM-DDT00:00:00')
-        formatted_to   = date.format('YYYY-MM-DDT23:00:00')
-        base_url = 'http://gse.com.ge/apps/gsebackend/rest/diagramDownload'
-        url_parameters = '?fromDate={}.0000Z&' \
-                         'toDate={}.0000Z&type=FACT&lang=EN'.format(
-                         formatted_from, formatted_to)
-        url = base_url + url_parameters
-        # Download the xls file and parse the table it contains
-        data = pd.read_excel(url, skiprows=2).iloc[2:6, 3:27]
-        data.index = [ 'gas', 'hydro', 'wind', 'solar' ]
-        data.columns = pd.date_range(start=formatted_from,
-                                     end=formatted_to,
-                                     freq='1H')
-        data = data.dropna(axis=1)
-        # Create list of dictionaries with production data
-        production_mix_by_hour = []
-        for hour, datapoint in data.items():
-            production_mix = {
-                'zoneKey': zone_key,
-                'datetime': arrow.get(hour, 'Asia/Tbilisi').datetime,
+        # Get the production mix for every hour on the day of interest.
+        timestamp_from, timestamp_to = arrow.get(target_datetime, TIMEZONE) \
+                                            .replace(hour=0) \
+                                            .floor('hour') \
+                                            .span('day')
+        response = session.get(
+            f'{URL_STRING}/diagramDownload',
+            params={
+                'fromDate': timestamp_from.format('YYYY-MM-DDTHH:mm:ss'),
+                'lang': 'EN',
+                'toDate': timestamp_to.format('YYYY-MM-DDTHH:mm:ss'),
+                'type': 'FACT',
+            },
+            verify=False) # TODO: remove `verify=False` ASAP.
+        table = pandas.read_excel(response.content, header=2, index_col=1) \
+                      .iloc[2:6, 2:] \
+                      .dropna(axis='columns', how='all')
+        table.index = 'gas', 'hydro', 'wind', 'solar'
+        table.columns = pandas.date_range(start=timestamp_from.datetime,
+                                          freq='1H',
+                                          periods=table.shape[1])
+
+        # Collect the data into a list of dictionaries, then validate and
+        # return it.
+        production_mixes = (
+            {
+                'datetime': arrow.get(timestamp, TIMEZONE).datetime,
                 'production': {
-                    'gas'   : datapoint['gas'],
-                    'hydro' : datapoint['hydro'],
-                    'wind'  : datapoint['wind'],
-                    'solar' : datapoint['solar']
+                    'gas'   : production_mix['gas'],
+                    'hydro' : production_mix['hydro'],
+                    'wind'  : production_mix['wind'],
+                    'solar' : production_mix['solar'],
                 },
-                'source': 'gse.com.ge'
+                'source': URL.netloc,
+                'zoneKey': zone_key,
             }
-            production_mix_by_hour.append(production_mix)
-        # Return production data for the entire day of the requested
-        # target_datetime
-        return production_mix_by_hour
+            for timestamp, production_mix in table.items()
+        )
+        return [validation.validate(production_mix,
+                                    logger,
+                                    remove_negative=True,
+                                    floor=MINIMUM_PRODUCTION_THRESHOLD)
+                for production_mix in production_mixes]
 
-def fetch_exchange(zone_key1='GE', zone_key2='TR', session=None, target_datetime=None,
-                   logger=None) -> dict:
-    """Requests the last known power exchange (in MW) between two countries."""
-    
-    exch_map = {'AM->GE':'armeniaSum', 'AZ->GE':'azerbaijanSum',
-            'GE->TR':'turkeySum'}
-    
-    r = session or requests.session()
-    
+
+def fetch_exchange(zone_key1='GE',
+                   zone_key2='TR',
+                   session=None,
+                   target_datetime=None,
+                   logger=logging.getLogger(__name__)) -> dict:
+    """Request the last known power exchange (in MW) between two countries."""
     if target_datetime:
-        raise NotImplementedError('This parser is not yet able to parse past dates')
-    else:    
-        url = 'http://www.gse.com.ge/apps/gsebackend/rest/map'
-        response = r.get(url)
-        obj = response.json()
-    
-        exch = '->'.join(sorted([zone_key1, zone_key2]))
-                     
-        data = {
-            'sortedZoneKeys': exch,
-            'source': 'gse.com.ge',
-        }
-    
-    
-        # The net flow to be reported should be from the first country to the second
-        # (sorted alphabetically). This is NOT necessarily the same direction as the flow
-        # from country1 to country2                 
-        if exch == 'AM->GE':
-            data['netFlow'] = obj['areaSum']['armeniaSum']
-        elif exch == 'AZ->GE':
-            data['netFlow'] = obj['areaSum']['azerbaijanSum']
-        # GE->RU might be falsely reported, exchanges.json has a definition to use the Russian TSO for this flow
-        elif exch == 'GE->RU':
-            netFlow = -obj['areaSum']['russiaSum']-obj['areaSum']['russiaJavaSum']-obj['areaSum']['russiaSalkhinoSum']
-            data['netFlow'] = netFlow
-            #data['netFlow'] = None
-        elif exch == 'GE->TR':
-            data['netFlow'] = -obj['areaSum']['turkeySum']
-        else:
-            raise NotImplementedError('This exchange pair is not implemented.')
-        
+        raise NotImplementedError('Past exchange data is not yet supported')
 
-        data['datetime']=arrow.now('Asia/Tbilisi').floor('minute').datetime
-    
-        return data
-    
-    
-if __name__ == '__main__':
-    """Main method, never used by the Electricity Map backend, but handy for testing."""
+    session = session or requests.session()
+    # The API uses the convention of positive net flow into GE.
+    net_flows = session.get(f'{URL_STRING}/map', verify=False) \
+                       .json()['areaSum'] # TODO: remove `verify=False` ASAP.
+
+    # Positive net flow should be in the same direction as the arrow in
+    # `exchange`. This is not necessarily the same as positive flow into GE.
+    exchange = '->'.join(sorted((zone_key1, zone_key2)))
+    if exchange == 'AM->GE':
+        net_flow = net_flows['armeniaSum']
+    elif exchange == 'AZ->GE':
+        net_flow = net_flows['azerbaijanSum']
+    elif exchange == 'GE->RU':
+        # GE->RU might be falsely reported, exchanges.json has a definition to
+        # use the Russian TSO for this flow.
+        net_flow = -(net_flows['russiaSum']
+                     + net_flows['russiaJavaSum']
+                     + net_flows['russiaSalkhinoSum'])
+    elif exchange == 'GE->TR':
+        net_flow = -net_flows['turkeySum']
+    else:
+        raise NotImplementedError(f'{exchange} pair is not implemented')
 
+    return {
+        'datetime': arrow.now(TIMEZONE).floor('minute').datetime,
+        'netFlow': net_flow,
+        'sortedZoneKeys': exchange,
+        'source': URL.netloc,
+    }
+
+
+if __name__ == '__main__':
+    # Never used by the Electricity Map backend, but handy for testing.
     print('fetch_production() ->')
     print(fetch_production())
-    print('fetch_exchange(GE, AM) ->')
+    print('fetch_production(target_datetime=datetime.datetime(2020, 1, 1)) ->')
+    print(fetch_production(target_datetime=datetime.datetime(2020, 1, 1)))
+    print("fetch_exchange('GE', 'AM') ->")
     print(fetch_exchange('GE', 'AM'))
-    print('fetch_exchange(GE, AZ) ->')
+    print("fetch_exchange('GE', 'AZ') ->")
     print(fetch_exchange('GE', 'AZ'))
-    print('fetch_exchange(GE, RU) ->')
+    print("fetch_exchange('GE', 'RU') ->")
     print(fetch_exchange('GE', 'RU'))
-    print('fetch_exchange(GE, TR) ->')
+    print("fetch_exchange('GE', 'TR') ->")
     print(fetch_exchange('GE', 'TR'))
-    
+    print("fetch_exchange('AB', 'YZ') ->")
+    print(fetch_exchange('AB', 'YZ'))


Test patch: