Repository: electricitymaps/electricitymaps-contrib

Issue:
KR production parser down
## Description

This is an automatic error report generated for South Korea (KR).

Issues:
- No recent data found for `production` parser

## Suggestions
- Try running the parser locally using the command `poetry run test_parser KR production`
- <a href="https://storage.googleapis.com/electricitymap-parser-logs/KR.html">Explore the runtime logs</a>

You can see an overview of all parser issues [here](https://github.com/tmrowco/electricitymap-contrib/wiki/Parser-issues).



Discussion:

Resolving patch:
diff --git a/parsers/KR.py b/parsers/KR.py
--- a/parsers/KR.py
+++ b/parsers/KR.py
@@ -1,212 +1,268 @@
 #!/usr/bin/env python3
 
-from typing import Union
+import pandas as pd
+
 import arrow
+import datetime
+import json
+import logging
+import pprint
 import re
 import requests
-from bs4 import BeautifulSoup
-from logging import getLogger
 
-LOAD_URL = 'http://kpx.or.kr/eng/index.do'
+from bs4 import BeautifulSoup
 
-HYDRO_URL = 'https://cms.khnp.co.kr/eng/realTimeMgr/water.do?mnCd=EN040203'
+from parsers.lib.config import refetch_frequency
 
-NUCLEAR_URLS = ['http://cms.khnp.co.kr/eng/kori/realTimeMgr/list.do?mnCd=EN03020201&brnchCd=BR0302',
-                'http://cms.khnp.co.kr/eng/hanbit/realTimeMgr/list.do?mnCd=EN03020202&brnchCd=BR0303',
-                'http://cms.khnp.co.kr/eng/wolsong/realTimeMgr/list.do?mnCd=EN03020203&brnchCd=BR0305',
-                'http://cms.khnp.co.kr/eng/hanul/realTimeMgr/list.do?mnCd=EN03020204&brnchCd=BR0304',
-                'http://cms.khnp.co.kr/eng/saeul/realTimeMgr/list.do?mnCd=EN03020205&brnchCd=BR0312']
+TIMEZONE = 'Asia/Seoul'
+REAL_TIME_URL = 'https://new.kpx.or.kr/powerinfoSubmain.es?mid=a10606030000'
+PRICE_URL = 'https://new.kpx.or.kr/smpInland.es?mid=a10606080100&device=pc'
+LONG_TERM_PRODUCTION_URL = 'https://new.kpx.or.kr/powerSource.es?mid=a10606030000&device=chart'
 
-HYDRO_CAPACITIES = {'Hwacheon': 108,
-                    'Chuncheon': 62.28,
-                    'Anheung': 0.6,
-                    'Uiam': 46.5,
-                    'Cheongpyeong': 139.6,
-                    'Paldang': 120,
-                    'Goesan': 2.8,
-                    'Chilbo': 34.8,
-                    'Boseonggang': 4.5,
-                    'Euiam': 14.8,
-                    'Seomjingang': 8
-                    }
+pp = pprint.PrettyPrinter(indent=4)
 
-# Gangneung hydro plant used only for peak load and backup, capacity info not available.
+#### Classification of New & Renewable Energy Sources ####
+# Source: https://cms.khnp.co.kr/eng/content/563/main.do?mnCd=EN040101
+# New energy: Hydrogen, Fuel Cell, Coal liquefied or gasified energy, and vacuum residue gasified energy, etc.
+# Renewable: Solar, Wind power, Water power, ocean energy, Geothermal, Bio energy, etc.
 
+# src: https://stackoverflow.com/questions/3463930/how-to-round-the-minute-of-a-datetime-object
+def time_floor(time, delta, epoch=None):
+    if epoch is None:
+        epoch = datetime.datetime(1970, 1, 1, tzinfo=time.tzinfo)
+    mod = (time - epoch) % delta
+    return time - mod
 
-def timestamp_processor(timestamps, with_tz=False, check_delta=False):
+def extract_chart_data(html):
     """
-    Compares naive arrow objects, returning the average.
-    Optionally can determine if timestamps are too disparate to be used.
-    Returns an arrow object, with optional timezone.
+        Extracts generation breakdown chart data from the source code of the page.
+    """
+    # Extract object with data
+    data_source = re.search(r"var ictArr = (\[\{.+\}\]);", html).group(1)  
+    # Un-quoted keys ({key:"value"}) are valid JavaScript but not valid JSON (which requires {"key":"value"}). 
+    # Will break if other keys than these are introduced. Alternatively, use a JSON5 library (JSON5 allows un-quoted keys)
+    data_source = re.sub(r'"(localCoal|newRenewable|oil|once|gas|nuclearPower|coal|regDate|raisingWater|waterPower|seq)"', r'"\1"', data_source)  
+    json_obj = json.loads(data_source)
+
+    timed_data = {}
+
+    for item in json_obj:
+        if item['regDate'] == '0':
+            break
+
+        date = datetime.datetime.strptime(item['regDate'], '%Y-%m-%d %H:%M')
+        date = arrow.get(date, TIMEZONE).datetime
+        
+        timed_data[date] = {
+            'coal': round(float(item['coal']) + float(item['localCoal']), 5),
+            'gas': round(float(item['gas']), 5),
+            'hydro': round(float(item['waterPower']), 5),
+            'nuclear': round(float(item['nuclearPower']), 5),
+            'oil': round(float(item['oil']), 5),
+            'renewable': round(float(item['newRenewable']), 5),
+            'pumpedHydro': round(float(item['raisingWater']), 5),
+        }
+
+    return timed_data
+
+
+@refetch_frequency(datetime.timedelta(minutes=5))
+def fetch_consumption(
+    zone_key="KR", session=None, target_datetime=None, logger=logging.getLogger(__name__)) -> dict:
+    """
+      Fetches consumption.
     """
-    if timestamps.count(timestamps[0]) == len(timestamps):
-        unified_timestamp = timestamps[0]
-    else:
-        average_timestamp = sum([dt.timestamp for dt in timestamps])/len(timestamps)
-        unified_timestamp = arrow.get(average_timestamp)
 
-    if check_delta:
-        for ts in timestamps:
-            delta = unified_timestamp - arrow.get(ts)
-            second_difference = abs(delta.total_seconds())
+    if target_datetime:
+        raise NotImplementedError("This parser is not yet able to parse past dates")
 
-            if second_difference > 3600:
-                # more than 1 hour difference
-                raise ValueError("""South Korea generation data is more than 1 hour apart,
-                                 saw {} hours difference""".format(second_difference/3600))
+    r = session or requests.session()
+    url = REAL_TIME_URL
 
-    if with_tz:
-        unified_timestamp = unified_timestamp.replace(tzinfo='Asia/Seoul')
+    response = r.get(url)
+    assert response.status_code == 200
 
-    return unified_timestamp
+    soup = BeautifulSoup(response.text, 'html.parser')
+    consumption_title = soup.find("th", string=re.compile(r"\s*현재부하\s*"))
+    consumption_val = float(consumption_title.find_next_sibling().text.split()[0].replace(",", ""))
 
+    consumption_date_list = soup.find("p", {"class": "info_top"}).text.split(" ")[:2]
+    consumption_date_list[0] = consumption_date_list[0].replace(".", "-").split("(")[0]
+    consumption_date = datetime.datetime.strptime(" ".join(consumption_date_list), "%Y-%m-%d %H:%M")
+    consumption_date = arrow.get(consumption_date, TIMEZONE).datetime
 
-def check_hydro_capacity(plant_name, value, logger) -> Union[bool, ValueError]:
-    """Makes sure that generation for each hydro plant isn't above listed capacity."""
-    try:
-        max_value = HYDRO_CAPACITIES[plant_name]
-    except KeyError:
-        if value != 0.0:
-            logger.warning('New hydro plant seen - {} - {}MW'.format(plant_name, value), extra={'key': 'KR'})
-        return True
+    data = {
+        'consumption': consumption_val,
+        'datetime': consumption_date,
+        'source': url,
+        "zoneKey": zone_key,
+    }
 
-    if value > max_value:
-        logger.warning('{} reports {}MW generation with capacity of {}MW - discarding'.format(plant_name, value, max_value), extra={'key': 'KR'})
-        raise ValueError
-    else:
-        return True
+    return data
 
+@refetch_frequency(datetime.timedelta(hours=1))
+def fetch_price(zone_key='KR', session=None, target_datetime: datetime.datetime = None, logger=logging.getLogger(__name__)):
 
-def fetch_hydro(session, logger):
-    """Returns 2 element tuple in form (float, arrow object)."""
-    req = session.get(HYDRO_URL, verify=False)
-    soup = BeautifulSoup(req.content, 'html.parser')
-    table = soup.find("div", {"class": "dep02Sec"})
+    first_available_date = time_floor(arrow.now(TIMEZONE).shift(days=-6), datetime.timedelta(days=1)).shift(hours=1)
 
-    rows = table.find_all("tr")
+    if target_datetime is not None and target_datetime < first_available_date:
+        raise NotImplementedError("This parser is not able to parse dates more than one week in the past.")
 
-    plant_dts = []
-    total = 0.0
-    for row in rows:
-        sub_row = row.find("td")
-        if not sub_row:
-            # column headers
-            continue
+    if target_datetime is None:
+        target_datetime = arrow.now(TIMEZONE).datetime
 
-        plant_name = row.find("th").text
+    r = session or requests.session()
+    url = PRICE_URL
 
-        generation = sub_row.find("div", {"class": "tdCont alC"})
-        value = float(generation.text)
+    response = r.get(url)
+    assert response.status_code == 200
 
-        tag_dt = generation.findNext("td")
-        dt_naive = arrow.get(tag_dt.text, "YYYY-MM-DD HH:mm:ss")
+    all_data = []
+    table_prices = pd.read_html(response.text, header=0)[0]
 
-        try:
-            check_hydro_capacity(plant_name, value, logger)
-        except ValueError:
-            continue
+    for col_idx in range(1, table_prices.shape[1]):
+        for row_idx in range(24):
 
-        plant_dts.append(dt_naive)
-        total += value
+            day = col_idx
+            hour = row_idx + 1
 
-    dt = timestamp_processor(plant_dts)
+            if hour == 24:
+                hour = 0
+                day += 1
 
-    return total, dt
+            arw_day = arrow.now(TIMEZONE).shift(days=-1 * (7 - day)).replace(hour=hour, minute=0, second=0, microsecond=0)
+            price_value = table_prices.iloc[row_idx, col_idx]
+            
+            data = {
+                'zoneKey': zone_key,
+                'datetime': arw_day.datetime,
+                'currency': 'KRW',
+                'price': price_value,
+                'source': 'new.kpx.or.kr',
+            }
 
+            all_data.append(data)
 
-def fetch_nuclear(session):
-    """Returns 2 element tuple in form (float, arrow object)."""
-    plant_dts = []
-    total = 0.0
-    for url in NUCLEAR_URLS:
-        req = session.get(url)
-        soup = BeautifulSoup(req.content, 'html.parser')
-        table = soup.find("tbody")
-        rows = table.find_all("tr")
+    return all_data
 
-        for row in rows:
-            sub_row = row.find("td")
-            generation = sub_row.find("div", {"class": "tdCont alC"})
+def get_long_term_prod_data(session=None, target_datetime: datetime.datetime = None) -> dict:
+    target_datetime_formatted_daily = target_datetime.strftime("%Y-%m-%d")
 
-            # 1,059 style used
-            deformatted_generation = generation.text.replace(",", "")
+    r = session or requests.session()
 
-            try:
-                total += float(deformatted_generation)
-            except ValueError:
-                # NOTE '0 ○' returned when plant shutdown
-                continue
+    # CSRF token is needed to access the production data
+    r.get(LONG_TERM_PRODUCTION_URL)
+    cookies_dict = r.cookies.get_dict()
 
-            tag_dt = sub_row.findNext("td")
-            text_dt = tag_dt.find("div", {"class": "tdCont alC"})
-            dt = arrow.get(tag_dt.text, "YYYY-MM-DD HH:mm")
+    payload = {
+        'mid': 'a10606030000',
+        'device': 'chart',
+        'view_sdate': target_datetime_formatted_daily,
+        'view_edate': target_datetime_formatted_daily,
+        '_csrf': cookies_dict['XSRF-TOKEN'],
+    }
 
-            plant_dts.append(dt)
+    res = r.post(LONG_TERM_PRODUCTION_URL, payload)
 
-    dt = timestamp_processor(plant_dts)
+    assert res.status_code == 200
 
-    return total, dt
+    all_data = []
 
+    soup = BeautifulSoup(res.text, 'html.parser')
+    table_rows = soup.find_all("tr")[1:]
 
-def fetch_load(session):
-    """Returns 2 element tuple in form (float, arrow object)."""
-    req = session.get(LOAD_URL)
-    soup = BeautifulSoup(req.content, 'html.parser')
+    for row in table_rows:
 
-    load_tag = soup.find("div", {"class": "actual"})
-    present_load = load_tag.find("dt", text=re.compile(r'Present Load'))
-    value = present_load.find_next("dd").text.strip()
+        sanitized_date = [value[:-1] for value in row.find_all("td")[0].text.split(" ")]
+        curr_prod_datetime_string = "-".join(sanitized_date[:3]) + "T" + ":".join(sanitized_date[3:]) + ":00"
+        arw_datetime = arrow.get(curr_prod_datetime_string, "YYYY-MM-DDTHH:mm:ss", tzinfo=TIMEZONE).datetime
 
-    # remove MW units
-    num = value.split(" ")[0]
+        data = {
+            'zoneKey': 'KR',
+            'datetime': arw_datetime,
+            'capacity': {},
+            'production': {},
+            'storage': {},
+            'source': 'https://new.kpx.or.kr'
+        }
 
-    load = float(num.replace(",", ""))
+        row_values = row.find_all("td")
+        production_values = [int("".join(value.text.split(","))) for value in row_values[1:]]
 
-    date_tag = load_tag.find("p", {"class": "date"})
-    despaced = re.sub(r'\s+', '', date_tag.text)
+        # order of production_values
+        # 0. other, 1. gas, 2. renewable, 3. coal, 4. nuclear
+        # other can be negative as well as positive due to pumped hydro
 
-    # remove (day_of_week) part
-    dejunked = re.sub(r'\(.*?\)', ' ', despaced)
-    dt = arrow.get(dejunked, "YYYY.MM.DD HH:mm")
+        data["datetime"] = arw_datetime
+        data["production"]["unknown"] = production_values[0] + production_values[2]
+        data["production"]["gas"] = production_values[1]
+        data["production"]["coal"] = production_values[3]
+        data["production"]["nuclear"] = production_values[4]
 
-    return load, dt
+        all_data.append(data)
+    
+    return all_data
 
+def get_granular_real_time_prod_data(session=None) -> dict:
+    r0 = session or requests.session()
+    res_0 = r0.get(REAL_TIME_URL)
+    chart_data = extract_chart_data(res_0.text)
 
-def fetch_production(zone_key = 'KR', session=None, target_datetime=None, logger=getLogger(__name__)) -> dict:
-    """Requests the last known production mix (in MW) of a given zone."""
-    if target_datetime:
-        raise NotImplementedError('This parser is not yet able to parse past dates')
+    return chart_data
+
+@refetch_frequency(datetime.timedelta(minutes=5))
+def fetch_production(zone_key='KR', session=None,
+                     target_datetime: datetime.datetime = None,
+                     logger: logging.Logger = logging.getLogger(__name__)) -> dict:
 
-    s = session or requests.Session()
+    if target_datetime is not None and target_datetime < arrow.get(2021, 12, 22, 0, 0, 0, tzinfo=TIMEZONE):
+        raise NotImplementedError("This parser is not able to parse dates before 2021-12-22.")
 
-    hydro, hydro_dt = fetch_hydro(s, logger)
-    nuclear, nuclear_dt = fetch_nuclear(s)
-    load, load_dt = fetch_load(s)
+    if target_datetime is None:
+        target_datetime = arrow.now(TIMEZONE).datetime
 
-    generation_dts = [hydro_dt, nuclear_dt, load_dt]
+    all_data = []
 
-    dt_aware = timestamp_processor(generation_dts, with_tz=True, check_delta=True)
+    if target_datetime.date() == arrow.now(TIMEZONE).date():
+        chart_data = get_granular_real_time_prod_data(session=session)
 
-    unknown = load - nuclear - hydro
+        for datetime_key, chart_data_values in chart_data.items():
+            data = {
+                'zoneKey': 'KR',
+                'datetime': datetime_key,
+                'capacity': {},
+                'production': {},
+                'storage': {},
+                'source': 'https://new.kpx.or.kr'
+            }
 
-    production = {
-                  'production': {
-                    'nuclear': nuclear,
-                    'hydro': hydro,
-                    'unknown': unknown
-                  },
-                   'source': 'khnp.co.kr, kpx.or.kr',
-                   'zoneKey': zone_key,
-                   'datetime': dt_aware.datetime,
-                   'storage': {
-                     'hydro': None,
-                     'battery': None
-                  }
-                  }
+            data["storage"]["hydro"] = chart_data_values["pumpedHydro"]
 
-    return production
+            data["production"]["coal"] = chart_data_values["coal"]
+            data["production"]["gas"] = chart_data_values["gas"]
+            data["production"]["nuclear"] = chart_data_values["nuclear"]
+            data["production"]["oil"] = chart_data_values["oil"]
+            data["production"]["hydro"] = chart_data_values["hydro"]
+            data["production"]["unknown"] = chart_data_values["renewable"]
 
+            all_data.append(data)
+
+    else:
+        all_data = get_long_term_prod_data(session=session, target_datetime=target_datetime)
+    
+    return all_data
 
 if __name__ == '__main__':
-    print('fetch_production() -> ')
-    print(fetch_production())
+    # Testing datetime on specific date
+    target_datetime = arrow.get(2022, 2, 7, 16, 35, 0, tzinfo=TIMEZONE).datetime
+
+    print('fetch_production() ->')
+    # pp.pprint(fetch_production(target_datetime=target_datetime))
+    pp.pprint(fetch_production())
+
+    print('fetch_price() -> ')
+    # pp.pprint(fetch_price(target_datetime=target_datetime))
+    pp.pprint(fetch_price())
+
+    print('fetch_consumption() -> ')
+    pp.pprint(fetch_consumption())
\ No newline at end of file


Test patch: