Repository: electricitymaps/electricitymaps-contrib

Issue:
Add JP forecasts
@tmslaine discovered forecasted data for Japan, which we could add!

See https://github.com/tmrowco/electricitymap-contrib/pull/1543


Discussion:
From the conversation in the issue it looks like we can get forecasts for demand and the exchanges. @tmslaine can you detail a bit more where this forecast data is as opposed to the actual data as it's a bit unclear in the issue.

Resolving patch:
diff --git a/parsers/JP.py b/parsers/JP.py
--- a/parsers/JP.py
+++ b/parsers/JP.py
@@ -19,6 +19,18 @@
 # JP-ON  : Okinawa
 # JP-CG  : Chūgoku
 
+sources = {
+        'JP-HKD':'denkiyoho.hepco.co.jp',
+        'JP-TH':'setsuden.tohoku-epco.co.jp',
+        'JP-TK':'www.tepco.co.jp',
+        'JP-CB':'denki-yoho.chuden.jp',
+        'JP-HR':'www.rikuden.co.jp/denki-yoho',
+        'JP-KN':'www.kepco.co.jp',
+        'JP-SK':'www.yonden.co.jp',
+        'JP-CG':'www.energia.co.jp',
+        'JP-KY':'www.kyuden.co.jp/power_usages/pc.html',
+        'JP-ON':'www.okiden.co.jp/denki/'
+        }
 
 def fetch_production(zone_key='JP-TK', session=None, target_datetime=None,
                      logger=logging.getLogger(__name__)):
@@ -53,20 +65,11 @@ def fetch_production(zone_key='JP-TK', session=None, target_datetime=None,
         else:
             df['imports'] = df['imports']-df[exchname]
     df['prod'] = df['cons']-df['imports']
-    df = df[['datetime', 'prod']]
     # add a row to production for each entry in the dictionary:
-    sources = {
-        'JP-HKD':'denkiyoho.hepco.co.jp',
-        'JP-TH':'setsuden.tohoku-epco.co.jp',
-        'JP-TK':'www.tepco.co.jp',
-        'JP-CB':'denki-yoho.chuden.jp',
-        'JP-HR':'www.rikuden.co.jp/denki-yoho',
-        'JP-KN':'www.kepco.co.jp',
-        'JP-SK':'www.energia.co.jp',
-        'JP-CG':'www.yonden.co.jp'
-        }
+    
     datalist = []
-    for i in range(df.shape[0]):
+    #for i in range(df.shape[0]):
+    for i in df.index:
         data = {
             'zoneKey': zone_key,
             'datetime': df.loc[i, 'datetime'].to_pydatetime(),
@@ -103,7 +106,9 @@ def fetch_consumption_df(zone_key='JP-TK', target_datetime=None,
         'JP-CB': 'http://denki-yoho.chuden.jp/denki_yoho_content_data/juyo_cepco003.csv',
         'JP-KN': 'http://www.kepco.co.jp/yamasou/juyo1_kansai.csv',
         'JP-CG': 'http://www.energia.co.jp/jukyuu/sys/juyo_07_{}.csv'.format(datestamp),
-        'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv'
+        'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv',
+        'JP-KY': 'http://www.kyuden.co.jp/power_usages/csv/juyo-hourly-{}.csv'.format(datestamp),
+        'JP-ON': 'https://www.okiden.co.jp/denki/juyo_10_{}.csv'.format(datestamp)
         }
     # First roughly 40 rows of the consumption files have hourly data,
     # the parser skips to the rows with 5-min actual values 
@@ -111,7 +116,7 @@ def fetch_consumption_df(zone_key='JP-TK', target_datetime=None,
         startrow = 44
     else:
         startrow = 42
-    df = pd.read_csv(consumption_url[zone_key], skiprows=list(range(startrow)),
+    df = pd.read_csv(consumption_url[zone_key], skiprows=startrow,
                      encoding='shift-jis')
     df.columns = ['Date', 'Time', 'cons']
     # Convert 万kW to MW
@@ -121,6 +126,67 @@ def fetch_consumption_df(zone_key='JP-TK', target_datetime=None,
     df = df[['datetime', 'cons']]
     return df
 
+def fetch_consumption_forecast(zone_key='JP-KY', target_datetime=None,
+                      logger=logging.getLogger(__name__)):
+    """
+    Gets consumption forecast for specified zone.
+    Returns a list of dictionaries.
+    """
+    # Currently past dates not implemented for areas with no date in their demand csv files
+    if target_datetime and (zone_key in ['JP-TK', 'JP-TH', 'JP-CB', 'JP-KN', 'JP-SK']):
+        raise NotImplementedError(
+            "Past dates not yet implemented for selected region")
+    datestamp = arrow.get(target_datetime).to('Asia/Tokyo').strftime('%Y%m%d')
+    # Forecasts ahead of current date are not available
+    if datestamp > arrow.get().to('Asia/Tokyo').strftime('%Y%m%d'):
+        raise NotImplementedError(
+            "Future dates(local time) not implemented for selected region")
+        
+    consumption_url = {
+                   'JP-HKD': 'http://denkiyoho.hepco.co.jp/area/data/juyo_01_{}.csv'.format(datestamp),
+                   'JP-TH': 'http://setsuden.tohoku-epco.co.jp/common/demand/juyo_02_{}.csv'.format(datestamp),
+                   'JP-TK': 'http://www.tepco.co.jp/forecast/html/images/juyo-j.csv',
+                   'JP-HR': 'http://www.rikuden.co.jp/denki-yoho/csv/juyo_05_{}.csv'.format(datestamp),
+                   'JP-CB': 'http://denki-yoho.chuden.jp/denki_yoho_content_data/juyo_cepco003.csv',
+                   'JP-KN': 'http://www.kepco.co.jp/yamasou/juyo1_kansai.csv',
+                   'JP-CG': 'http://www.energia.co.jp/jukyuu/sys/juyo_07_{}.csv'.format(datestamp),
+                   'JP-SK': 'http://www.yonden.co.jp/denkiyoho/juyo_shikoku.csv',
+                   'JP-KY': 'http://www.kyuden.co.jp/power_usages/csv/juyo-hourly-{}.csv'.format(datestamp),
+                   'JP-ON': 'https://www.okiden.co.jp/denki/juyo_10_{}.csv'.format(datestamp)
+                   }
+    # Skip non-tabular data at the start of source files
+    if zone_key == 'JP-KN':
+        startrow = 9
+    else:
+        startrow = 7
+    # Read the 24 hourly values
+    df = pd.read_csv(consumption_url[zone_key],skiprows=startrow, nrows = 24, encoding = 'shift-jis')
+    if zone_key == 'JP-KN':
+        df = df[['DATE', 'TIME', '予想値(万kW)']]
+    else:
+        try:
+            df = df[['DATE', 'TIME', '予測値(万kW)']]
+        except KeyError:
+            df = df[['DATE', 'TIME', '予測値（万kW)']]
+    df.columns = ['Date', 'Time', 'fcst']
+
+    df['datetime'] = df.apply(parse_dt, axis=1)
+
+    # convert from 万kW to MW
+    df['fcst'] = 10*df['fcst']
+    # validate
+    df = df.loc[df['fcst']>0]
+    # return format
+    data = []
+    #for i in range(df.shape[0]):
+    for i in df.index:
+            data.append({
+                'zoneKey': zone_key,
+                'datetime': df.loc[i,'datetime'].to_pydatetime(),
+                'value': df.loc[i,'fcst'],
+                'source': sources[zone_key]
+            })
+    return data
 
 def fetch_price(zone_key='JP-TK', session=None, target_datetime=None,
                 logger=logging.getLogger(__name__)):
@@ -155,7 +221,7 @@ def fetch_price(zone_key='JP-TK', session=None, target_datetime=None,
             'zoneKey': zone_key,
             'currency': 'JPY',
             'datetime': row[1]['datetime'].datetime,
-            'price': 1000*row[1][zone_key],# Convert from JPY/kWh to JPY/MWh
+            'price': round(int(1000*row[1][zone_key]),-1),# Convert from JPY/kWh to JPY/MWh
             'source': 'jepx.org'
         })
 
@@ -177,3 +243,19 @@ def parse_dt(row):
     print(fetch_production())
     print('fetch_price() ->')
     print(fetch_price())
+    print('fetch_consumption_forecast() ->')
+    print(fetch_consumption_forecast())
+    print('fetch_consumption_forecast(JP-TH) ->')
+    print(fetch_consumption_forecast('JP-TH'))
+    print('fetch_consumption_forecast(JP-TK) ->')
+    print(fetch_consumption_forecast('JP-TK'))
+    print('fetch_consumption_forecast(JP-CB) ->')
+    print(fetch_consumption_forecast('JP-CB'))
+    print('fetch_consumption_forecast(JP-KN) ->')
+    print(fetch_consumption_forecast('JP-KN'))
+    print('fetch_consumption_forecast(JP-SK) ->')
+    print(fetch_consumption_forecast('JP-SK'))
+    print('fetch_consumption_forecast(JP-KY) ->')
+    print(fetch_consumption_forecast('JP-KY'))
+    print('fetch_consumption_forecast(JP-ON) ->')
+    print(fetch_consumption_forecast('JP-ON'))
diff --git a/parsers/occtonet.py b/parsers/occtonet.py
--- a/parsers/occtonet.py
+++ b/parsers/occtonet.py
@@ -91,8 +91,8 @@ def fetch_exchange(zone_key1='JP-TH', zone_key2='JP-TK', session=None,
     Headers = get_request_token(r, Headers, Cookies)
     Headers_1 = get_request_token(r, Headers_1, Cookies)
     # Query data
-    data = get_data(r, Headers, Cookies)
-    data_1 = get_data(r, Headers_1, Cookies)
+    data = get_exchange(r, Headers, Cookies)
+    data_1 = get_exchange(r, Headers_1, Cookies)
     # Concatenate d-1 and current day
     df = pd.concat([data_1,data])
     # CB-HR -exceptions
@@ -106,8 +106,8 @@ def fetch_exchange(zone_key1='JP-TH', zone_key2='JP-TK', session=None,
         Headers = get_request_token(r, Headers, Cookies)
         Headers_1 = get_request_token(r, Headers_1, Cookies)
     
-        data = get_data(r, Headers, Cookies)
-        data_1 = get_data(r, Headers_1, Cookies)
+        data = get_exchange(r, Headers, Cookies)
+        data_1 = get_exchange(r, Headers_1, Cookies)
     
         df2 = pd.concat([data_1,data])
         df2 = df2.set_index(['Date', 'Time'])
@@ -135,7 +135,59 @@ def fetch_exchange(zone_key1='JP-TH', zone_key2='JP-TK', session=None,
     for result in results:
         result['datetime'] = result['datetime'].to_pydatetime()
     return results
+
+def fetch_exchange_forecast(zone_key1='JP-TH', zone_key2='JP-TK', session=None,
+                   target_datetime=None, logger=logging.getLogger(__name__)):
+    """
+    Gets exchange forecast between two specified zones.
+    Returns a list of dictionaries.
+    """
+    #get target date in time zone Asia/Tokyo
+    query_date = arrow.get(target_datetime).to('Asia/Tokyo').strftime('%Y/%m/%d')
+    # Forecasts ahead of current date are not available
+    if query_date > arrow.get().to('Asia/Tokyo').strftime('%Y/%m/%d'):
+        raise NotImplementedError(
+            "Future dates(local time) not implemented for selected exchange")
     
+    sortedZoneKeys = '->'.join(sorted([zone_key1, zone_key2]))
+    exch_id = exchange_mapping[sortedZoneKeys]
+    # Login to occtonet
+    r = session or requests.session()
+    Cookies = get_cookies(r)
+    Headers = get_headers(r, exch_id[0], query_date, Cookies)
+    # Query data
+    Headers = get_request_token(r, Headers, Cookies)
+    df = get_exchange_fcst(r, Headers, Cookies)
+    # CB-HR -exceptions
+    if sortedZoneKeys == 'JP-CB->JP-HR':
+        
+        df = df.set_index(['Date', 'Time'])
+        Headers = get_headers(r, exch_id[1], query_date, Cookies)
+        Headers = get_request_token(r, Headers, Cookies)
+        df2 = get_exchange_fcst(r, Headers, Cookies)
+        df2 = df2.set_index(['Date', 'Time'])
+        df = df + df2
+        df = df.reset_index()
+        
+    # fix possible occurrences of 24:00hrs    
+    list24 = list(df.index[df['Time']=='24:00'])
+    for idx in list24:
+        df.loc[idx, 'Date'] = arrow.get(str(df.loc[idx, 'Date'])).shift(days=1).strftime('%Y/%m/%d')
+        df.loc[idx, 'Time'] = '00:00'
+    # correct flow direction, if needed
+    flows_to_revert = ['JP-CB->JP-TK', 'JP-CG->JP-KN', 'JP-CG->JP-SK']
+    if sortedZoneKeys in flows_to_revert:
+        df['netFlow'] = -1 * df['netFlow']
+    # Add zonekey, source    
+    df['source'] = 'occtonet.occto.or.jp'
+    df['datetime'] = df.apply(parse_dt, axis=1)
+    df['sortedZoneKeys'] = sortedZoneKeys
+    df = df[['source', 'datetime', 'netFlow', 'sortedZoneKeys']]
+    # Format output
+    results = df.to_dict('records')
+    for result in results:
+        result['datetime'] = result['datetime'].to_pydatetime()
+    return results   
     
 def get_cookies(session=None):
     s = session or requests.session()
@@ -196,7 +248,7 @@ def get_request_token(session, payload, cookies):
         payload['requestToken'] = headers['root']['bizRoot']['header']['requestToken']['value']
     return payload
 
-def get_data(session, payload, cookies):
+def get_exchange(session, payload, cookies):
     s = session
     payload['fwExtention.actionSubType']='download'
     r = s.post('http://occtonet.occto.or.jp/public/dfw/RP11/OCCTO/SD/CA01S070C?'
@@ -209,9 +261,160 @@ def get_data(session, payload, cookies):
     df = df.dropna()
     return df
 
+def get_exchange_fcst(session, payload, cookies):
+    s = session
+    payload['fwExtention.actionSubType']='download'
+    r = s.post('http://occtonet.occto.or.jp/public/dfw/RP11/OCCTO/SD/CA01S070C?fwExtention.pathInfo=CA01S070C&fwExtention.prgbrh=0',
+                   cookies=cookies, data=payload)
+    r.encoding = 'shift-jis'
+    df = pd.read_csv(StringIO(r.text), delimiter=',')
+    df = df[['対象日付', '対象時刻', '計画潮流(順方向)']]
+    df.columns = ['Date', 'Time', 'netFlow']
+    df = df.dropna()
+    return df
+
 def parse_dt(row):
     return arrow.get(' '.join([row['Date'], row['Time']]).replace('/', '-')).replace(tzinfo='Asia/Tokyo').datetime
 
+def zone_headers(zone, date):
+    """
+    Get query headers for querying intra-zone flows
+    """
+    area_map={
+    'JP-HKD':1,
+    'JP-TH':2,
+    'JP-TK':3,
+    'JP-CB':4,
+    'JP-HR':5,
+    'JP-KN':6,
+    'JP-CG':7,
+    'JP-SK':8,
+    'JP-KY':9,
+    'JP-ON':10
+    }
+    payload = {
+            'ajaxToken':'',
+            'areaCdAreaSumNon':'{:02d}'.format(area_map[zone]),
+            'daPtn':'00',
+            'daPtn1':'187',
+            'daPtn2':'275',
+            'daPtn3':'220',
+            'daPtn4':'187',
+            'daPtn5':'132',
+            'downloadKey':'',
+            'fwExtention.actionSubType':'headerInput',
+            'fwExtention.actionType':'reference',
+            'fwExtention.formId':'CB01S020P',
+            'fwExtention.jsonString':'',
+            'fwExtention.pagingTargetTable':'',
+            'fwExtention.pathInfo':'CB01S020C',
+            'fwExtention.prgbrh':'0',
+            'msgArea':'',
+            'requestToken':'',
+            'requestTokenBk':'',
+            'searchReqHdn':'',
+            'table1.currentPage':'',
+            'table1.dispRowNum':'200',
+            'table1.endIndex':'',
+            'table1.nextPageStartIndex':'',
+            'table1.pagingMode':'editable',
+            'table1.rows[0].rowParams.originRowIndex':'0',
+            'table1.rows[0].rowParams.rowAddUpdate':'',
+            'table1.rows[0].rowParams.rowDelete':'',
+            'table1.rows[0].rowParams.rowNum':'',
+            'table1.rows[0].rowParams.selected':'',
+            'table1.startIndex':'',
+            'table1.tableIdToken':'',
+            'table1.totalPage':'',
+            'tgtAreaHdn':'',
+            'tgtDaHdn':'',
+            'tgtNngp': date,
+            'tgtNngpHdn':'',
+            'transitionContextKey':'DEFAULT',
+            'updDaytime':''
+          }
+
+    with requests.Session() as s:
+    
+    
+        r = s.get('http://occtonet.occto.or.jp/public/dfw/RP11/OCCTO/SD/LOGIN_login')
+    
+    r2 = s.post('http://occtonet.occto.or.jp/public/dfw/RP11/OCCTO/SD/CB01S020C?fwExtention.pathInfo=CB01S020C&fwExtention.prgbrh=0',
+                   cookies=s.cookies, data=payload)
+    r2.encoding = 'utf-8'
+    headers=r2.text
+    headers = eval(headers.replace('false', 'False').replace('null', 'None'))
+    return headers
+
+def solve_flows(headers):
+    """
+    Solves flow equations for each node inside a zone for a given day,
+    allowing possibly to add large isolated thermal generators, pump storages etc. to eMap
+    Japanese regions in the future
+    
+    Expects a dictionary argument obtained by function zone_headers(zone, date)
+    Returns net generation for each node at 30 minute intervals in a Pandas DataFrame
+    """
+    dict1 = headers['root']['bizRoot']['table']['table1']['table1']['rows']
+    # For each transmission line:
+    for entry in dict1:
+        # Collect flows at each timestamp
+        for i in range(1,49):
+            column = 'colonLbl'+'{:02d}'.format((i*30)//60)+'{:02d}'.format((i*30)%60)
+            try:
+                entry[column] = entry[column]['value']
+            except TypeError:
+                pass
+        # Collect line names
+        if str(type(entry['flowHukuSeiHuku']))=="<class 'dict'>":
+            entry['flowHukuSeiHuku'] = entry['flowHukuSeiHuku']['value']
+        # Remove unneeded keys from dictionary for parsing it into DataFrame
+        for key in ['daKv','rowParams.originRowIndex', 'rowParams.rowAddUpdate',
+                   'rowParams.rowDelete', 'rowParams.rowNum', 'rowParams.selected',
+                   'sdlNm']:
+            try:
+                del entry[key]
+            except KeyError:
+                pass
+    # Format to DataFrame, with timestamps as rows
+    df = pd.DataFrame(dict1).T
+    df.columns = df.iloc[-1]
+    df=df.drop(df.index[-1])
+
+    df2 = pd.DataFrame()
+    #Sum flows for each node
+    for i in range(len(df.columns)):
+        col = df.columns[i]
+        src = col.split(' → ')[0]
+        tgt = col.split(' → ')[-1]
+    
+        if src in df2.columns:
+            df2[src] = df2[src]+pd.to_numeric(df.iloc[:,i])
+        else:
+            df2[src] = pd.to_numeric(df.iloc[:,i])
+    
+        if tgt in df2.columns:
+            df2[tgt] = df2[tgt]-pd.to_numeric(df.iloc[:,i])
+        else:
+            df2[tgt] = -1*pd.to_numeric(df.iloc[:,i])
+    timestamps = []
+    # convert timestamps
+    date = headers['root']['bizRoot']['header']['tgtNngp']['value']
+    for ts in list(df2.index.values):
+        time = headers['root']['bizRoot']['header'][ts]['label']
+        timestamps.append(
+            arrow.get('{0} {1}'.format(date, time).replace(
+            '/', '-')).replace(tzinfo='Asia/Tokyo').datetime
+        )
+    df2 = df2.reset_index()
+    df2['timestamps']=pd.Series(timestamps)
+    df2=df2.set_index('timestamps')
+    df2=df2.drop('index', axis=1)
+    df2=df2.dropna()
+    return df2
+
+
+
 if __name__ == '__main__':
     """Main method, never used by the Electricity Map backend, but handy for testing."""
 
@@ -219,4 +422,8 @@ def parse_dt(row):
     print(fetch_exchange('JP-CB', 'JP-HR')[-3:])
     print('fetch_exchange(JP-CG, JP-KY) ->')
     print(fetch_exchange('JP-CG', 'JP-KY')[-3:])
+    print('fetch_exchange_forecast(JP-CB, JP-HR) ->')
+    print(fetch_exchange_forecast('JP-CB', 'JP-HR')[-3:])
+    print('fetch_exchange_forecast(JP-CG, JP-KY) ->')
+    print(fetch_exchange_forecast('JP-CG', 'JP-KY')[-3:])
     


Test patch: