Repository: electricitymaps/electricitymaps-contrib

Issue:
More up-to-date data source for Bangladesh
Hey all,

the Power Grid Company of Bangladesh has a new table for its production and import data. They seem to be updated only in batches every 5-6 hours, therefore afaik basically never fulfill the real time requirement of a <2h delay. The current parser parses Daily Reports from the day before via excel files (which match this data source). Is there interest in a parser for this data source, because it contains more recent data, but still not real-time?

https://erp.pgcb.gov.bd/web/generations/view_generations
https://erp.pgcb.gov.bd/web/generations/imported_power


Discussion:
Hi @MuffinCompiler and thanks for finding the source!

Looks like the current parser isn't even functioning as expected so if you want to either fix that up or create a new one with this source that would be great and much appreciated.

Having realtime data would be great but if there is just a few hours delay that is not an issue.
I also noticed that the exchange config for the exchange between Bangladesh and India is wrong and should be a subzone of India instead so take note of that if you want to create a parse so you don't have to change it later. 

Resolving patch:
diff --git a/parsers/BD.py b/parsers/BD.py
--- a/parsers/BD.py
+++ b/parsers/BD.py
@@ -1,299 +1,320 @@
 #!/usr/bin/env python3
 
-"""Parser for Bangladesh."""
-
-from datetime import datetime
+from datetime import datetime, timedelta
 from logging import Logger, getLogger
-from typing import List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import arrow
-import pandas as pd
-from requests import Session
-
-from .lib.exceptions import ParserException
-
-GENERATION_MAPPING = {
-    "Gas (Public)": "gas_public",
-    "Gas (Private)": "gas_private",
-    "Oil (Public)": "oil_public",
-    "Oil (Private)": "oil_private",
-    "Coal": "coal",
-    "Hydro": "hydro",
-    "Solar": "solar",
-}
-
-
-OLD_GENERATION_MAPPING = {
-    "GAS": "gas_public",
-    "P.GEN(GAS).": "gas_private",
-    "OIL": "oil_public",
-    "P.GEN(OIL).": "oil_private",
-    "COAL": "coal",
-    "HYDRO": "hydro",
-    "SOLAR": "solar",
-}
-
-
-def timestamp_converter(raw_timestamp, target_datetime):
-    """Converts string timestamp (e.g. 10:00:00) to arrow object."""
-
-    hour, minute = raw_timestamp.split(":")[0:2]
+import pytz
+from bs4 import BeautifulSoup, Tag
+from requests import Response, Session
+
+from parsers.lib.config import refetch_frequency
+from parsers.lib.exceptions import ParserException
+
+# Power Grid Company of Bangladesh: erp.pgcb.gov.bd
+# Has table (also historical) of production, consumption and exchange.
+# This table gets updated batch-wise every few hours, so most of the time, the delay will be >2h.
+
+tz = pytz.timezone("Asia/Dhaka")
+latest_url = "https://erp.pgcb.gov.bd/web/generations/view_generations"
+url_by_date = (
+    "https://erp.pgcb.gov.bd/web/generations/view_generations?search="  # DD-MM-YYYY
+)
+
+TABLE_HEADERS = [
+    "Date",
+    "Time",
+    "Generation(MW)",
+    "Demand(MW)",
+    "Loadshed",
+    "Gas",
+    "Liquid Fuel",
+    "Coal",
+    "Hydro",
+    "Solar",
+    "Bheramara HVDC",
+    "Tripura",
+    "Remarks",
+]
+
+
+def table_entry_to_float(entry: str):
+    """
+    Helper function to handle empty cells in table.
+    """
+    if entry == "":
+        return None
     try:
-        dt_aware = target_datetime.replace(
-            hour=int(hour), minute=int(minute), tzinfo="Asia/Dhaka"
-        )
+        return float(entry)
     except ValueError:
-        # 24:00 present in data, requires shifting forward.
-        dt_aware = target_datetime.shift(days=+1)
-
-    return dt_aware
-
-
-def old_format_converter(df):
-    """Returns a dataframe."""
-
-    df = df.dropna(axis="columns", thresh=20)
-    df = df.dropna(axis="index", thresh=4)
-    df["TIME"] = df["TIME"].astype(str)
-    df = df.reset_index(drop=True)
-    df = df.set_index("TIME")
-
-    return df
-
-
-def new_format_converter(df, logger: Logger):
-    """Returns a dataframe."""
-
-    df = df.rename(columns={"Plant Name": "Hour"})
-    df["Hour"] = df["Hour"].astype(str)
-    df = df.reset_index(drop=True)
-    df = df.set_index("Hour")
-    total_index = df.columns.get_loc("Total (MW)")
-    df = df.iloc[:, total_index:]
-
-    try:
-        time_index = df.index.get_loc("24:00")
-    except KeyError:
         raise ParserException(
-            "BD.py", "Structure of xlsm file for BD has altered, unable to parse.", "BD"
-        )
-
-    df = df.iloc[: time_index + 1]
-
-    # check for new columns
-    if df.shape[1] != 12:
-        logger.warning(
-            "New data columns may be present xlsm file.", extra={"key": "BD"}
+            parser="BD.py",
+            message=(f'Failed to parse entry: "{entry}" to float in table.'),
         )
 
-    return df
 
-
-def production_processer(df, target_datetime, old_format=False) -> list:
+def parse_table_body(table_body: Tag):
     """
-    Takes dataframe and extracts all production data and timestamps.
-    Returns a list of 2 element tuples in form (dict, arrow object).
+    Parse the table body returned by the URL.
+    Returns a list of rows represented by dicts.
     """
 
-    if old_format:
-        MAPPING = OLD_GENERATION_MAPPING
-    else:
-        MAPPING = GENERATION_MAPPING
-
-    processed_data = []
-    for index, row in df.iterrows():
-        production = row.to_dict()
-        dt = timestamp_converter(index, target_datetime)
-
-        mapped_production = {
-            MAPPING[k]: v for k, v in production.items() if k in MAPPING
-        }
-        mapped_production["gas"] = mapped_production.pop(
-            "gas_public"
-        ) + mapped_production.pop("gas_private")
-        mapped_production["oil"] = mapped_production.pop(
-            "oil_public"
-        ) + mapped_production.pop("oil_private")
-        processed_data.append((mapped_production, dt))
+    rows = table_body.find_all("tr")
+    row_data = []
+
+    for row in rows:
+        row_items = row.find_all("td")
+        row_items = [item.text.strip() for item in row_items]
+
+        row_dict = dict()
+
+        # date and time in [0]; [1] are DD-MM-YYYY; HH:mm[:ss]
+        parsed_day = arrow.get(row_items[0], "DD-MM-YYYY", tzinfo=tz).datetime
+        try:
+            parsed_time = arrow.get(row_items[1], "HH:mm:ss", tzinfo=tz).time()
+        except arrow.parser.ParserMatchError:
+            # very old data points are sometimes in HH:mm format
+            parsed_time = arrow.get(row_items[1], "HH:mm", tzinfo=tz).time()
+
+        row_dict["time"] = datetime.combine(parsed_day, parsed_time, tzinfo=tz)
+
+        # [2] is total generation in MW
+        row_dict["total_generation"] = table_entry_to_float(row_items[2])
+        # [3] is total demand in MW
+        row_dict["total_demand"] = table_entry_to_float(row_items[3])
+        # [4] is loadshed, not interesting for us
+        row_dict["loadshed"] = table_entry_to_float(row_items[4])
+        # [5] is generation from gas
+        row_dict["gas"] = table_entry_to_float(row_items[5])
+        # [6] is generation from liquid fuel (oil)
+        row_dict["oil"] = table_entry_to_float(row_items[6])
+        # [7] is generation from coal
+        row_dict["coal"] = table_entry_to_float(row_items[7])
+        # [8] is generation from hydro
+        row_dict["hydro"] = table_entry_to_float(row_items[8])
+        # [9] is generation from solar
+        row_dict["solar"] = table_entry_to_float(row_items[9])
+        # [10], [11] is import from Bheramara PP (IN-EA) and Tripura (IN-NE)
+        row_dict["bd_import_bheramara"] = table_entry_to_float(row_items[10])
+        row_dict["bd_import_tripura"] = table_entry_to_float(row_items[11])
+        row_dict["remarks"] = row_items[12]
+
+        row_data.append(row_dict)
+
+    return row_data
+
+
+def verify_table(table_header: Tag):
+    """
+    Validate the table headers, that it looks like expected.
+    Don't parse if the table has changed.
+    """
+    header_items = table_header.find_all("th")
+    header_items = [item.text.strip() for item in header_items]
 
-    return processed_data
+    if header_items != TABLE_HEADERS:
+        raise ParserException(
+            parser="BD.py",
+            message=(
+                f"Table headers mismatch with expected ones."
+                f"Expected: {TABLE_HEADERS}"
+                f"Parsed: {header_items}"
+            ),
+        )
 
 
-def exchange_processer(df, target_datetime, old_format=False) -> list:
+def query(
+    session: Session, target_datetime: Optional[datetime], logger: Logger
+) -> List[Dict[str, Any]]:
     """
-    Takes dataframe and extracts all exchange data and timestamps.
-    Returns a list of 2 element tuples in form (dict, arrow object).
+    Query the table and read it into list.
     """
+    # build URL to call
+    if target_datetime is None:
+        target_url = latest_url
+    else:
+        # Convert timezone of target_datetime, and build URL from day
+        target_datetime_bd = arrow.get(target_datetime).to(tz)
+        target_dt_str = target_datetime_bd.format("DD-MM-YYYY")
+        target_url = url_by_date + target_dt_str
 
-    # Positive means import from India hence sign reversal needed for EM.
-    processed_data = []
-    for index, row in df.iterrows():
-        exchange = row.to_dict()
-        flow = exchange.pop("HVDC", 0.0) + exchange.pop("Tripura", 0.0)
-        dt = timestamp_converter(index, target_datetime)
-        processed_data.append((-1 * flow, dt))
+    target_response: Response = session.get(target_url)
 
-    return processed_data
+    if not target_response.ok:
+        raise ParserException(
+            parser="BD.py",
+            message=f"Data request did not succeed: {target_response.status_code}",
+        )
 
+    response_soup = BeautifulSoup(target_response.text)
 
-def excel_handler(shifted_target_datetime, logger: Logger) -> tuple:
-    """
-    Decides which url to request based on supplied arrow object.
-    Converts returned excel data into dataframe, format of data varies by date.
-    Returns a tuple containing (dataframe, bool).
-    """
+    # Find the table, there is only one, and verify its headers.
+    table = response_soup.find("table")
+    if table is None:
+        raise ParserException(
+            parser="BD.py",
+            message=f"Could not find table in returned HTML.",
+        )
 
-    # NOTE file named 11-01-2019 actually covers 10-01-2019, pattern repeats!
-    # xls structure very different pre 11-01-18
-    # Before 03-07-2015 it's Daily_Report not Daily Report
-    # File name format adds space on 11-01-18
-    # Odd format around this time (XLS_SPAN)
-    # However file ending changes several days later! (╯°□°）╯︵ ┻━┻
+    table_head = table.find("thead")
+    if table_head is None:
+        raise ParserException(
+            parser="BD.py",
+            message=("Could not find table header in returned HTML."),
+        )
+    verify_table(table_head)
 
-    OLD_FORMAT = False
-    if shifted_target_datetime <= arrow.get("20180111", "YYYYMMDD"):
-        OLD_FORMAT = True
+    # Table valid as we expect, parse the rows.
+    table_body = table.find("tbody")
+    if table_body is None:
+        raise ParserException(
+            parser="BD.py",
+            message=("Could not find table body in returned HTML."),
+        )
+    row_data = parse_table_body(table_body)
 
-    XLS_SPAN = (datetime(2018, 1, 12), datetime(2018, 1, 13), datetime(2018, 1, 14))
+    return row_data
 
-    XLS_END = False
-    if shifted_target_datetime.naive in XLS_SPAN:
-        XLS_END = True
 
-    UNDERSCORE = False
-    if shifted_target_datetime <= arrow.get("20150703", "YYYYMMDD"):
-        UNDERSCORE = True
+@refetch_frequency(timedelta(days=1))
+def fetch_production(
+    zone_key: str = "BD",
+    session: Session = Session(),
+    target_datetime: Optional[datetime] = None,
+    logger: Logger = getLogger(__name__),
+) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
 
-    day = shifted_target_datetime.format("DD")
-    month = shifted_target_datetime.format("MM")
-    short_year = shifted_target_datetime.format("YY")
+    row_data = query(session, target_datetime, logger)
 
-    # NOTE %20 padders needed, format is day-month-year
-    if OLD_FORMAT and UNDERSCORE:
-        URL = "https://pgcb.org.bd/PGCB/upload/Reports/Daily_Report{}-{}-{}.xls".format(
-            day, month, short_year
-        )
-    elif OLD_FORMAT:
-        URL = (
-            "https://pgcb.org.bd/PGCB/upload/Reports/Daily%20Report{}-{}-{}.xls".format(
-                day, month, short_year
-            )
-        )
-    elif XLS_END:
-        URL = "https://pgcb.org.bd/PGCB/upload/Reports/Daily%20Report%20{}-{}-{}.xls".format(
-            day, month, short_year
-        )
-    else:
-        URL = "https://pgcb.org.bd/PGCB/upload/Reports/Daily%20Report%20{}-{}-{}.xlsm".format(
-            day, month, short_year
-        )
+    production_data_list = []
+    for row in row_data:
 
-    if OLD_FORMAT:
-        df = pd.read_excel(URL, sheet_name="En.Curve", skiprows=[0, 1, 2, 3])
-        df = old_format_converter(df)
-    else:
-        if XLS_END:
-            df = pd.read_excel(
-                URL,
-                sheet_name="YesterdayGen",
-                skiprows=[0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
-            )
-            df = new_format_converter(df, logger)
-        else:
-            df = pd.read_excel(URL, sheet_name="YesterdayGen", skiprows=[0, 1, 3])
-            df = new_format_converter(df, logger)
+        # Create data with empty production
+        data = {
+            "zoneKey": zone_key,
+            "datetime": row["time"],
+            "production": {},
+            "source": "erp.pgcb.gov.bd",
+        }
 
-    return df, OLD_FORMAT
+        # And add sources if they are present in the table
+        known_sources_sum_mw = 0.0
+        for source_type in ["coal", "gas", "hydro", "oil", "solar"]:
+            if row[source_type] is not None:
+                # also accumulate the sources to infer 'unknown'
+                known_sources_sum_mw += row[source_type]
+                data["production"][source_type] = row[source_type]
+
+        # infer 'unknown'
+        if row["total_generation"] is not None:
+            unknown_source_mw = row["total_generation"] - known_sources_sum_mw
+            if unknown_source_mw >= 0:
+                data["production"]["unknown"] = unknown_source_mw
+            else:
+                logger.warn(
+                    f"Sum of production sources exceeds total generation by {-unknown_source_mw}MW."
+                    f"There is probably something wrong..."
+                )
+
+        production_data_list.append(data)
+
+    if not len(production_data_list):
+        raise ParserException(
+            parser="BD.py",
+            message=f"No valid consumption data for requested day found.",
+        )
+    return production_data_list
 
 
-def fetch_production(
+@refetch_frequency(timedelta(days=1))
+def fetch_consumption(
     zone_key: str = "BD",
-    session: Optional[Session] = None,
-    target_datetime=None,
+    session: Session = Session(),
+    target_datetime: Optional[datetime] = None,
     logger: Logger = getLogger(__name__),
-) -> List[dict]:
-    """Requests the last known production mix (in MW) of a given country."""
-
-    if not target_datetime:
-        raise NotImplementedError(
-            """This parser is only able to get historical
-                                  data up to the previous day, please pass a
-                                  target_datetime in format YYYYMMDD."""
-        )
+) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
 
-    target_datetime = arrow.get(target_datetime, "YYYYMMDD")
-    shifted_target_datetime = target_datetime.shift(days=+1)
+    row_data = query(session, target_datetime, logger)
 
-    df, OLD_FORMAT = excel_handler(shifted_target_datetime, logger)
+    result_list = []
 
-    generation = production_processer(df, target_datetime, old_format=OLD_FORMAT)
+    for row in row_data:
+        # get the demand from the table
+        consumption = row["total_demand"]
+        if consumption is None:
+            continue  # no data in table
 
-    data = []
-    for item in generation:
-        datapoint = {
-            "zoneKey": zone_key,
-            "datetime": item[1].datetime,
-            "production": item[0],
-            "storage": {},
-            "source": "pgcb.org.bd",
-        }
+        result_list.append(
+            {
+                "datetime": row["time"],
+                "consumption": consumption,
+                "zoneKey": zone_key,
+                "source": "erp.pgcb.gov.bd",
+            }
+        )
 
-        data.append(datapoint)
+    if not len(result_list):
+        raise ParserException(
+            parser="BD.py",
+            message=f"No valid consumption data for requested day found.",
+        )
 
-    return data
+    return result_list
 
 
+@refetch_frequency(timedelta(days=1))
 def fetch_exchange(
     zone_key1: str,
     zone_key2: str,
-    session: Optional[Session] = None,
+    session: Session = Session(),
     target_datetime: Optional[datetime] = None,
     logger: Logger = getLogger(__name__),
-) -> list:
-    """Requests the last known power exchange (in MW) between two zones."""
-    if not target_datetime:
-        raise NotImplementedError(
-            """This parser is only able to get historical
-                                  data up to the previous day, please pass a
-                                  target_datetime in format YYYYMMDD."""
-        )
+) -> List[Dict[str, Any]]:
 
-    sorted_codes = "->".join(sorted([zone_key1, zone_key2]))
+    # Query table, contains import from india.
+    row_data = query(session, target_datetime, logger)
 
-    target_datetime = arrow.get(target_datetime, "YYYYMMDD")
-    shifted_target_datetime = target_datetime.shift(days=+1)
+    result_list = []
+    sortedZoneKeys = "->".join(sorted([zone_key1, zone_key2]))
 
-    df, OLD_FORMAT = excel_handler(shifted_target_datetime, logger)
-    exchange = exchange_processer(df, target_datetime, old_format=OLD_FORMAT)
+    for row in row_data:
 
-    data = []
-    for item in exchange:
-        datapoint = {
-            "sortedZoneKeys": sorted_codes,
-            "datetime": item[1].datetime,
-            "netFlow": item[0],
-            "source": "pgcb.org.bd",
-        }
+        # BD -> IN_xx
+        if zone_key2 == "IN-NE":
+            # Export to India NorthEast via Tripura
+            bd_import = row["bd_import_tripura"]
+        elif zone_key2 == "IN-EA":
+            # Export to India East via Bheramara
+            bd_import = row["bd_import_bheramara"]
+        else:
+            raise ParserException(
+                parser="BD.py",
+                message=f"Exchange pair {sortedZoneKeys} is not implemented.",
+            )
 
-        data.append(datapoint)
+        if bd_import is None:
+            continue  # no data in table
+        bd_export = -1.0 * bd_import
+
+        result_list.append(
+            {
+                "sortedZoneKeys": sortedZoneKeys,
+                "datetime": row["time"],
+                "netFlow": bd_export,
+                "source": "erp.pgcb.gov.bd",
+            }
+        )
 
-    return data
+    return result_list
 
 
 if __name__ == "__main__":
-    """Main method, never used by the Electricity Map backend, but handy for testing."""
-
-    print("fetch_production(target_datetime=20190110)")
-    print(fetch_production(target_datetime="20190110"))
-    print("fetch_production(target_datetime=20180113)")
-    print(fetch_production(target_datetime="20180113"))
-    print("fetch_production(target_datetime=20170110)")
-    print(fetch_production(target_datetime="20170110"))
-    print("fetch_production(target_datetime=20150601)")
-    print(fetch_production(target_datetime="20150601"))
-    print("fetch_exchange(target_datetime=20190109)")
-    print(fetch_exchange("BD", "IN", target_datetime="20190109"))
-    print("fetch_exchange(target_datetime=20170109)")
-    print(fetch_exchange("BD", "IN", target_datetime="20170109"))
-    print("fetch_exchange(target_datetime=20150109)")
-    print(fetch_exchange("BD", "IN", target_datetime="20150109"))
+    print("fetch_production() ->")
+    print(fetch_production())
+    print("fetch_consumption() ->")
+    print(fetch_consumption())
+    print("fetch_exchange('BD', 'IN-NE') ->")
+    print(fetch_exchange("BD", "IN-NE"))
+    print("fetch_exchange('BD', 'IN-EA') ->")
+    print(fetch_exchange("BD", "IN-EA"))


Test patch: