Repository: electricitymaps/electricitymaps-contrib

Issue:
Broken Vietnam parser
## Description
Vietnam parser (consumption and price) crashes when run. This happens because the currently used URLs redirect to the 404 page. Data seems to have moved to other URLs inside the same page. After investigating I've found:
- [https://www.nldc.evn.vn/api/services/app/Pages/GetChartPhuTaiVM](https://www.nldc.evn.vn/api/services/app/Pages/GetChartPhuTaiVM)
  - Contains consumption data for all subregions and whole country, but only for the present day
- [https://www.nldc.evn.vn/api/services/app/Pages/GetChartGiaBienVM](https://www.nldc.evn.vn/api/services/app/Pages/GetChartGiaBienVM)
  - Similar to previous but for price
- [https://www.nldc.evn.vn/api/services/app/Dashboard/GetBieuDoTuongQuanPT?day=24%2F08%2F2023](https://www.nldc.evn.vn/api/services/app/Dashboard/GetBieuDoTuongQuanPT?day=24%2F08%2F2023)
  - Contains consumption and price data for the whole country only, but allows historical
- [https://www.nldc.evn.vn/ThiTruongDien](https://www.nldc.evn.vn/ThiTruongDien)
  - This page contains a table with the last day production data. I link it because maybe the estimation model could use it to predict production.

The current parser seems to be designed to get historical regional data, so I ask for guidance as to what to implement in this case to fix the parser (only daily regional data, historical country data, both (don't now if backend supports this), other, etc)


Discussion:
Thanks for the report!

Resolving patch:
diff --git a/parsers/VN.py b/parsers/VN.py
--- a/parsers/VN.py
+++ b/parsers/VN.py
@@ -1,220 +1,207 @@
 #!/usr/bin/env python3
 
-from datetime import datetime
+import json
+from datetime import datetime, timedelta
 from logging import Logger, getLogger
-from typing import Any
 from zoneinfo import ZoneInfo
 
-import arrow
-from requests import Response, Session
+from requests import Session
 
+from electricitymap.contrib.config import ZoneKey
+from electricitymap.contrib.lib.models.event_lists import (
+    PriceList,
+    TotalConsumptionList,
+)
 from parsers.lib.exceptions import ParserException
 
 ## Vietnamese National Load Dispatch Center https://www.nldc.evn.vn/
 # Access via day, can also parse historical data
 
 tz = ZoneInfo("Asia/Ho_Chi_Minh")
-base_url_from_type = {
-    "consumption": "https://www.nldc.evn.vn/PhuTaiHandle.ashx?d=",  # + dd/mm/yyyy
-    "price": "https://www.nldc.evn.vn/GiaBienHandle.ashx?d=",  # + dd/mm/yyyy
+LIVE_DATA = {
+    "consumption": "https://www.nldc.evn.vn/api/services/app/Pages/GetChartPhuTaiVM",
+    "price": "https://www.nldc.evn.vn/api/services/app/Pages/GetChartGiaBienVM",
 }
 
+HISTORICAL_DATA = "https://www.nldc.evn.vn/api/services/app/Dashboard/GetBieuDoTuongQuanPT"  # + dd/mm/yyyy
 
-def fetch_latest_data(
-    data_type: str,
-    session: Session,
-):
-    """
-    Fetch latest available
-    """
-    fetched_day = arrow.now(tz=tz)
-
-    base_url = base_url_from_type[data_type]
-
-    today_dt_str = fetched_day.format("DD/MM/YYYY")
-
-    # Try fetch data for today.
-    today_response: Response = session.get(base_url + today_dt_str)
-
-    # Check if today's data is there.
-    if today_response.ok and today_response.json() is not None:
-        return fetched_day.floor("day"), today_response.json()
-
-    # If it is too early in the day, it might not contain data yet. Get from yesterday.
-    fetched_day = fetched_day.shift(days=-1)
-    yesterday_dt_str = fetched_day.format("DD/MM/YYYY")
-
-    yesterday_response: Response = session.get(base_url + yesterday_dt_str)
-    # Check if yesterday's data is there.
-    if yesterday_response.ok and yesterday_response.json() is not None:
-        return fetched_day.floor("day"), yesterday_response.json()
-
-    # Also no data from yesterday, cant fetch recent data.
-    raise ParserException(
-        parser="VN.py",
-        message=(
-            f"Requested recent data - request endpoint can't provide data for 2 days."
-            f"Return codes: {today_response.status_code}, {yesterday_response.status_code} at {fetched_day}"
-        ),
-    )
-
+ZONE_CODE = {
+    ZoneKey("VN"): "HT",
+    ZoneKey("VN-N"): "MB",
+    ZoneKey("VN-C"): "MT",
+    ZoneKey("VN-S"): "MN",
+}
 
-def fetch_target_data(data_type: str, target_datetime, session: Session):
-    """
-    Fetch the day containing the given target_datetime.
-    """
 
-    # Convert timezone, and get the day
-    target_datetime_vn = arrow.get(target_datetime).to(tz)
+def fetch_live_data(data_type: str, session: Session):
+    res = session.get(LIVE_DATA[data_type])
+    if not res.ok:
+        raise ParserException(
+            parser="VN.py", message=f"Request failed: {res.status_code}"
+        )
+    return json.loads(res.text)["result"]["data"]
 
-    fetched_day = target_datetime_vn.floor("day")
-    target_dt_str = fetched_day.format("DD/MM/YYYY")
 
-    target_response: Response = session.get(
-        base_url_from_type[data_type] + target_dt_str
-    )
-    if target_response.ok and target_response.json() is not None:
-        return fetched_day, target_response.json()
-
-    # Requested data not available.
-    raise ParserException(
-        parser="VN.py",
-        message=(
-            f"Requested {target_datetime} - request endpoint can't provide data - {target_response.status_code}."
-        ),
+def fetch_historical_data(
+    target_datetime: datetime,
+    session: Session,
+):
+    res = session.get(
+        f"{HISTORICAL_DATA}?day={(target_datetime.astimezone(tz)).strftime('%d/%m/%Y')}"
     )
+    if not res.ok:
+        raise ParserException(
+            parser="VN.py", message=f"Request failed: {res.status_code}"
+        )
+    return json.loads(res.text)["result"]["data"]["phuTai"]
 
 
-def data_index_to_valid_time(base_day: datetime, idx: int):
-    """
-    Convert index in fetched data to associated time, given base date (day)
-    Valid time of indices is: [0:30, 1:00, 1:30, ..., 23:30, 24:00]
-    """
-    idx_datetime = base_day.replace(
-        hour=int((idx + 1) / 2), minute=(0 if idx % 2 else 30)
-    ).floor("minute")
+def fetch_live_price(
+    zone_key: ZoneKey,
+    session: Session,
+    logger: Logger = getLogger(__name__),
+):
+    data_list = fetch_live_data("price", session)["giaBiens"]
 
-    return idx_datetime
+    result_list = PriceList(logger)
 
+    for data in data_list:
+        result_list.append(
+            datetime=datetime.fromisoformat(data["thoiGian"]).replace(tzinfo=tz)
+            - timedelta(minutes=30),
+            price=data[f"giaBien{ZONE_CODE[zone_key]}"],
+            currency="VND",
+            zoneKey=zone_key,
+            source="nldc.evn.vn",
+        )
+    return result_list.to_list()
 
-def fetch_consumption(
-    zone_key: str,
-    session: Session = Session(),
-    target_datetime: datetime | None = None,
+
+def fetch_live_consumption(
+    zone_key: ZoneKey,
+    session: Session,
     logger: Logger = getLogger(__name__),
-) -> dict[str, Any] | list[dict[str, Any]]:
-    request_latest = target_datetime is None
+):
+    data_list = fetch_live_data("consumption", session)["phuTais"]
 
-    if request_latest:
-        fetched_day, fetched_data = fetch_latest_data("consumption", session)
+    result_list = TotalConsumptionList(logger)
 
-    else:
-        # TODO API bug for past data:
-        # data for given date can only be retrieved until the current hour.
-        fetched_day, fetched_data = fetch_target_data(
-            "consumption", target_datetime, session
+    for data in data_list:
+        result_list.append(
+            datetime=datetime.fromisoformat(data["thoiGian"]).replace(tzinfo=tz)
+            - timedelta(minutes=30),
+            consumption=data[f"congSuat{ZONE_CODE[zone_key]}"],
+            zoneKey=zone_key,
+            source="nldc.evn.vn",
         )
+    return result_list.to_list()
 
-    # [0] = load of Northern Vietnam
-    # [1] = load of Central Vietnam
-    # [2] = load of Southern Vietnam
-    # [3] = load of Vietnam (total)
-    per_zone_load = {
-        "VN-N": fetched_data["data"][0],
-        "VN-C": fetched_data["data"][1],
-        "VN-S": fetched_data["data"][2],
-        "VN": fetched_data["data"][3],
-    }
 
-    result_list = []
-
-    for i in range(len(per_zone_load[zone_key])):
-        # Convert i to datetime
-        i_datetime = data_index_to_valid_time(fetched_day, i)
+def fetch_historical_price(
+    target_datetime: datetime,
+    session: Session,
+    logger: Logger = getLogger(__name__),
+):
+    data_list = fetch_historical_data(session=session, target_datetime=target_datetime)
 
-        # Data also goes ~2h into the future! Seems to be some kind of prediction, since these
-        # future values are changing until their valid time is reached.
-        if request_latest and i_datetime > arrow.now(tz=tz):
-            continue
+    result_list = PriceList(logger)
 
+    for data in data_list:
         result_list.append(
-            {
-                "datetime": i_datetime.datetime,
-                "consumption": per_zone_load[zone_key][i],
-                "zoneKey": zone_key,
-                "source": "nldc.evn.vn",
-            }
+            datetime=datetime.fromisoformat(data["thoiGian"]).replace(tzinfo=tz)
+            - timedelta(minutes=30),
+            currency="VND",
+            price=data["giaBan"],
+            zoneKey=ZoneKey("VN"),
+            source="nldc.evn.vn",
         )
+    return result_list.to_list()
 
-    if not len(result_list):
-        raise ParserException(
-            parser="VN.py",
-            message=f"No valid consumption data for requested day found.",
-        )
 
-    return result_list
+def fetch_historical_consumption(
+    target_datetime: datetime,
+    session: Session,
+    logger: Logger = getLogger(__name__),
+):
+    data_list = fetch_historical_data(session=session, target_datetime=target_datetime)
+    result_list = TotalConsumptionList(logger)
+
+    for data in data_list:
+        result_list.append(
+            datetime=datetime.fromisoformat(data["thoiGian"]).replace(tzinfo=tz)
+            - timedelta(
+                minutes=30
+            ),  # Substract 30 minutes to convert backwards looking timestamp to forward looking
+            consumption=data["congSuat"],
+            zoneKey=ZoneKey("VN"),
+            source="nldc.evn.vn",
+        )
+    return result_list.to_list()
 
 
 def fetch_price(
-    zone_key: str,
-    session: Session = Session(),
+    zone_key: ZoneKey,
+    session: Session | None = None,
     target_datetime: datetime | None = None,
     logger: Logger = getLogger(__name__),
-) -> list[dict] | dict:
-    request_latest = target_datetime is None
-
-    if request_latest:
-        fetched_day, fetched_data = fetch_latest_data("price", session)
+):
+    session = session or Session()
+    if target_datetime is None:
+        live_consumption = fetch_live_price(zone_key, session, logger)
+        if len(live_consumption) > 0:
+            return live_consumption
+        else:
+            if zone_key is ZoneKey("VN"):
+                return fetch_historical_price(
+                    datetime.now() - timedelta(days=1), session, logger
+                )
+            else:
+                raise ParserException(
+                    "VN.py",
+                    "No real time data found",
+                    zone_key,
+                )
+    elif target_datetime is not None and zone_key is ZoneKey("VN"):
+        return fetch_historical_price(target_datetime, session, logger)
     else:
-        # TODO API bug for past data:
-        # data for given date can only be retrieved until the current hour.
-        fetched_day, fetched_data = fetch_target_data("price", target_datetime, session)
-
-    # [0] = load of Northern Vietnam
-    # [1] = load of Central Vietnam
-    # [2] = load of Southern Vietnam
-    per_zone_price = {
-        "VN-N": fetched_data["data"][0],
-        "VN-C": fetched_data["data"][1],
-        "VN-S": fetched_data["data"][2]
-        # no total VN data
-    }
-
-    result_list = []
-
-    for i in range(len(per_zone_price[zone_key])):
-        # Convert i to datetime
-        i_datetime = data_index_to_valid_time(fetched_day, i)
-
-        # Data also goes ~2h into the future! Seems to be some kind of prediction, since these
-        # future values are changing until their valid time is reached.
-        if request_latest and i_datetime > arrow.now(tz=tz):
-            continue
-
-        # faulty data at given time?
-        if per_zone_price[zone_key][i] is None:
-            continue
-
-        result_list.append(
-            {
-                "datetime": i_datetime.datetime,
-                "price": per_zone_price[zone_key][i],
-                "currency": "VND",
-                "zoneKey": zone_key,
-                "source": "nldc.evn.vn",
-            }
+        raise NotImplementedError(
+            "This parser is not yet able to parse past dates for this zone"
         )
 
-    if len(result_list) == 0:
-        raise ParserException(
-            parser="VN.py", message=f"No valid prices for requested day found."
-        )
 
-    return result_list
+def fetch_consumption(
+    zone_key: ZoneKey,
+    session: Session | None = None,
+    target_datetime: datetime | None = None,
+    logger: Logger = getLogger(__name__),
+):
+    session = session or Session()
+    if target_datetime is None:
+        live_consumption = fetch_live_consumption(zone_key, session, logger)
+        # Live data is empty at 00:00, resort to historical from previous day
+        if len(live_consumption) > 0:
+            return live_consumption
+        else:
+            if zone_key is ZoneKey("VN"):
+                return fetch_historical_consumption(
+                    datetime.now() - timedelta(days=1), session, logger
+                )
+            else:
+                raise ParserException(
+                    "VN.py",
+                    "No real time data found",
+                    zone_key,
+                )
+    elif target_datetime is not None and zone_key is ZoneKey("VN"):
+        return fetch_historical_consumption(target_datetime, session, logger)
+    else:
+        raise NotImplementedError(
+            "This parser is not yet able to parse past dates for this zone"
+        )
 
 
 if __name__ == "__main__":
     print("fetch_consumption() ->")
-    print(fetch_consumption())
+    print(fetch_consumption(ZoneKey("VN")))
     print("fetch_price() ->")
-    print(fetch_price())
+    print(fetch_price(ZoneKey("VN")))


Test patch: