Repository: electricitymaps/electricitymaps-contrib

Issue:
CO production parser down
## Description

This is an automatic error report generated for Colombia (CO).

Issues:
- No recent data found for `production` parser
- No recent data found for `price` parser

## Suggestions
- Try running the parser locally using the command `poetry run test_parser CO production`
- <a href="https://storage.googleapis.com/electricitymap-parser-logs/CO.html">Explore the runtime logs</a>

You can see an overview of all parser issues [here](https://github.com/tmrowco/electricitymap-contrib/wiki/Parser-issues).



Discussion:
Looks like this might just be a change in the data availability delay for the parser? I was able to reproduce errors for both production and price by just running the parser locally, and changing the delay parameter to `XM_DELAY = 3` returned data for both parsers.
> Looks like this might just be a change in the data availability delay for the parser? I was able to reproduce errors for both production and price by just running the parser locally, and changing the delay parameter to `XM_DELAY = 3` returned data for both parsers.

Let's just incase the delay to 3 then.
Could you open a PR for it? 
Hmm, looks like I had to change it to `XM_DELAY = 4` to get the data, now that some time has passed. Should I wait a couple days to see what the actual update interval is and then make the PR?
Maybe you could try wrapping it in a loop that increment until it get data (with a reasonable max value).

(It don't have to be a loop, could be a if statement or try catch as well). 
Currently I'm getting 503 errors from the parser, will try again later to see if the service is back.

Resolving patch:
diff --git a/parsers/CO.py b/parsers/CO.py
--- a/parsers/CO.py
+++ b/parsers/CO.py
@@ -37,7 +37,8 @@
     "JET-A1": "oil",
 }
 
-XM_DELAY = 2
+XM_DELAY_MIN = 2
+XM_DELAY_MAX = 5
 
 
 @refetch_frequency(timedelta(days=1))
@@ -47,7 +48,6 @@ def fetch_consumption(
     target_datetime: Optional[datetime] = None,
     logger: Logger = getLogger(__name__),
 ) -> List[Dict[str, Any]]:
-
     session = session or Session()
 
     if target_datetime is None:
@@ -59,7 +59,6 @@ def fetch_consumption(
 def fetch_live_consumption(
     zone_key: ZoneKey, session: Session, logger: Logger
 ) -> List[Dict[str, Any]]:
-
     response: Response = session.get(colombia_demand_URL, verify=False)
 
     if not response.ok:
@@ -104,7 +103,6 @@ def fetch_historical_consumption(
     if not df_consumption.empty:
         hour_columns = [col for col in df_consumption.columns if "Hour" in col]
         for hour_col in hour_columns:
-
             target_datetime_in_tz = target_arrow_in_tz.datetime.replace(
                 hour=int(hour_col[-2:]) - 1
             )
@@ -131,27 +129,53 @@ def fetch_production(
     target_datetime: Optional[datetime] = None,
     logger: Logger = getLogger(__name__),
 ) -> List[Dict[str, Any]]:
+    objetoAPI = pydataxm.ReadDB()
+
+    df_recursos = None
+    df_generation = None
+    target_arrow_in_tz = arrow.now()
+
     if target_datetime is None:
-        target_arrow_in_tz = arrow.now().floor("day").to(TZ).shift(days=-XM_DELAY)
+        target_arrow_in_tz = arrow.now().floor("day").to(TZ).shift(days=-XM_DELAY_MIN)
+        # Allow retries for most recent data
+        for xm_delay in range(XM_DELAY_MIN, XM_DELAY_MAX + 1):
+            target_arrow_in_tz = arrow.now().floor("day").to(TZ).shift(days=-xm_delay)
+
+            # API request list of power plants with ID (column 1) and type (column 7)
+            df_recursos = objetoAPI.request_data(
+                "ListadoRecursos",
+                "Sistema",
+                target_arrow_in_tz.date(),
+                target_arrow_in_tz.date(),
+            )
+            df_generation = objetoAPI.request_data(
+                "Gene", "Recurso", target_arrow_in_tz.date(), target_arrow_in_tz.date()
+            )
+
+            if not df_generation.empty and not df_recursos.empty:
+                break
     else:
         target_arrow_in_tz = arrow.get(target_datetime).to(TZ)
 
-    objetoAPI = pydataxm.ReadDB()
-
-    # API request list of power plants with ID (column 1) and type (column 7)
-    df_recursos = objetoAPI.request_data(
-        "ListadoRecursos",
-        "Sistema",
-        target_arrow_in_tz.date(),
-        target_arrow_in_tz.date(),
-    )
+        # API request list of power plants with ID (column 1) and type (column 7)
+        df_recursos = objetoAPI.request_data(
+            "ListadoRecursos",
+            "Sistema",
+            target_arrow_in_tz.date(),
+            target_arrow_in_tz.date(),
+        )
 
-    # API request generation per power plant
-    df_generation = objetoAPI.request_data(
-        "Gene", "Recurso", target_arrow_in_tz.date(), target_arrow_in_tz.date()
-    )
+        # API request generation per power plant
+        df_generation = objetoAPI.request_data(
+            "Gene", "Recurso", target_arrow_in_tz.date(), target_arrow_in_tz.date()
+        )
 
-    if not df_generation.empty and not df_recursos.empty:
+    if (
+        df_generation is not None
+        and not df_generation.empty
+        and df_recursos is not None
+        and not df_recursos.empty
+    ):
         df_units = (
             df_recursos[["Values_Code", "Values_EnerSource"]]
             .copy()
@@ -209,22 +233,39 @@ def fetch_price(
     target_datetime: Optional[datetime] = None,
     logger: Logger = getLogger(__name__),
 ) -> List[Dict[str, Any]]:
-
     session = session or Session()
 
+    objetoAPI = pydataxm.ReadDB()
+
+    df_price = None
+    target_arrow_in_tz = arrow.now()
+
     if target_datetime is None:
-        target_arrow_in_tz = arrow.now().floor("day").to(TZ).shift(days=-XM_DELAY)
+        # Allow retries for most recent data
+        for xm_delay in range(XM_DELAY_MIN, XM_DELAY_MAX + 1):
+            target_arrow_in_tz = arrow.now().floor("day").to(TZ).shift(days=-xm_delay)
+
+            df_price = objetoAPI.request_data(
+                "PrecBolsNaci",
+                "Sistema",
+                target_arrow_in_tz.date(),
+                target_arrow_in_tz.date(),
+            )
+
+            if not df_price.empty:
+                break
     else:
         target_arrow_in_tz = arrow.get(target_datetime).to(TZ)
+        # API request consumption
+        df_price = objetoAPI.request_data(
+            "PrecBolsNaci",
+            "Sistema",
+            target_arrow_in_tz.date(),
+            target_arrow_in_tz.date(),
+        )
 
-    objetoAPI = pydataxm.ReadDB()
-
-    # API request consumption
-    df_price = objetoAPI.request_data(
-        "PrecBolsNaci", "Sistema", target_arrow_in_tz.date(), target_arrow_in_tz.date()
-    )
     price_list = PriceList(logger)
-    if not df_price.empty:
+    if df_price is not None and not df_price.empty:
         hour_columns = [col for col in df_price.columns if "Hour" in col]
         for col in hour_columns:
             target_datetime_in_tz = target_arrow_in_tz.datetime.replace(


Test patch: