Repository: cmu-delphi/covidcast-indicators
Issue: GHT indicator is failing on 8/20

GHT has not been able to deliver an update to the API since the 8/20 issue because of the error below. What's causing it? Can we repair it? If not, can we modify the pipeline to permit gaps like this?


Traceback:

```
Traceback (most recent call last):
  File "/home/indicators/.pyenv/versions/3.8.2/lib/python3.8/runpy.py", line 193, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/home/indicators/.pyenv/versions/3.8.2/lib/python3.8/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/home/indicators/runtime/google_health/delphi_google_health/__main__.py", line 11, in <module>
    run_module()  # pragma: no cover
  File "/home/indicators/runtime/google_health/delphi_google_health/run.py", line 59, in run_module
    df_hrr, df_msa = derived_counts_from_dma(df_dma, static_dir=static_dir)
  File "/home/indicators/runtime/google_health/delphi_google_health/map_values.py", line 30, in derived_counts_from_dma
    dma_mat, day_list = _dma_df_to_matrix(df_dma, static_dir)
  File "/home/indicators/runtime/google_health/delphi_google_health/map_values.py", line 97, in _dma_df_to_matrix
    raise ValueError(f"Missing values for day={day} and dma={dma}")
ValueError: Missing values for day=2020-08-20 and dma=534
```

Changed files:
google_health/delphi_google_health/pull_api.py, google_health/setup.py, google_health/delphi_google_health/run.py

diff --git a/google_health/delphi_google_health/pull_api.py b/google_health/delphi_google_health/pull_api.py
index 92877062..faa3734d 100644
--- a/google_health/delphi_google_health/pull_api.py
+++ b/google_health/delphi_google_health/pull_api.py
@@ -11,6 +11,7 @@ import numpy as np
 import pandas as pd
 import googleapiclient
 import googleapiclient.discovery
+from tenacity import retry, wait_fixed, stop_after_attempt
 
 DISCOVERY_URL = "https://www.googleapis.com/discovery/v1/apis/trends/v1beta/rest"
 TERMS_IDS = "anosmia_ms"
@@ -80,7 +81,7 @@ class GoogleHealthTrends:
 
         return data
 
-
+@retry(wait=wait_fixed(60), stop = stop_after_attempt(5))
 def get_counts_states(
     ght: GoogleHealthTrends,
     start_date: str,
@@ -127,7 +128,7 @@ def get_counts_states(
 
     return state_df
 
-
+@retry(wait=wait_fixed(60), stop = stop_after_attempt(5))
 def get_counts_dma(
     ght: GoogleHealthTrends,
     start_date: str,
@@ -211,23 +212,22 @@ def _get_counts_geoid(
     output_dates = set(pd.date_range(start_date, end_date).to_native_types())
     cache_dates = set(dt["timestamp"].values)
     req_dates = list(output_dates - cache_dates)
-
     try:
         if req_dates:
             sdate = min(req_dates)
             edate = max(req_dates)
             new_data = _api_data_to_df(
-                ght.query(start_date=sdate, end_date=edate, geo_id=geo_id, dma=dma),
-                geo_id=geo_id,
-            )
+                    ght.query(start_date=sdate, end_date=edate, geo_id=geo_id, dma=dma),
+                    geo_id=geo_id,
+                )
             new_data = new_data[new_data["timestamp"].isin(req_dates)]
             dt = dt.append(new_data).sort_values("timestamp")
             dt = dt.drop_duplicates(subset="timestamp")
             _write_cached_file(dt, geo_id, data_dir)
             dt = _load_cached_file(geo_id, data_dir)
     except googleapiclient.errors.HttpError:
-        #  This is thrown in there is no data yet for the given days. Need to
-        #  investigate this further.
+        #  This is thrown in there is no data yet for the given days. Need to
+        #  investigate this further.
         pass
 
     dt = dt[dt["timestamp"].isin(output_dates)]
diff --git a/google_health/delphi_google_health/run.py b/google_health/delphi_google_health/run.py
index e39b8a97..051ac6bf 100644
--- a/google_health/delphi_google_health/run.py
+++ b/google_health/delphi_google_health/run.py
@@ -107,7 +107,7 @@ def run_module():
         print(f"Failed to archive '{exported_file}'")
 
 
-def add_prefix(signal_names, wip_signal, prefix="wip_"):
+def add_prefix(signal_names, wip_signal, prefix):
     """Adds prefix to signal if there is a WIP signal
     Parameters
     ----------
diff --git a/google_health/setup.py b/google_health/setup.py
index 90f433d0..49ae2965 100644
--- a/google_health/setup.py
+++ b/google_health/setup.py
@@ -11,7 +11,8 @@ required = [
     "delphi-utils",
     "covidcast",
     "boto3",
-    "moto"
+    "moto",
+    "tenacity"
 ]
 
 setup(