Repository: cmu-delphi/covidcast-indicators
Issue: Combo cases/deaths should not publish dates where only JHU is available

Because combo cases/deaths re-aggregates HHS and nation geo levels, any dates where only JHU is available wind up with only puerto rico counting toward HHS=2 and nation=us, which is more wrong than not publishing anything for those dates.

Adjust the null data frame checker to generate the following behavior:

| USAFacts | JHU | result |
|-----------|-----|-----------|
|  data | data | usafacts + jhu |
| data | null | usafacts |
| null | data | ~jhu~ **null** |
| null | null | null |

Changed files:
combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py, combo_cases_and_deaths/tests/test_run.py

diff --git a/combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py b/combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py
index fedfe250..b89de2c4 100755
--- a/combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py
+++ b/combo_cases_and_deaths/delphi_combo_cases_and_deaths/run.py
@@ -28,6 +28,8 @@ COLUMN_MAPPING = {"time_value": "timestamp",
                   "stderr": "se",
                   "sample_size": "sample_size"}
 
+EMPTY_FRAME = pd.DataFrame({}, columns=COLUMN_MAPPING.values())
+
 covidcast.covidcast._ASYNC_CALL = True  # pylint: disable=protected-access
 
 
@@ -39,18 +41,21 @@ def check_none_data_frame(data_frame, label, date_range):
     return False
 
 
-def maybe_append(df1, df2):
+def maybe_append(usa_facts, jhu):
     """
-    Append dataframes if available, otherwise return non-None one.
+    Append dataframes if available, otherwise return USAFacts.
+
+    If both data frames are available, append them and return.
 
-    If both data frames are available, append them and return. Otherwise, return
-    whichever frame is not None.
+    If only USAFacts is available, return it.
+
+    If USAFacts is not available, return None.
     """
-    if df1 is None:
-        return df2
-    if df2 is None:
-        return df1
-    return df1.append(df2)
+    if usa_facts is None:
+        return None
+    if jhu is None:
+        return usa_facts
+    return usa_facts.append(jhu)
 
 
 def compute_special_geo_dfs(df, signal, geo):
@@ -128,13 +133,12 @@ def get_updated_dates(signal, geo, date_range, issue_range=None, fetcher=covidca
         issues=issue_range
     )
 
-    if check_none_data_frame(usafacts_df, "USA-FACTS", date_range) and \\
-       (geo not in ('state', 'county', 'msa') or
-        check_none_data_frame(jhu_df, "JHU", date_range)):
+    if check_none_data_frame(usafacts_df, "USA-FACTS", date_range):
         return None
 
     merged_df = merge_dfs_by_geos(usafacts_df, jhu_df, geo)
-    unique_dates = merged_df["timestamp"].unique()
+    timestamp_mask = merged_df["timestamp"]<=usafacts_df["timestamp"].max()
+    unique_dates = merged_df.loc[timestamp_mask]["timestamp"].unique()
     return unique_dates
 
 
@@ -154,7 +158,7 @@ def combine_usafacts_and_jhu(signal, geo, date_range, issue_range=None, fetcher=
 
     # This occurs if the usafacts ~and the jhu query were empty
     if unique_dates is None:
-        return pd.DataFrame({}, columns=COLUMN_MAPPING.values())
+        return EMPTY_FRAME
 
     # Query only the represented window so that every geo is represented; a single window call is
     # faster than a fetch for every date in unique_dates even in cases of 1:10 sparsity,
diff --git a/combo_cases_and_deaths/tests/test_run.py b/combo_cases_and_deaths/tests/test_run.py
index b0733f22..adcde30e 100644
--- a/combo_cases_and_deaths/tests/test_run.py
+++ b/combo_cases_and_deaths/tests/test_run.py
@@ -2,7 +2,7 @@
 from datetime import date
 from itertools import product
 import unittest
-from unittest.mock import patch
+from unittest.mock import patch, call
 import pandas as pd
 import numpy as np
 
@@ -46,8 +46,8 @@ def test_unstable_sources(mock_covidcast_signal):
     frame correctly for all cases where 0, 1, or both signals are
     available.
     """
-    def jhu(geo, c=[0]):
-        c[0] += 1
+    date_count = [1]
+    def jhu(geo, c=date_count):
         if geo == "state":
             geo_val = "pr"
         elif geo == "msa":
@@ -56,9 +56,8 @@ def test_unstable_sources(mock_covidcast_signal):
             geo_val = "72001"
         return pd.DataFrame(
             [(date.fromordinal(c[0]),geo_val,1,1,1)],
-            columns="time_value geo_value value stderr sample_size".split())
-    def uf(geo, c=[0]):
-        c[0] += 1
+            columns="timestamp geo_value value stderr sample_size".split())
+    def uf(geo, c=date_count):
         if geo == "state":
             geo_val = "ny"
         elif geo == "msa":
@@ -67,7 +66,7 @@ def test_unstable_sources(mock_covidcast_signal):
             geo_val = "36001"
         return pd.DataFrame(
             [(date.fromordinal(c[0]),geo_val,1,1,1)],
-            columns="time_value geo_value value stderr sample_size".split())
+            columns="timestamp geo_value value stderr sample_size".split())
     def make_mock(geo):
         # The first two in each row provide a unique_date array of the appropriate length for
         # query of the latter two (in combine_usafacts_and_jhu)
@@ -75,7 +74,7 @@ def test_unstable_sources(mock_covidcast_signal):
             # 1 0
             uf(geo), None, uf(geo), None,
             # 0 1
-            None, jhu(geo), None, jhu(geo),
+            None, jhu(geo),
             # 1 1
             uf(geo), jhu(geo), uf(geo), jhu(geo),
             # 0 0
@@ -88,28 +87,29 @@ def test_unstable_sources(mock_covidcast_signal):
 
     date_range = [date.today(), date.today()]
 
-    calls = -1
+    calls = 0
     for geo in geos:
-        for config, expected_size in [
-                ("1 0", 1),
-                ("0 1", 1),
-                ("1 1", 1 if geo in ["nation", "hhs"] else 2),
-                ("0 0", 0)
+        for config, call_size, expected_size in [
+                ("1 0", 4, 1),
+                ("0 1", 2, 0),
+                ("1 1", 4, 1 if geo in ["nation", "hhs"] else 2),
+                ("0 0", 2, 0)
         ]:
-            calls += 1
             df = combine_usafacts_and_jhu("", geo, date_range, fetcher=mock_covidcast_signal)
             assert df.size == expected_size * len(COLUMN_MAPPING), f"""
 Wrong number of rows in combined data frame for the number of available signals.
 
 input for {geo} {config}:
-{outputs[2*calls]}
-{outputs[2*calls + 1]}
+{outputs[calls]}
+{outputs[calls + 1]}
 
 output:
 {df}
 
 expected rows: {expected_size}
 """
+            calls += call_size
+            date_count[0] += 1
 
 @patch("covidcast.covidcast.signal")
 def test_multiple_issues(mock_covidcast_signal):
@@ -206,6 +206,44 @@ def test_combine_usafacts_and_jhu_special_geos(mock_covidcast_signal):
                      index=[0, 1, 0])
     )
 
+@patch("covidcast.covidcast.signal")
+def test_no_nation_jhu(mock_covidcast_signal):
+    """
+    If we get JHU data that extends farther into the future than USAFacts data, trim it off.
+    """
+    cvc_columns = "time_value geo_value value stderr sample_size".split()
+    mock_covidcast_signal.side_effect = [
+        pd.DataFrame({"geo_value": ["01000"],
+                      "value": [50],
+                      "timestamp": [20200101]},),
+        pd.DataFrame({"geo_value": ["72001", "72001"],
+                      "value": [1, 1],
+                      "timestamp": [20200101, 20200102]}),
+        pd.DataFrame({"geo_value": ["01000"],
+                      "value": [50],
+                      "timestamp": [20200101]},),
+        pd.DataFrame({"geo_value": ["72001"],
+                      "value": [1],
+                      "timestamp": [20200101]})
+    ]
+    result = combine_usafacts_and_jhu("_num", "nation", date_range=(0, 1), fetcher=mock_covidcast_signal)
+
+    assert mock_covidcast_signal.call_args_list[-1] == call(
+        "jhu-csse",
+        "_num",
+        20200101,
+        20200101,
+        "county"
+    )
+    pd.testing.assert_frame_equal(
+        result,
+        pd.DataFrame({"timestamp":[20200101],
+                      "geo_id":["us"],
+                      "val":[51],
+                      "se": [None],
+                      "sample_size": [None]},)
+    )
+
 
 if __name__ == '__main__':
     unittest.main()