Repository: cmu-delphi/covidcast-indicators
Issue: CHNG randomly losing track of data?

A weird thing:

* CHNG generates an empty file for day X, issue Y. This file fails ingestion.
* There is a successful file for day X, issue Z<Y, which has data.
* The raw data file processed for issue Y has strictly more counts than the raw data file processed for issue Z.

Example: There's a failed file for 20210226 from issue 20210503, but a successful one from issue 20210501. The failed file contains only the header, but the earlier successful file contains records for all states.
```
[krivard@delphi-master-prod-01 covidcast]$ find archive -name "20210226_state_smoothed_adj_outpatient_covid.*" -exec ls -l {} +
-rw-r--r--. 1 indicators indicators  26 May  3 13:16 archive/failed/chng/20210226_state_smoothed_adj_outpatient_covid.csv
-rw-rw-r--. 1 automation automation 524 May  1 23:00 archive/successful/chng/20210226_state_smoothed_adj_outpatient_covid.csv.gz
[krivard@delphi-master-prod-01 covidcast]$ wc -l archive/failed/chng/20210226_state_smoothed_adj_outpatient_covid.csv
1 archive/failed/chng/20210226_state_smoothed_adj_outpatient_covid.csv
[krivard@delphi-master-prod-01 covidcast]$ zcat archive/successful/chng/20210226_state_smoothed_adj_outpatient_covid.csv.gz |wc -l
55
```

The failed file was generated from the data drops labeled 20210502; the successful one from data drops labeled 20210430. Here's the diff:

```
[indicators@delphi-master-prod-01 changehc]$ diff <(zcat cache/20210430_Counts_Products_Covid.dat.gz |grep 20210226) <(zcat cache/20210502_Counts_Products_Covid.dat.gz |grep 20210226)
176c176
< 20210226,09003,87
---
> 20210226,09003,88
218c218
< 20210226,12086,337
---
> 20210226,12086,338
773a774
> 20210226,27007,3 or less
786c787
< 20210226,27053,50
---
> 20210226,27053,51
1001c1002
< 20210226,36029,63
---
> 20210226,36029,64
1003c1004
< 20210226,36033,13
---
> 20210226,36033,14
1132c1133
< 20210226,39035,163
---
> 20210226,39035,164
1163c1164
< 20210226,39103,24
---
> 20210226,39103,25
```

In no case does the new 20210502 file remove any counts for reference date 20210226, yet, an empty file was generated.

Changed files:
changehc/delphi_changehc/update_sensor.py, changehc/tests/test_update_sensor.py, changehc/delphi_changehc/config.py

diff --git a/changehc/delphi_changehc/config.py b/changehc/delphi_changehc/config.py
index 95d6878a..a7232ddf 100644
--- a/changehc/delphi_changehc/config.py
+++ b/changehc/delphi_changehc/config.py
@@ -19,7 +19,7 @@ class Config:
     BURN_IN_PERIOD = timedelta(days=1)
 
     # shift dates forward for labeling purposes
-    DAY_SHIFT = timedelta(days=1)
+    DAY_SHIFT = timedelta(days=0)
 
     ## data columns
     COVID_COL = "COVID"
diff --git a/changehc/delphi_changehc/update_sensor.py b/changehc/delphi_changehc/update_sensor.py
index 5d9bd00b..6707665b 100644
--- a/changehc/delphi_changehc/update_sensor.py
+++ b/changehc/delphi_changehc/update_sensor.py
@@ -156,11 +156,13 @@ class CHCSensorUpdater:  # pylint: disable=too-many-instance-attributes
                                                  Config.MIN_DEN,
                                                  Config.MAX_BACKFILL_WINDOW,
                                                  thr_col="den",
-                                                 mega_col=geo)
+                                                 mega_col=geo,
+                                                 date_col=Config.DATE_COL)
         elif geo == "state":
-            data_frame = gmpr.replace_geocode(data, "fips", "state_id", new_col="state")
+            data_frame = gmpr.replace_geocode(data, "fips", "state_id", new_col="state",
+                                              date_col=Config.DATE_COL)
         else:
-            data_frame = gmpr.replace_geocode(data, "fips", geo)
+            data_frame = gmpr.replace_geocode(data, "fips", geo, date_col=Config.DATE_COL)
 
         unique_geo_ids = pd.unique(data_frame[geo])
         data_frame.set_index([geo, Config.DATE_COL],inplace=True)
diff --git a/changehc/tests/test_update_sensor.py b/changehc/tests/test_update_sensor.py
index 29cccc81..318437b9 100644
--- a/changehc/tests/test_update_sensor.py
+++ b/changehc/tests/test_update_sensor.py
@@ -132,6 +132,47 @@ class TestCHCSensorUpdater:
         assert outputs["20200319_hhs_smoothed_outpatient_covid.csv"].empty
         assert outputs["20200319_nation_smoothed_outpatient_covid.csv"].empty
 
+    def test_update_sensor_output_daterange(self):
+        """Tests that output does not change when data range changes"""
+        small_test_data = pd.DataFrame({
+            "num": [0, 100, 200, 300, 400, 500, 600, 100, 200, 300, 400, 500, 600] * 2,
+            "fips": ["01001"] * 13 + ["42003"] * 13,
+            "den": [30, 50, 50, 10, 1, 5, 5, 50, 50, 50, 0, 0, 0] * 2,
+            "timestamp": list(pd.date_range("20200301", "20200313")) * 2
+        }).set_index(["fips", "timestamp"])
+        startdates = ["2020-03-01", "2020-03-05"]
+        outputs = {s:{} for s in startdates}
+        for startdate in startdates:
+            for geo in ["county", "state", "hhs", "nation"]:
+                td = TemporaryDirectory()
+                su_inst = CHCSensorUpdater(
+                    startdate,
+                    "03-22-2020",
+                    "03-27-2020",
+                    geo,
+                    self.parallel,
+                    self.weekday,
+                    self.numtype,
+                    self.se,
+                    "",
+                    TEST_LOGGER
+                )
+                su_inst.update_sensor(small_test_data.copy(), td.name)
+                for f in os.listdir(td.name):
+                    outputs[startdate][f] = pd.read_csv(os.path.join(td.name, f))
+            assert len(os.listdir(td.name)) == len(su_inst.sensor_dates),\\
+                f"failed {geo} update sensor test"
+            td.cleanup()
+
+        def pretty(key):
+            return "\\n".join(f"{s}[{key}]: {len(outputs[s][key])}" for s in startdates)
+        for f in outputs[startdates[-1]]:
+            assert len(outputs[startdates[0]][f]) == len(outputs[startdates[1]][f]), \\
+                f"\\n{pretty(f)}"
+            assert np.array_equal(
+                outputs[startdates[0]][f].val.values,
+                outputs[startdates[1]][f].val.values
+            ), f
 
 class TestWriteToCsv:
     """Tests for writing output files to CSV."""
@@ -141,7 +182,7 @@ class TestWriteToCsv:
             "val": [0.1, 0.5, 1.5] + [1, 2, 3],
             "se": [0.1, 1, 1.1] + [0.5, np.nan, 0.5],
             "sample_size": [np.nan] * 6,
-            "timestamp": pd.to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"] * 2),
+            "timestamp": pd.to_datetime(["2020-05-02", "2020-05-03", "2020-05-05"] * 2),
             "include": [True, True, True] + [True, False, True],
             "geo_id": ["a"] * 3 + ["b"] * 3,
         })
@@ -197,7 +238,7 @@ class TestWriteToCsv:
             "val": [0.1, 0.5, 1.5] + [1, 2, 3],
             "se": [0.1, 1, 1.1] + [0.5, np.nan, 0.5],
             "sample_size": [np.nan] * 6,
-            "timestamp": pd.to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"] * 2),
+            "timestamp": pd.to_datetime(["2020-05-02", "2020-05-03", "2020-05-05"] * 2),
             "include": [True, True, True] + [True, False, True],
             "geo_id": ["a"] * 3 + ["b"] * 3,
         })
@@ -231,7 +272,7 @@ class TestWriteToCsv:
             "val": [0.1, 0.5, 1.5] + [1, 2, 3],
             "se": [0.1, 1, 1.1] + [0.5, 0.5, 0.5],
             "sample_size": [np.nan] * 6,
-            "timestamp": pd.to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"] * 2),
+            "timestamp": pd.to_datetime(["2020-05-02", "2020-05-03", "2020-05-05"] * 2),
             "include": [True, True, True] + [True, False, True],
             "geo_id": ["a"] * 3 + ["b"] * 3,
         }).set_index(["timestamp", "geo_id"]).sort_index()
@@ -241,7 +282,7 @@ class TestWriteToCsv:
         # nan value for included loc-date
         res1 = res0.copy()
         res1 = res1[res1['include']]
-        res1.loc[("2020-05-01", "a"), "val"] = np.nan
+        res1.loc[("2020-05-02", "a"), "val"] = np.nan
         res1.reset_index(inplace=True)
         with pytest.raises(AssertionError):
             write_to_csv(
@@ -257,7 +298,7 @@ class TestWriteToCsv:
         # nan se for included loc-date
         res2 = res0.copy()
         res2 = res2[res2['include']]
-        res2.loc[("2020-05-01", "a"), "se"] = np.nan
+        res2.loc[("2020-05-02", "a"), "se"] = np.nan
         res2.reset_index(inplace=True)
         with pytest.raises(AssertionError):
             write_to_csv(
@@ -273,7 +314,7 @@ class TestWriteToCsv:
         # large se value
         res3 = res0.copy()
         res3 = res3[res3['include']]
-        res3.loc[("2020-05-01", "a"), "se"] = 10
+        res3.loc[("2020-05-02", "a"), "se"] = 10
         res3.reset_index(inplace=True)
         with pytest.raises(AssertionError):
             write_to_csv(