Repository: cmu-delphi/covidcast-indicators
Issue: DSEW-CPR hospital admissions submits megacounties with NULL values

DSEW-CPR shouldn't be submitting null values, however, it has apparently been doing so since its initial release. There are however some patterns. 

The "53" below accounts for 52 megacounties and 1 MSA in Puerto Rico. My understanding is that DSEW-CPR should typically cover all counties, so that megacounties are not used.

We should prevent this indicator from publishing megacounties, if possible, or drop NULL-valued entries before exporting to CSV, if not.

```
MariaDB [epidata]> select issue,count(1) from covidcast where source="dsew-cpr" and `signal`="confirmed_admissions_covid_1d_7dav" and time_type="day" and value is NULL group by issue;
+----------+----------+
| issue    | count(1) |
+----------+----------+
| 20220202 |   215027 |
| 20220203 |      795 |
| 20220204 |      795 |
| 20220207 |      795 |
| 20220208 |      795 |
| 20220209 |      795 |
| 20220210 |      795 |
| 20220211 |      795 |
| 20220214 |      795 |
| 20220215 |      795 |
| 20220216 |      795 |
| 20220217 |      795 |
| 20220218 |      795 |
| 20220222 |      795 |
| 20220223 |      795 |
| 20220303 |      795 |
| 20220304 |       53 |
| 20220307 |       53 |
| 20220308 |       53 |
| 20220309 |       53 |
| 20220310 |       53 |
| 20220311 |       53 |
| 20220314 |       53 |
| 20220315 |       53 |
| 20220316 |       53 |
| 20220317 |       53 |
| 20220318 |       53 |
| 20220321 |       53 |
| 20220322 |       53 |
| 20220323 |       53 |
| 20220324 |       53 |
| 20220325 |       53 |
| 20220328 |       53 |
| 20220329 |       53 |
| 20220330 |       53 |
| 20220331 |       53 |
| 20220401 |       53 |
| 20220404 |       53 |
| 20220405 |       53 |
| 20220406 |       53 |
| 20220407 |       53 |
| 20220408 |       53 |
| 20220411 |       53 |
| 20220412 |       53 |
| 20220413 |       53 |
| 20220414 |       53 |
| 20220415 |       53 |
| 20220418 |       53 |
| 20220419 |   528145 |
| 20220420 |       53 |
+----------+----------+
50 rows in set (5.84 sec)

MariaDB [epidata]> select time_value,geo_type,geo_value,value,sample_size,stderr,missing_value,missing_sample_size,missing_stderr from covidcast where source="dsew-cpr" and `signal`="confirmed_admissions_covid_1d_7dav" and time_type="day" and value is NULL and issue=20220304;
+------------+----------+-----------+-------+-------------+--------+---------------+---------------------+----------------+
| time_value | geo_type | geo_value | value | sample_size | stderr | missing_value | missing_sample_size | missing_stderr |
+------------+----------+-----------+-------+-------------+--------+---------------+---------------------+----------------+
|   20220301 | county   | 01000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 02000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 04000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 05000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 06000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 08000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 09000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 10000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 11000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 12000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 13000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 15000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 16000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 17000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 18000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 19000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 20000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 21000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 22000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 23000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 24000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 25000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 26000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 27000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 28000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 29000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 30000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 31000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 32000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 33000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 34000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 35000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 36000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 37000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 38000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 39000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 40000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 41000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 42000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 44000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 45000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 46000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 47000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 48000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 49000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 50000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 51000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 53000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 54000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 55000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 56000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | county   | 72000     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
|   20220301 | msa      | 25020     |  NULL |        NULL |   NULL |             5 |                   5 |              5 |
+------------+----------+-----------+-------+-------------+--------+---------------+---------------------+----------------+
53 rows in set (0.68 sec)
```



Changed files:
dsew_community_profile/tests/test_pull.py, dsew_community_profile/delphi_dsew_community_profile/pull.py

diff --git a/dsew_community_profile/delphi_dsew_community_profile/pull.py b/dsew_community_profile/delphi_dsew_community_profile/pull.py
index 6f3ed66c..b7c27a05 100644
--- a/dsew_community_profile/delphi_dsew_community_profile/pull.py
+++ b/dsew_community_profile/delphi_dsew_community_profile/pull.py
@@ -628,14 +628,14 @@ def interpolate_missing_values(dfs: DataDict) -> DataDict:
             if "val" in reindexed_group_df.columns and not reindexed_group_df["val"].isna().all():
                 reindexed_group_df["val"] = (
                     reindexed_group_df["val"]
-                    .interpolate(method="linear", limit_area="inside")
                     .astype(float)
+                    .interpolate(method="linear", limit_area="inside")
                 )
             if "se" in reindexed_group_df.columns:
                 reindexed_group_df["se"] = (
                     reindexed_group_df["se"]
-                    .interpolate(method="linear", limit_area="inside")
                     .astype(float)
+                    .interpolate(method="linear", limit_area="inside")
                 )
             if (
                 "sample_size" in reindexed_group_df.columns
@@ -643,13 +643,14 @@ def interpolate_missing_values(dfs: DataDict) -> DataDict:
             ):
                 reindexed_group_df["sample_size"] = (
                     reindexed_group_df["sample_size"]
-                    .interpolate(method="linear", limit_area="inside")
                     .astype(float)
+                    .interpolate(method="linear", limit_area="inside")
                 )
             if "publish_date" in reindexed_group_df.columns:
                 reindexed_group_df["publish_date"] = reindexed_group_df["publish_date"].fillna(
                     method="bfill"
                 )
+            reindexed_group_df = reindexed_group_df[~reindexed_group_df.val.isna()]
             geo_dfs.append(reindexed_group_df)
         interpolate_df[key] = (
             pd.concat(geo_dfs).reset_index().rename(columns={"index": "timestamp"})
diff --git a/dsew_community_profile/tests/test_pull.py b/dsew_community_profile/tests/test_pull.py
index 3a9fc84a..8f2c1790 100644
--- a/dsew_community_profile/tests/test_pull.py
+++ b/dsew_community_profile/tests/test_pull.py
@@ -4,7 +4,7 @@ from datetime import date, datetime, timedelta
 from itertools import chain
 from typing import Any, Dict, List, Union
 import pandas as pd
-from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
 import numpy as np
 import pytest
 from unittest.mock import patch, Mock
@@ -506,7 +506,7 @@ class TestPull:
             "sample_size": [line(i) for i in range(0, 10)],
             "publish_date": pd.to_datetime("2022-01-10")
         }), dtypes=DTYPES)
-        # A signal missing everything, should be left alone.
+        # A signal missing everything, should be dropped since it's all NAs.
         missing_sig3 = sig3[(sig3.timestamp <= "2022-01-05") | (sig3.timestamp >= "2022-01-08")]
 
         sig4 = _set_df_dtypes(pd.DataFrame({
@@ -517,12 +517,33 @@ class TestPull:
             "sample_size": [line(i) for i in range(0, 10)],
             "publish_date": pd.to_datetime("2022-01-10")
         }), dtypes=DTYPES)
-        # A signal missing everything except for one point, should be left alone.
+        # A signal missing everything except for one point, should output a reduced range without NAs.
         missing_sig4 = sig4[(sig4.timestamp <= "2022-01-05") | (sig4.timestamp >= "2022-01-08")]
 
         missing_dfs = [missing_sig1, missing_sig2, missing_sig3, missing_sig4]
         interpolated_dfs1 = interpolate_missing_values({("src", "sig", False): pd.concat(missing_dfs)})
-        expected_dfs = pd.concat([sig1, sig2, sig3, sig4])
+        expected_dfs = pd.concat([sig1, sig2, sig4.loc[9:]])
+        _assert_frame_equal(interpolated_dfs1[("src", "sig", False)], expected_dfs, index_cols=["geo_id", "timestamp"])
+
+    def test_interpolation_object_type(self):
+        DTYPES = {"geo_id": str, "timestamp": "datetime64[ns]", "val": float, "se": float, "sample_size": float, "publish_date": "datetime64[ns]"}
+        line = lambda x: 3 * x + 5
+
+        sig1 = _set_df_dtypes(pd.DataFrame({
+            "geo_id": "1",
+            "timestamp": pd.date_range("2022-01-01", "2022-01-10"),
+            "val": [line(i) for i in range(2, 12)],
+            "se": [line(i) for i in range(1, 11)],
+            "sample_size": [line(i) for i in range(0, 10)],
+            "publish_date": pd.to_datetime("2022-01-10")
+        }), dtypes=DTYPES)
+        # A linear signal missing two days which should be filled exactly by the linear interpolation.
+        missing_sig1 = sig1[(sig1.timestamp <= "2022-01-05") | (sig1.timestamp >= "2022-01-08")]
+        # set all columns to object type to simulate the miscast we sometimes see when combining dfs
+        missing_sig1 = _set_df_dtypes(missing_sig1, {key: object for key in DTYPES.keys()})
+
+        interpolated_dfs1 = interpolate_missing_values({("src", "sig", False): missing_sig1})
+        expected_dfs = pd.concat([sig1])
         _assert_frame_equal(interpolated_dfs1[("src", "sig", False)], expected_dfs, index_cols=["geo_id", "timestamp"])
 
     @patch("delphi_dsew_community_profile.pull.INTERP_LENGTH", 2)