Repository: cmu-delphi/covidcast-indicators
Issue: chng failing with `assert len(filepaths_to_download) <= 6, "more files dropped than expected" AssertionError: more files dropped than expected`

**Actual Behavior:**

<!--Provide a description of the problem and a minimal reproducible example, if relevant. Please include the source and signal names, as well as sample observations, with geo region name, date, and data, demonstrating the problem.-->

source: `chng`

Per @jingjtang's investigation:

>It seems that there are no counts delivered from 2021-12-23 to 2021-12-26.

Fails with:

Slack ref: https://delphi-org.slack.com/archives/C01LZ3A2UMU/p1640748872124100

<details>
  <summary>:exclamation: Indicator exception</summary>
  
  ```
  Logger: delphi_changehc.run
Timestamp: 2021-12-29T03:34:02.771Z
Exception: Traceback (most recent call last):
 File "/home/indicators/.pyenv/versions/3.8.2/lib/python3.8/runpy.py", line 193, in _run_module_as_main
   return _run_code(code, main_globals, None,
 File "/home/indicators/.pyenv/versions/3.8.2/lib/python3.8/runpy.py", line 86, in _run_code
   exec(code, run_globals)
 File "/mnt/data2/indicators/runtime/changehc/env/lib/python3.8/site-packages/delphi_utils/runner.py", line 64, in <module>
   run_indicator_pipeline(indicator_module.run.run_module,
 File "/mnt/data2/indicators/runtime/changehc/env/lib/python3.8/site-packages/delphi_utils/runner.py", line 43, in run_indicator_pipeline
   indicator_fn(params)
 File "/mnt/data2/indicators/runtime/changehc/delphi_changehc/run.py", line 129, in run_module
   file_dict = retrieve_files(params, filedate, logger)
 File "/mnt/data2/indicators/runtime/changehc/delphi_changehc/run.py", line 29, in retrieve_files
   download_counts(filedate, params["indicator"]["input_cache_dir"], params["indicator"]["ftp_conn"])
 File "/mnt/data2/indicators/runtime/changehc/delphi_changehc/download_ftp_files.py", line 64, in download_counts
   get_files_from_dir(sftp, filedate, out_path)
 File "/mnt/data2/indicators/runtime/changehc/delphi_changehc/download_ftp_files.py", line 36, in get_files_from_dir
   assert len(filepaths_to_download) <= 6, "more files dropped than expected"
AssertionError: more files dropped than expected
```
</details>

**Expected behavior**

<!--A clear and concise description of what you expected to happen.-->

`chng` should not fail.

**Context**

<!--Add any context about the problem here.-->

This looks like provider issue. As this is happening during the Winter Holiday period we aren't meeting with them for a week or two.

@jingjtang has emailed CHNG 2022-01-03, waiting for CHNG to add those files to the SFTP server

Changed files:
changehc/tests/test_download_ftp_files.py, changehc/delphi_changehc/download_ftp_files.py, changehc/delphi_changehc/constants.py

diff --git a/changehc/delphi_changehc/constants.py b/changehc/delphi_changehc/constants.py
index 38ee1e2c..c934c556 100644
--- a/changehc/delphi_changehc/constants.py
+++ b/changehc/delphi_changehc/constants.py
@@ -9,3 +9,5 @@ SIGNALS = [SMOOTHED, SMOOTHED_ADJ, SMOOTHED_CLI, SMOOTHED_ADJ_CLI, SMOOTHED_FLU,
 NA = "NA"
 HRR = "hrr"
 FIPS = "fips"
+
+EXPECTED_FILES_PER_DROP = 7
diff --git a/changehc/delphi_changehc/download_ftp_files.py b/changehc/delphi_changehc/download_ftp_files.py
index 8a1cb1d6..f85ef994 100644
--- a/changehc/delphi_changehc/download_ftp_files.py
+++ b/changehc/delphi_changehc/download_ftp_files.py
@@ -8,6 +8,7 @@ from os import path
 # third party
 import paramiko
 
+from .constants import EXPECTED_FILES_PER_DROP
 
 def print_callback(filename, bytes_so_far, bytes_total):
     """Log file transfer progress."""
@@ -32,8 +33,8 @@ def get_files_from_dir(sftp, filedate, out_path):
                 not path.exists(path.join(out_path, filename)):
             filepaths_to_download[filename] = path.join(out_path, filename)
 
-    # make sure we don't download more than 6 files per day
-    assert len(filepaths_to_download) <= 6, "more files dropped than expected"
+    # make sure we don't download too many files per day
+    assert len(filepaths_to_download) <= EXPECTED_FILES_PER_DROP, "more files dropped than expected"
 
     # download!
     for infile, outfile in filepaths_to_download.items():
diff --git a/changehc/tests/test_download_ftp_files.py b/changehc/tests/test_download_ftp_files.py
index 8e8dd45c..0338e2cd 100644
--- a/changehc/tests/test_download_ftp_files.py
+++ b/changehc/tests/test_download_ftp_files.py
@@ -6,6 +6,7 @@ from datetime import timedelta
 
 # first party
 from delphi_changehc.download_ftp_files import *
+from delphi_changehc.constants import EXPECTED_FILES_PER_DROP
 
 class TestDownloadFTPFiles:
 
@@ -51,22 +52,18 @@ class TestDownloadFTPFiles:
         get_files_from_dir(one_new_one_old, "00005566", "")
         assert one_new_one_old.num_gets == 1
 
-        # When seven new files are present, AssertionError
-        new_file1 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo1")
-        new_file2 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo2")
-        new_file3 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo3")
-        new_file4 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo4")
-        new_file5 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo5")
-        new_file6 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo6")
-        new_file7 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo7")
-        seven_new = self.MockSFTP([new_file1, new_file2, new_file3, new_file4,
-                                    new_file5, new_file6, new_file7])
+        # When too many new files are present, AssertionError
+        file_batch = [
+            self.FileAttr(dt.timestamp(dt.now()), f"00001122_foo{i}")
+            for i in range(EXPECTED_FILES_PER_DROP + 1)
+        ]
+        too_many_new = self.MockSFTP(file_batch)
         with pytest.raises(AssertionError):
-            get_files_from_dir(seven_new, "00001122", "")
+            get_files_from_dir(too_many_new, "00001122", "")
 
         # When the file already exists, no files are downloaded
         mock_path.exists.return_value = True
-        one_exists = self.MockSFTP([new_file1])
+        one_exists = self.MockSFTP([file_batch[0]])
         get_files_from_dir(one_new, "00001122", "")
         assert one_exists.num_gets == 0