Repository: reichlab/cladetime

Issue:
Missing date values
Clade counts have rows with a missing `date`:
```
> read_parquet("https://covid-clade-counts.s3.amazonaws.com/2024-10-14_covid_clade_counts.parquet") |>
+   filter(is.na(date))
# A tibble: 197 × 4
   location     date   clade count
   <chr>        <date> <chr> <int>
 1 South Dakota NA     23I       3
 2 Virginia     NA     21H      12
 3 Mississippi  NA     20I       8
 4 Virginia     NA     23H      12
 5 Louisiana    NA     22C       1
 6 Virginia     NA     23E       7
 7 Maryland     NA     22E       1
 8 South Dakota NA     20A     504
 9 Nebraska     NA     21C       2
10 Maryland     NA     21F       1
# ℹ 187 more rows
# ℹ Use `print(n = ...)` to see more rows
```

The above file was generated using [this script](https://github.com/reichlab/get-covid-clade-counts/blob/main/get_covid_clade_counts.py).

Do we have an understanding of how these missing dates arise?  Is this a bug we need to fix, or something about the source data?

Discussion:

Patch:
diff --git a/src/cladetime/util/sequence.py b/src/cladetime/util/sequence.py
index 1057073..a50d19d 100644
--- a/src/cladetime/util/sequence.py
+++ b/src/cladetime/util/sequence.py
@@ -147,7 +147,11 @@ def _get_ncov_metadata(
         )
         return {}
 
-    return response.json()
+    metadata = response.json()
+    if metadata.get("nextclade_dataset_name", "").lower() == "sars-cov-2":
+        metadata["nextclade_dataset_name_full"] = "nextstrain/sars-cov-2/wuhan-hu-1/orfs"
+
+    return metadata
 
 
 def filter_covid_genome_metadata(metadata: pl.LazyFrame, cols: list = []) -> pl.LazyFrame:
@@ -175,11 +179,15 @@ def filter_covid_genome_metadata(metadata: pl.LazyFrame, cols: list = []) -> pl.
         .filter(
             pl.col("country") == "USA",
             pl.col("division").is_in(states),
-            pl.col("date").is_not_null(),
             pl.col("host") == "Homo sapiens",
         )
         .rename({"clade_nextstrain": "clade", "division": "location"})
         .cast({"date": pl.Date}, strict=False)
+        # date filtering at the end ensures we filter out null
+        # values created by the above .cast operation
+        .filter(
+            pl.col("date").is_not_null(),
+        )
     )
 
     return filtered_metadata


Test patch:
diff --git a/tests/unit/util/test_sequence.py b/tests/unit/util/test_sequence.py
index b5f21b7..2532c7d 100644
--- a/tests/unit/util/test_sequence.py
+++ b/tests/unit/util/test_sequence.py
@@ -94,15 +94,23 @@ def test_download_covid_genome_metadata_no_history(s3_setup, tmp_path, mock_sess
 
 def test_filter_covid_genome_metadata():
     test_genome_metadata = {
-        "date": ["2022-01-01", "2022-01-02", "2022-01-03", "2023-12-25", None, "2023-12-27"],
-        "host": ["Homo sapiens", "Homo sapiens", "Homo sapiens", "Narwhals", "Homo sapiens", "Homo sapiens"],
-        "country": ["USA", "Argentina", "USA", "USA", "USA", "USA"],
-        "division": ["Alaska", "Maine", "Guam", "Puerto Rico", "Utah", "Pennsylvania"],
-        "clade_nextstrain": ["AAA", "BBB", "CCC", "DDD", "EEE", "FFF"],
-        "location": ["Vulcan", "Reisa", "Bajor", "Deep Space 9", "Earth", "Cardassia"],
-        "genbank_accession": ["A1", "A2", "B1", "B2", "C1", "C2"],
-        "genbank_accession_rev": ["A1.1", "A2.4", "B1.1", "B2.5", "C1.1", "C2.1"],
-        "unwanted_column": [1, 2, 3, 4, 5, 6],
+        "date": ["2022-01-01", "2022-01-02", "2022-01-03", "2023-12-25", None, "2023-12-27", "2023-05"],
+        "host": [
+            "Homo sapiens",
+            "Homo sapiens",
+            "Homo sapiens",
+            "Narwhals",
+            "Homo sapiens",
+            "Homo sapiens",
+            "Homo sapiens",
+        ],
+        "country": ["USA", "Argentina", "USA", "USA", "USA", "USA", "USA"],
+        "division": ["Alaska", "Maine", "Guam", "Puerto Rico", "Utah", "Pennsylvania", "Pennsylvania"],
+        "clade_nextstrain": ["AAA", "BBB", "CCC", "DDD", "EEE", "FFF", "FFF"],
+        "location": ["Vulcan", "Reisa", "Bajor", "Deep Space 9", "Earth", "Cardassia", "Cardassia"],
+        "genbank_accession": ["A1", "A2", "B1", "B2", "C1", "C2", "C2"],
+        "genbank_accession_rev": ["A1.1", "A2.4", "B1.1", "B2.5", "C1.1", "C2.1", "C2.1"],
+        "unwanted_column": [1, 2, 3, 4, 5, 6, 7],
     }
 
     lf_metadata = pl.LazyFrame(test_genome_metadata)