Repository: lifeomic/phc-sdk-py

Issue:
Auto-caching breaks when no results
It appears that this behavior happens because it's the "last batch" and the APICache callback expects that the file already exists at that point.

Stacktrace:
```python
/opt/conda/lib/python3.7/site-packages/phc/easy/procedure.py in get_data_frame(all_results, raw, patient_id, query_overrides, auth_args, ignore_cache, expand_args)
     96             query_overrides,
     97             auth_args,
---> 98             ignore_cache,
     99         )

/opt/conda/lib/python3.7/site-packages/phc/easy/query/__init__.py in execute_fhir_dsl_with_options(query, transform, all_results, raw, query_overrides, auth_args, ignore_cache)
    157                 auth_args,
    158                 callback=APICache.build_cache_fhir_dsl_callback(
--> 159                     query, transform
    160                 ),
    161             )

/opt/conda/lib/python3.7/site-packages/phc/easy/query/__init__.py in execute_fhir_dsl(query, all_results, auth_args, callback)
    110             return with_progress(
    111                 lambda: tqdm(total=MAX_RESULT_SIZE),
--> 112                 lambda progress: recursive_execute_fhir_dsl(
    113                     {
    114                         "limit": [

/opt/conda/lib/python3.7/site-packages/phc/easy/query/fhir_dsl.py in with_progress(init_progress, func)
     20     if _has_tqdm:
     21         progress = init_progress()
---> 22         result = func(progress)
     23         progress.close()
     24         return result

/opt/conda/lib/python3.7/site-packages/phc/easy/query/__init__.py in <lambda>(progress)
    126                     progress=progress,
    127                     callback=callback,
--> 128                     auth_args=auth_args,
    129                 ),
    130             )

/opt/conda/lib/python3.7/site-packages/phc/easy/query/fhir_dsl.py in recursive_execute_fhir_dsl(query, scroll, progress, auth_args, callback, _scroll_id, _prev_hits)
     72         callback(current_results, False)
     73     elif callback and is_last_batch:
---> 74         return callback(current_results, True)
     75     elif is_last_batch:
     76         suffix = "+" if actual_count == MAX_RESULT_SIZE else ""

/opt/conda/lib/python3.7/site-packages/phc/util/api_cache.py in handle_batch(batch, is_finished)
     76             if is_finished:
     77                 print(f'Loading data frame from "{filename}"')
---> 78                 return APICache.read_csv(filename)
     79 
     80             df = pd.DataFrame(map(lambda r: r["_source"], batch))

/opt/conda/lib/python3.7/site-packages/phc/util/api_cache.py in read_csv(filename)
     85     @staticmethod
     86     def read_csv(filename: str) -> pd.DataFrame:
---> 87         df = pd.read_csv(filename)
     88         min_count = max(min(int(len(df) / 3), 5), 1)
     89 

/opt/conda/lib/python3.7/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
    674         )
    675 
--> 676         return _read(filepath_or_buffer, kwds)
    677 
    678     parser_f.__name__ = name

/opt/conda/lib/python3.7/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
    446 
    447     # Create the parser.
--> 448     parser = TextFileReader(fp_or_buf, **kwds)
    449 
    450     if chunksize or iterator:

/opt/conda/lib/python3.7/site-packages/pandas/io/parsers.py in __init__(self, f, engine, **kwds)
    878             self.options["has_index_names"] = kwds["has_index_names"]
    879 
--> 880         self._make_engine(self.engine)
    881 
    882     def close(self):

/opt/conda/lib/python3.7/site-packages/pandas/io/parsers.py in _make_engine(self, engine)
   1112     def _make_engine(self, engine="c"):
   1113         if engine == "c":
-> 1114             self._engine = CParserWrapper(self.f, **self.options)
   1115         else:
   1116             if engine == "python":

/opt/conda/lib/python3.7/site-packages/pandas/io/parsers.py in __init__(self, src, **kwds)
   1889         kwds["usecols"] = self.usecols
   1890 
-> 1891         self._reader = parsers.TextReader(src, **kwds)
   1892         self.unnamed_cols = self._reader.unnamed_cols
   1893 

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source()

FileNotFoundError: [Errno 2] File ~/Downloads/phc/api-cache/fhir_dsl_procedure_where_********.csv does not exist: '~/Downloads/phc/api-cache/fhir_dsl_procedure_where_********.csv'
```

Comments:

Patch:
diff --git a/phc/easy/patients/__init__.py b/phc/easy/patients/__init__.py
index c6ecfb0..fe3c26d 100644
--- a/phc/easy/patients/__init__.py
+++ b/phc/easy/patients/__init__.py
@@ -1,24 +1,23 @@
 import pandas as pd
 
-from phc.easy.auth import Auth
 from phc.easy.frame import Frame
+from phc.easy.item import Item
 from phc.easy.patients.address import expand_address_column
 from phc.easy.patients.name import expand_name_column
-from phc.easy.query import Query
 
 
-class Patient:
+class Patient(Item):
     @staticmethod
-    def get_count(query_overrides: dict = {}, auth_args=Auth.shared()):
-        return Query.find_count_of_dsl_query(
-            {
-                "type": "select",
-                "columns": "*",
-                "from": [{"table": "patient"}],
-                **query_overrides,
-            },
-            auth_args=auth_args,
-        )
+    def table_name():
+        return "patient"
+
+    @staticmethod
+    def code_keys():
+        return [
+            "extension.valueCodeableConcept.coding",
+            "identifier.type.coding",
+            "meta.tag",
+        ]
 
     @staticmethod
     def transform_results(data_frame: pd.DataFrame, **expand_args):
@@ -32,65 +31,3 @@ class Patient:
         }
 
         return Frame.expand(data_frame, **args)
-
-    @staticmethod
-    def get_data_frame(
-        all_results: bool = False,
-        raw: bool = False,
-        query_overrides: dict = {},
-        auth_args: Auth = Auth.shared(),
-        ignore_cache: bool = False,
-        expand_args: dict = {},
-    ):
-        """Retrieve patients as a data frame with unwrapped FHIR columns
-
-        Attributes
-        ----------
-        all_results : bool = False
-            Override limit to retrieve all patients
-
-        raw : bool = False
-            If raw, then values will not be expanded (useful for manual
-            inspection if something goes wrong). Note that this option will
-            override all_results if True.
-
-        query_overrides : dict = {}
-            Override any part of the elasticsearch FHIR query
-
-        auth_args : Any
-            The authenication to use for the account and project (defaults to shared)
-
-        ignore_cache : bool = False
-            Bypass the caching system that auto-saves results to a CSV file.
-            Caching only occurs when all results are being retrieved.
-
-        expand_args : Any
-            Additional arguments passed to phc.Frame.expand
-
-        Examples
-        --------
-        >>> import phc.easy as phc
-        >>> phc.Auth.set({'account': '<your-account-name>'})
-        >>> phc.Project.set_current('My Project Name')
-        >>> phc.Patient.get_data_frame()
-
-        """
-        query = {
-            "type": "select",
-            "columns": "*",
-            "from": [{"table": "patient"}],
-            **query_overrides,
-        }
-
-        def transform(df: pd.DataFrame):
-            return Patient.transform_results(df, **expand_args)
-
-        return Query.execute_fhir_dsl_with_options(
-            query,
-            transform,
-            all_results,
-            raw,
-            query_overrides,
-            auth_args,
-            ignore_cache,
-        )
diff --git a/phc/easy/patients/address.py b/phc/easy/patients/address.py
index edbd1c8..c7af9a3 100644
--- a/phc/easy/patients/address.py
+++ b/phc/easy/patients/address.py
@@ -1,4 +1,6 @@
 import pandas as pd
+
+from funcy import first
 from phc.easy.codeable import generic_codeable_to_dict
 from phc.easy.util import concat_dicts
 
@@ -22,18 +24,29 @@ def expand_address_value(value):
     if type(value) is not list:
         return {}
 
-    # Value is always list of one item
-    assert len(value) == 1
-    value = value[0]
+    primary_address = first(
+        filter(lambda v: v.get("use") != "old", value)
+    ) or first(value)
+
+    other_addresses = list(
+        filter(lambda address: address != primary_address, value)
+    )
 
-    return concat_dicts(
-        [
-            expand_address_attr(f"address_{key}", item_value)
-            for key, item_value in value.items()
-            if key != "text"
-        ]
+    other_attrs = (
+        {"other_addresses": other_addresses} if len(other_addresses) > 0 else {}
     )
 
+    return {
+        **concat_dicts(
+            [
+                expand_address_attr(f"address_{key}", item_value)
+                for key, item_value in primary_address.items()
+                if key != "text"
+            ]
+        ),
+        **other_attrs,
+    }
+
 
 def expand_address_column(address_col):
     return pd.DataFrame(map(expand_address_value, address_col.values))
diff --git a/phc/util/api_cache.py b/phc/util/api_cache.py
index 0e7f58d..475d88b 100644
--- a/phc/util/api_cache.py
+++ b/phc/util/api_cache.py
@@ -1,5 +1,6 @@
 import hashlib
 import json
+import os
 from pathlib import Path
 from typing import Callable
 
@@ -86,6 +87,9 @@ class APICache:
         writer = CSVWriter(filename)
 
         def handle_batch(batch, is_finished):
+            if is_finished and not os.path.exists(filename):
+                return pd.DataFrame()
+
             if is_finished:
                 print(f'Loading data frame from "{filename}"')
                 return APICache.read_csv(filename)


Test patch:
diff --git a/tests/test_easy_patient_address.py b/tests/test_easy_patient_address.py
index d51ff86..42bb1cd 100644
--- a/tests/test_easy_patient_address.py
+++ b/tests/test_easy_patient_address.py
@@ -1,18 +1,64 @@
 import pandas as pd
+import math
 from phc.easy.patients.address import expand_address_column
 
+
+def non_na_dict(dictionary: dict):
+    return {
+        k: v
+        for k, v in dictionary.items()
+        if not isinstance(v, float) or not math.isnan(v)
+    }
+
+
 def test_expand_address_column():
-    sample = pd.DataFrame([{
-        'address': [
-            {'line': ['123 ABC Court'], 'city': 'Zionsville', 'state': 'IN', 'use': 'home'}
+    sample = pd.DataFrame(
+        [
+            {
+                "address": [
+                    {
+                        "line": ["123 ABC Court"],
+                        "city": "Zionsville",
+                        "state": "IN",
+                        "use": "home",
+                    }
+                ]
+            },
+            {
+                "address": [
+                    {
+                        "use": "old",
+                        "state": "SC",
+                        "period": {"start": "1999", "end": "2001"},
+                    },
+                    {
+                        "state": "NC",
+                        "city": "Raleigh",
+                        "period": {"start": "2001"},
+                    },
+                ]
+            },
         ]
-    }])
+    )
 
     df = expand_address_column(sample.address)
 
-    assert df.iloc[0].to_dict() == {
-        'address_line_0': '123 ABC Court',
-        'address_city': 'Zionsville',
-        'address_state': 'IN',
-        'address_use': 'home'
+    assert non_na_dict(df.iloc[0].to_dict()) == {
+        "address_line_0": "123 ABC Court",
+        "address_city": "Zionsville",
+        "address_state": "IN",
+        "address_use": "home",
+    }
+
+    assert non_na_dict(df.iloc[1].to_dict()) == {
+        "address_city": "Raleigh",
+        "address_state": "NC",
+        "address_period_start": "2001",
+        "other_addresses": [
+            {
+                "use": "old",
+                "state": "SC",
+                "period": {"start": "1999", "end": "2001"},
+            }
+        ],
     }