Repository: regro/regolith

Issue:
ENH: filter and print key functions not so tightly coupled.
Basically, I kind of want the powerful kv_filtering available but then do other interesting things with the filtered data than just print ids or a key value.  Can we integrate the kv_filtering into the main flow?

Discussion:
mwinitch: I have modified `l_members` to show what I think could be the functionality you want. Basically I have made it so if the filter flags are given the new valid collection becomes the database the lister goes through, otherwise, the lister goes through the default collection. If this looks good to you I can modify the other listers in a similar way. 
codecov[bot]: # [Codecov](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=h1) Report
> Merging [#493](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=desc) into [master](https://codecov.io/gh/regro/regolith/commit/b81510e76309b09c3943db9c4d2cd552fda94d8c&el=desc) will **increase** coverage by `0.02%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/regro/regolith/pull/493/graphs/tree.svg?width=650&height=150&src=pr&token=YOWfAqEPVS)](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #493      +/-   ##
==========================================
+ Coverage   64.53%   64.55%   +0.02%     
==========================================
  Files          55       55              
  Lines        5081     5084       +3     
==========================================
+ Hits         3279     3282       +3     
  Misses       1802     1802              
```


| [Impacted Files](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [regolith/helpers/l\_membershelper.py](https://codecov.io/gh/regro/regolith/pull/493/diff?src=pr&el=tree#diff-cmVnb2xpdGgvaGVscGVycy9sX21lbWJlcnNoZWxwZXIucHk=) | `90.00% <100.00%> (+0.52%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=footer). Last update [b81510e...17b1be9](https://codecov.io/gh/regro/regolith/pull/493?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/news/kv_filter_expansion.rst b/news/kv_filter_expansion.rst
new file mode 100644
index 00000000..af320eaa
--- /dev/null
+++ b/news/kv_filter_expansion.rst
@@ -0,0 +1,12 @@
+**Added:** None
+
+**Changed:**
+    * Key value pair filter is now integrated with the other flags for listers
+
+**Deprecated:** None
+
+**Removed:** None
+
+**Fixed:** None
+
+**Security:** None
\ No newline at end of file
diff --git a/regolith/helpers/l_contactshelper.py b/regolith/helpers/l_contactshelper.py
index 468c934e..b3fc7c84 100644
--- a/regolith/helpers/l_contactshelper.py
+++ b/regolith/helpers/l_contactshelper.py
@@ -13,7 +13,9 @@ from regolith.tools import (
     all_docs_from_collection,
     get_pi_id,
     fragment_retrieval,
-    search_collection
+    search_collection,
+    key_value_pair_filter,
+    collection_str
 )
 
 TARGET_COLL = "contacts"
@@ -101,22 +103,22 @@ class ContactsListerHelper(SoutHelperBase):
     def sout(self):
         rc = self.rc
         if rc.filter:
-            results = search_collection(self.gtx["contacts"], rc.filter, rc.keys)
-            print(results, end="")
-            return
+            collection = key_value_pair_filter(self.gtx["contacts"], rc.filter)
+        else:
+            collection = self.gtx["contacts"]
         def_l = set(stringify(i) for i in
                     self.gtx['contacts'])
         if rc.name:
             namel = set(stringify(i) for i in
                         fragment_retrieval(
-                            self.gtx['contacts'], [
+                            collection, [
                                 "_id", "aka", "name"], rc.name))
         else:
             namel = def_l
         if rc.inst:
             instl = set(stringify(i) for i in
                         fragment_retrieval(
-                            self.gtx['contacts'],
+                            collection,
                             ["institution"],
                             rc.inst))
         else:
@@ -124,7 +126,7 @@ class ContactsListerHelper(SoutHelperBase):
         if rc.notes:
             notel = set(stringify(i) for i in
                         fragment_retrieval(
-                            self.gtx['contacts'], [
+                            collection, [
                                 "notes"], rc.notes))
         else:
             notel = def_l
@@ -140,7 +142,7 @@ class ContactsListerHelper(SoutHelperBase):
                              dateutil.relativedelta.relativedelta(
                                  months=int(
                                      rc.range))).isoformat()}
-            for contact in self.gtx["contacts"]:
+            for contact in collection:
                 curr_d = get_dates(contact)['date']
                 if is_current(temp_dict, now=curr_d):
                     date_list.append(stringify(contact))
@@ -148,6 +150,10 @@ class ContactsListerHelper(SoutHelperBase):
         else:
             datel = def_l
         res_l = set.intersection(namel, instl, notel, datel)
+        if rc.keys:
+            results = (collection_str(res_l, rc.keys))
+            print(results, end="")
+            return
         for item in res_l:
             print(item)
         return
diff --git a/regolith/helpers/l_grantshelper.py b/regolith/helpers/l_grantshelper.py
index df69db17..ea3f59e8 100644
--- a/regolith/helpers/l_grantshelper.py
+++ b/regolith/helpers/l_grantshelper.py
@@ -12,7 +12,9 @@ from regolith.fsclient import _id_key
 from regolith.tools import (
     all_docs_from_collection,
     get_pi_id,
-    search_collection
+    search_collection,
+    key_value_pair_filter,
+    collection_str
 )
 
 TARGET_COLL = "grants"
@@ -67,21 +69,25 @@ class GrantsListerHelper(SoutHelperBase):
     def sout(self):
         rc = self.rc
         if rc.filter:
-            results = search_collection(self.gtx["grants"], rc.filter, rc.keys)
-            print(results, end="")
-            return
+            collection = key_value_pair_filter(self.gtx["grants"], rc.filter)
+        else:
+            collection = self.gtx["grants"]
         grants = []
         if rc.date:
             desired_date = date_parser.parse(rc.date).date()
         else:
             desired_date = dt.date.today()
-        for grant in self.gtx["grants"]:
+        for grant in collection:
             if rc.current and not is_current(grant, now=desired_date):
                 continue
             grants.append(grant)
 
         # Sort the grants by end date in reverse chronological order
         grants.sort(key=lambda k: get_dates(k).get('end_date'), reverse=True)
+        if rc.keys:
+            results = (collection_str(grants, rc.keys))
+            print(results, end="")
+            return
         for g in grants:
             print("{}, awardnr: {}, acctn: {}, {} to {}".format(g.get('alias', ''), g.get('awardnr', ''),
                                                                 g.get('account', ''), get_dates(g).get('begin_date'),
diff --git a/regolith/helpers/l_membershelper.py b/regolith/helpers/l_membershelper.py
index d85926a6..9d6bf0ee 100644
--- a/regolith/helpers/l_membershelper.py
+++ b/regolith/helpers/l_membershelper.py
@@ -11,7 +11,8 @@ from regolith.helpers.basehelper import SoutHelperBase
 from regolith.fsclient import _id_key
 from regolith.tools import (
     all_docs_from_collection,
-    get_pi_id, search_collection
+    get_pi_id, search_collection,
+    key_value_pair_filter, collection_str
 )
 
 TARGET_COLL = "people"
@@ -68,16 +69,21 @@ class MembersListerHelper(SoutHelperBase):
     def sout(self):
         rc = self.rc
         if rc.filter:
-            results = search_collection(self.gtx["people"], rc.filter, rc.keys)
-            print(results, end="")
-            return
+            collection = key_value_pair_filter(self.gtx["people"], rc.filter)
+        else:
+            collection = self.gtx["people"]
         bad_stati = ["finished", "cancelled", "paused", "back_burner"]
         people = []
-        for person in self.gtx["people"]:
+        for person in collection:
             if rc.current and not person.get('active'):
                 continue
             people.append(person)
-        #people.sort()
+
+        if rc.filter and not rc.verbose:
+            results = (collection_str(people, rc.keys))
+            print(results, end="")
+            return
+
         for i in people:
             if rc.verbose:
                 print("{}, {} | group_id: {}".format(i.get('name'), i.get('position'), i.get('_id')))
diff --git a/regolith/helpers/l_milestoneshelper.py b/regolith/helpers/l_milestoneshelper.py
index 0eadd5e6..a22feebf 100644
--- a/regolith/helpers/l_milestoneshelper.py
+++ b/regolith/helpers/l_milestoneshelper.py
@@ -14,14 +14,16 @@ from regolith.fsclient import _id_key
 from regolith.tools import (
     all_docs_from_collection,
     get_pi_id,
-    search_collection
+    search_collection,
+    key_value_pair_filter,
+    collection_str
 )
 
 TARGET_COLL = "projecta"
 HELPER_TARGET = "l_milestones"
 ALLOWED_STATI = ["all", "proposed", "started", "finished", "back_burner",
                  "paused", "cancelled"]
-ROLES = ['pi',  'lead', 'group_members', 'collaborators']
+ROLES = ['pi', 'lead', 'group_members', 'collaborators']
 
 
 def subparser(subpi):
@@ -84,13 +86,13 @@ class MilestonesListerHelper(SoutHelperBase):
         rc = self.rc
         # This if statement should be in all listers. Make sure to change self.gtx to get the database the lister needs
         if rc.filter:
-            results = search_collection(self.gtx["projecta"], rc.filter, rc.keys)
-            print(results, end="")
-            return
+            collection = key_value_pair_filter(self.gtx["projecta"], rc.filter)
+        else:
+            collection = self.gtx["projecta"]
         all_milestones = []
         if not rc.stati:
             rc.stati = ['started']
-        for projectum in self.gtx["projecta"]:
+        for projectum in collection:
             if rc.lead and projectum.get('lead') != rc.lead:
                 continue
             projectum["deliverable"].update({"name": "deliverable",
@@ -115,18 +117,22 @@ class MilestonesListerHelper(SoutHelperBase):
                         })
                         all_milestones.append(ms)
         all_milestones.sort(key=lambda x: x['due_date'], reverse=True)
+        if rc.keys:
+            results = (collection_str(all_milestones, rc.keys))
+            print(results, end="")
+            return
         for ms in all_milestones:
             if rc.verbose:
                 print(
                     f"{ms.get('due_date')}: lead: {ms.get('lead')}, {ms.get('id')}, status: {ms.get('status')}")
-                print(f"    Type: {ms.get('type','')}")
+                print(f"    Type: {ms.get('type', '')}")
                 print(f"    Title: {ms.get('name')}")
                 print(f"    log url: {ms.get('log_url')}")
                 print(f"    Purpose: {ms.get('objective')}")
                 audience = []
                 for i in ms.get('audience'):
-                    if isinstance(ms.get(i,i), str):
-                        audience.append(ms.get(i,i))
+                    if isinstance(ms.get(i, i), str):
+                        audience.append(ms.get(i, i))
                     else:
                         if ms.get(i):
                             audience.extend(ms.get(i))
@@ -137,4 +143,3 @@ class MilestonesListerHelper(SoutHelperBase):
                     f"{ms.get('due_date')}: lead: {ms.get('lead')}, {ms.get('id')}, {ms.get('name')}, status: {ms.get('status')}")
 
         return
-
diff --git a/regolith/helpers/l_projectahelper.py b/regolith/helpers/l_projectahelper.py
index 5a195041..fa9ace1d 100644
--- a/regolith/helpers/l_projectahelper.py
+++ b/regolith/helpers/l_projectahelper.py
@@ -14,7 +14,9 @@ from regolith.fsclient import _id_key
 from regolith.tools import (
     all_docs_from_collection,
     get_pi_id,
-    search_collection
+    search_collection,
+    key_value_pair_filter,
+    collection_str
 )
 
 TARGET_COLL = "projecta"
@@ -98,11 +100,11 @@ class ProjectaListerHelper(SoutHelperBase):
     def sout(self):
         rc = self.rc
         if rc.filter:
-            results = search_collection(self.gtx["projecta"], rc.filter, rc.keys)
-            print(results, end="")
-            return
+            collection = key_value_pair_filter(self.gtx["projecta"], rc.filter)
+        else:
+            collection = self.gtx["projecta"]
 
-        if (not rc.lead) and (not rc.person) and (not rc.ended) and (not rc.grant) and (not rc.verbose) and (not rc.grp_by_lead):
+        if (not rc.lead) and (not rc.person) and (not rc.ended) and (not rc.grant) and (not rc.verbose) and (not rc.grp_by_lead) and (not rc.filter):
             return
         if rc.date:
             desired_date = date_parser.parse(rc.date).date()
@@ -120,7 +122,7 @@ class ProjectaListerHelper(SoutHelperBase):
         grouped_projecta = {}
         if rc.lead and rc.person:
             raise RuntimeError(f"please specify either lead or person, not both")
-        for projectum in self.gtx["projecta"]:
+        for projectum in collection:
             if isinstance(projectum.get('group_members'), str):
                 projectum['group_members'] = [projectum.get('group_members')]
             if rc.lead and projectum.get('lead') != rc.lead:
@@ -166,6 +168,8 @@ class ProjectaListerHelper(SoutHelperBase):
                                                                                           p.get("description"),
                                                                                           p.get("lead"), members,
                                                                                           collaborators))
+            return
+
         if rc.grp_by_lead:
             for p in projecta:
                 if p.get('lead') not in grouped_projecta:
@@ -179,6 +183,11 @@ class ProjectaListerHelper(SoutHelperBase):
             return
 
         projecta.sort(key=lambda prum: prum.get("_id"))
+        if rc.keys:
+            results = (collection_str(projecta, rc.keys))
+            print(results, end="")
+            return
+
         for i in projecta:
             print(i.get("_id"))
         return


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 8eec61a8..866f8a45 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -64,6 +64,10 @@ helper_map = [
     (["helper", "l_members", "--filter", "name", "sco"],
      "scopatz    \n"
      ),
+    (["helper", "l_members", "--filter", "name", "sco", "-v"],
+     "Anthony Scopatz, professor | group_id: scopatz\n"
+     "    orcid: 0000-0002-9432-4248 | github_id: ascopatz\n"
+     ),
     (["helper", "l_contacts", "run", "--name", "ny", "--inst", "col",
       "--notes", "coffee", "--date", "2020-01-15", "--range", "2"],
      "name: Anthony B Friend, institution: columbiau, email: friend@deed.com\n"