Repository: glotzerlab/signac

Issue:
groupby() by variables that not present in the entire data space
### Feature description

I am interested in obtaining all the jobs that contain a variable `b` which does not necessarily have to be present in all the jobs.


### Proposed solution
From @bdice in the slack group:

> Bradley Dice  10:43 AM
@Miguel This behavior could be improved, but what would you expect the behavior to be? A group where b is None isn't really an option because b could be set to None in the statepoint. ***We could return only groups where the value is set (and ignore jobs where the key is missing).***

### Additional context
Example of the current problem. First `groupby()` works, second does not.
```python
import signac

project = signac.init_project('test')
sp = { 'a' : 1.0,
       'c' : 2.0
        }
job = project.open_job(sp)
job.init()

sp = { 'a' : 2.0,
       'b' : 1.0,
        }
job = project.open_job(sp)
job.init()

sp = { 'a' : 1.0,
       'b' : 3.0,
        }
job = project.open_job(sp)
job.init()

for a, group in project.groupby('a'):
    print(a, list(group))

for b, group in project.groupby('b'):
    print(b, list(group))
```


Comments:

Patch:
diff --git a/changelog.txt b/changelog.txt
index 91e497f3..69cf687c 100644
--- a/changelog.txt
+++ b/changelog.txt
@@ -15,9 +15,11 @@ Added
 
  - Type annotations are validated during continuous integration (#313).
  - Added ``_repr_html_`` method in ``ProjectSchema`` class (#314, #324).
+ - Allow grouping by variables that are not present in all jobs in the project in ``JobsCursor.groupby`` (#321, #323).
 
 Fixed
 +++++
+
  - Fix the ``signac config verify`` command (previously broken) (#301, #302).
  - Warnings now appear when raised by the ``signac`` CLI (#317, #308).
 
diff --git a/signac/contrib/project.py b/signac/contrib/project.py
index c536c40a..9d85724d 100644
--- a/signac/contrib/project.py
+++ b/signac/contrib/project.py
@@ -1776,8 +1776,14 @@ class JobsCursor(object):
             A default value to be used when a given state point key is not present (must
             be sortable).
         """
+        _filter = self._filter
         if isinstance(key, str):
             if default is None:
+                if _filter is None:
+                    _filter = {key: {"$exists": True}}
+                else:
+                    _filter = {'$and': [{key: {"$exists": True}}, _filter]}
+
                 def keyfunction(job):
                     return job.sp[key]
             else:
@@ -1786,6 +1792,11 @@ class JobsCursor(object):
 
         elif isinstance(key, Iterable):
             if default is None:
+                if _filter is None:
+                    _filter = {k: {"$exists": True} for k in key}
+                else:
+                    _filter = {'$and': [{k: {"$exists": True} for k in key}, _filter]}
+
                 def keyfunction(job):
                     return tuple(job.sp[k] for k in key)
             else:
@@ -1800,7 +1811,8 @@ class JobsCursor(object):
         else:
             keyfunction = key
 
-        return groupby(sorted(iter(self), key=keyfunction), key=keyfunction)
+        return groupby(sorted(iter(JobsCursor(self._project, _filter, self._doc_filter)),
+                              key=keyfunction), key=keyfunction)
 
     def groupbydoc(self, key=None, default=None):
         """Groups jobs according to one or more document values.


Test patch:
diff --git a/tests/test_project.py b/tests/test_project.py
index a128dfd9..100617bc 100644
--- a/tests/test_project.py
+++ b/tests/test_project.py
@@ -836,9 +836,7 @@ class TestProject(TestProjectBase):
             assert len(list(g)) == 6
             for job in list(g):
                 assert job.sp['b'] == k
-        with pytest.raises(KeyError):
-            for k, g in self.project.groupby('d'):
-                pass
+        assert len(list(self.project.groupby('d'))) == 0
         for k, g in self.project.groupby('d', default=-1):
             assert k == -1
             assert len(list(g)) == len(self.project)
@@ -863,6 +861,17 @@ class TestProject(TestProjectBase):
                 assert str(job) == k
         assert group_count == len(list(self.project.find_jobs()))
 
+        self.project.open_job({'a': 20}).init()
+        for k, g in self.project.groupby('b'):
+            assert len(list(g)) == 6
+            for job in list(g):
+                assert job.sp['b'] == k
+        for k, g in self.project.groupby(('b', 'c')):
+            assert len(list(g)) == 2
+            for job in list(g):
+                assert job.sp['b'] == k[0]
+                assert job.sp['c'] == k[1]
+
     def test_jobs_groupbydoc(self):
         def get_doc(i):
             return {