Repository: pandas-dev/pandas

Issue:
Followup to #20583 (observed keyword for Groupby)
1. Change Grouping.__init__ default observed to False: https://github.com/pandas-dev/pandas/pull/20583#discussion_r185209589
2. whatsnew description on unobserved, cartesian product: https://github.com/pandas-dev/pandas/pull/20583#discussion_r185208219
3. Simplify implementation? https://github.com/pandas-dev/pandas/pull/20583#discussion_r185203879
4. rst formatting: https://github.com/pandas-dev/pandas/pull/20583/files/bdf7525812ca670f9406ab8df333030d36d30947#diff-03b380f521c43cf003207b0711bac67f
5. stale comment? https://github.com/pandas-dev/pandas/pull/20583#discussion_r185209781


Discussion:

Resolving patch:
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -6584,7 +6584,7 @@ def clip_lower(self, threshold, axis=None, inplace=False):
                                          axis=axis, inplace=inplace)
 
     def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,
-                group_keys=True, squeeze=False, observed=None, **kwargs):
+                group_keys=True, squeeze=False, observed=False, **kwargs):
         """
         Group series using mapper (dict or key function, apply given function
         to group, return result as series) or by a series of columns.
@@ -6617,11 +6617,10 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,
         squeeze : boolean, default False
             reduce the dimensionality of the return type if possible,
             otherwise return a consistent type
-        observed : boolean, default None
-            if True: only show observed values for categorical groupers.
-            if False: show all values for categorical groupers.
-            if None: if any categorical groupers, show a FutureWarning,
-                default to False.
+        observed : boolean, default False
+            This only applies if any of the groupers are Categoricals
+            If True: only show observed values for categorical groupers.
+            If False: show all values for categorical groupers.
 
             .. versionadded:: 0.23.0
 
diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py
--- a/pandas/core/groupby/groupby.py
+++ b/pandas/core/groupby/groupby.py
@@ -556,7 +556,7 @@ class _GroupBy(PandasObject, SelectionMixin):
     def __init__(self, obj, keys=None, axis=0, level=None,
                  grouper=None, exclusions=None, selection=None, as_index=True,
                  sort=True, group_keys=True, squeeze=False,
-                 observed=None, **kwargs):
+                 observed=False, **kwargs):
 
         self._selection = selection
 
@@ -2907,7 +2907,7 @@ class Grouping(object):
     """
 
     def __init__(self, index, grouper=None, obj=None, name=None, level=None,
-                 sort=True, observed=None, in_axis=False):
+                 sort=True, observed=False, in_axis=False):
 
         self.name = name
         self.level = level
@@ -2964,12 +2964,6 @@ def __init__(self, index, grouper=None, obj=None, name=None, level=None,
             # a passed Categorical
             elif is_categorical_dtype(self.grouper):
 
-                # observed can be True/False/None
-                # we treat None as False. If in the future
-                # we need to warn if observed is not passed
-                # then we have this option
-                # gh-20583
-
                 self.all_grouper = self.grouper
                 self.grouper = self.grouper._codes_for_groupby(
                     self.sort, observed)
@@ -3088,7 +3082,7 @@ def groups(self):
 
 
 def _get_grouper(obj, key=None, axis=0, level=None, sort=True,
-                 observed=None, mutated=False, validate=True):
+                 observed=False, mutated=False, validate=True):
     """
     create and return a BaseGrouper, which is an internal
     mapping of how to create the grouper indexers.
@@ -4734,26 +4728,28 @@ def _wrap_agged_blocks(self, items, blocks):
 
     def _reindex_output(self, result):
         """
-        if we have categorical groupers, then we want to make sure that
+        If we have categorical groupers, then we want to make sure that
         we have a fully reindex-output to the levels. These may have not
         participated in the groupings (e.g. may have all been
-        nan groups)
+        nan groups);
 
         This can re-expand the output space
         """
 
-        # TODO(jreback): remove completely
-        # when observed parameter is defaulted to True
-        # gh-20583
-
-        if self.observed:
-            return result
-
+        # we need to re-expand the output space to accomodate all values
+        # whether observed or not in the cartesian product of our groupes
         groupings = self.grouper.groupings
         if groupings is None:
             return result
         elif len(groupings) == 1:
             return result
+
+        # if we only care about the observed values
+        # we are done
+        elif self.observed:
+            return result
+
+        # reindexing only applies to a Categorical grouper
         elif not any(isinstance(ping.grouper, (Categorical, CategoricalIndex))
                      for ping in groupings):
             return result


Test patch: