Deprecate default observed=False for groupby

Author: jseabold

pandas/core/frame.py

@@ -6524,7 +6524,7 @@ def groupby(
         sort: bool = True,
         group_keys: bool = True,
         squeeze: bool = no_default,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         dropna: bool = True,
     ) -> "DataFrameGroupBy":
         from pandas.core.groupby.generic import DataFrameGroupBy

pandas/core/generic.py

@@ -7495,6 +7495,14 @@ def clip(
             If True: only show observed values for categorical groupers.
             If False: show all values for categorical groupers.
 
+            The current default of ``observed=False`` is deprecated and will
+            change to ``observed=True`` in a future version of pandas.
+
+            Explicitly pass ``observed=True`` to silence the warning and not
+            show all observed values.
+            Explicitly pass ``observed=False`` to silence the warning and
+            show groups for all observed values.
+
             .. versionadded:: 0.23.0
         dropna : bool, default True
             If True, and if group keys contain NA values, NA values together

pandas/core/groupby/groupby.py

@@ -496,7 +496,7 @@ def __init__(
         sort: bool = True,
         group_keys: bool = True,
         squeeze: bool = False,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         mutated: bool = False,
         dropna: bool = True,
     ):
@@ -2912,7 +2912,7 @@ def get_groupby(
     sort: bool = True,
     group_keys: bool = True,
     squeeze: bool = False,
-    observed: bool = False,
+    observed: Optional[bool] = None,
     mutated: bool = False,
     dropna: bool = True,
 ) -> GroupBy:

pandas/core/groupby/grouper.py

@@ -2,6 +2,7 @@
 Provide user facing operators for doing the split part of the
 split-apply-combine paradigm.
 """
+import textwrap
 from typing import Dict, Hashable, List, Optional, Tuple
 import warnings
 
@@ -31,6 +32,17 @@
 
 from pandas.io.formats.printing import pprint_thing
 
+_observed_msg = textwrap.dedent(
+    """\
+Using 'observed=False', because grouping on a categorical. A future version
+of pandas will change to 'observed=True'.
+
+To silence the warning and switch to the future behaior, pass 'observed=True'.
+
+To keep the current behavior and silence the behavior, pass 'observed=False'.
+"""
+)
+
 
 class Grouper:
     """
@@ -418,7 +430,7 @@ def __init__(
         name=None,
         level=None,
         sort: bool = True,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         in_axis: bool = False,
         dropna: bool = True,
     ):
@@ -481,6 +493,9 @@ def __init__(
             # a passed Categorical
             elif is_categorical_dtype(self.grouper):
 
+                if observed is None:
+                    warnings.warn(_observed_msg, FutureWarning, stacklevel=5)
+
                 self.grouper, self.all_grouper = recode_for_groupby(
                     self.grouper, self.sort, observed
                 )
@@ -603,7 +618,7 @@ def get_grouper(
     axis: int = 0,
     level=None,
     sort: bool = True,
-    observed: bool = False,
+    observed: Optional[bool] = None,
     mutated: bool = False,
     validate: bool = True,
     dropna: bool = True,

pandas/tests/groupby/test_grouping.py

@@ -691,6 +691,21 @@ def test_groupby_multiindex_level_empty(self):
         )
         tm.assert_frame_equal(result, expected)
 
+    def test_default_observed_deprecated(self):
+        df = pd.DataFrame([
+            ['A', 1, 1], ['A', 2, 1], ['B', 1, 1]
+        ], columns=['x', 'y', 'z'])
+        df.x = df.x.astype('category')
+        df.y = df.x.astype('category')
+
+        with tm.assert_produces_warning(expected_warning=FutureWarning):
+            df.groupby(['x', 'y'])
+
+        with pytest.warns(None) as any_warnings:
+            df.groupby(['x', 'y'], observed=True)
+            df.groupby(['x', 'y'], observed=False)
+        assert len(any_warnings) == 0
+
 
 # get_group
 # --------------------------------