Deprecate default observed=False for groupby

Author: jseabold

pandas/core/frame.py

@@ -6698,7 +6698,7 @@ def groupby(
         sort: bool = True,
         group_keys: bool = True,
         squeeze: bool = no_default,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         dropna: bool = True,
     ) -> DataFrameGroupBy:
         from pandas.core.groupby.generic import DataFrameGroupBy

pandas/core/groupby/groupby.py

@@ -526,7 +526,7 @@ def __init__(
         sort: bool = True,
         group_keys: bool = True,
         squeeze: bool = False,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         mutated: bool = False,
         dropna: bool = True,
     ):
@@ -3009,7 +3009,7 @@ def get_groupby(
     sort: bool = True,
     group_keys: bool = True,
     squeeze: bool = False,
-    observed: bool = False,
+    observed: Optional[bool] = None,
     mutated: bool = False,
     dropna: bool = True,
 ) -> GroupBy:

pandas/core/groupby/grouper.py

@@ -2,6 +2,7 @@
 Provide user facing operators for doing the split part of the
 split-apply-combine paradigm.
 """
+import textwrap
 from typing import Dict, Hashable, List, Optional, Set, Tuple
 import warnings
 
@@ -31,6 +32,17 @@
 
 from pandas.io.formats.printing import pprint_thing
 
+_observed_msg = textwrap.dedent(
+    """\
+Using 'observed=False', because grouping on a categorical. A future version
+of pandas will change to 'observed=True'.
+
+To silence the warning and switch to the future behaior, pass 'observed=True'.
+
+To keep the current behavior and silence the behavior, pass 'observed=False'.
+"""
+)
+
 
 class Grouper:
     """
@@ -432,7 +444,7 @@ def __init__(
         name=None,
         level=None,
         sort: bool = True,
-        observed: bool = False,
+        observed: Optional[bool] = None,
         in_axis: bool = False,
         dropna: bool = True,
     ):
@@ -495,6 +507,9 @@ def __init__(
             # a passed Categorical
             elif is_categorical_dtype(self.grouper):
 
+                if observed is None:
+                    warnings.warn(_observed_msg, FutureWarning, stacklevel=5)
+
                 self.grouper, self.all_grouper = recode_for_groupby(
                     self.grouper, self.sort, observed
                 )
@@ -631,7 +646,7 @@ def get_grouper(
     axis: int = 0,
     level=None,
     sort: bool = True,
-    observed: bool = False,
+    observed: Optional[bool] = None,
     mutated: bool = False,
     validate: bool = True,
     dropna: bool = True,

pandas/core/shared_docs.py

@@ -119,6 +119,15 @@
     This only applies if any of the groupers are Categoricals.
     If True: only show observed values for categorical groupers.
     If False: show all values for categorical groupers.
+
+    The current default of ``observed=False`` is deprecated and will
+    change to ``observed=True`` in a future version of pandas.
+
+    Explicitly pass ``observed=True`` to silence the warning and not
+    show all observed values.
+    Explicitly pass ``observed=False`` to silence the warning and
+    show groups for all observed values.
+
 dropna : bool, default True
     If True, and if group keys contain NA values, NA values together
     with row/column will be dropped.

pandas/tests/groupby/test_grouping.py

@@ -703,6 +703,21 @@ def test_groupby_multiindex_level_empty(self):
         )
         tm.assert_frame_equal(result, expected)
 
+    def test_default_observed_deprecated(self):
+        df = pd.DataFrame([
+            ['A', 1, 1], ['A', 2, 1], ['B', 1, 1]
+        ], columns=['x', 'y', 'z'])
+        df.x = df.x.astype('category')
+        df.y = df.x.astype('category')
+
+        with tm.assert_produces_warning(expected_warning=FutureWarning):
+            df.groupby(['x', 'y'])
+
+        with pytest.warns(None) as any_warnings:
+            df.groupby(['x', 'y'], observed=True)
+            df.groupby(['x', 'y'], observed=False)
+        assert len(any_warnings) == 0
+
 
 # get_group
 # --------------------------------