ENH: add BooleanArray extension array (pandas-dev#29555)

Author: jorisvandenbossche

doc/source/boolean.rst

@@ -0,0 +1,83 @@
+.. currentmodule:: pandas
+
+.. _boolean:
+
+**************************
+Nullable Boolean Data Type
+**************************
+
+.. versionadded:: 1.0.0
+
+.. _boolean.klean:
+
+Kleene Logic
+------------
+
+:class:`arrays.BooleanArray` implements Kleene logic (sometime called three-value logic) for
+logical operations like ``&`` (and), ``|`` (or) and ``^`` (exclusive-or).
+
+Here's a table for ``and``.
+
+==========  ===========  ============
+left value  right value  output value
+==========  ===========  ============
+True        True         True
+True        False        False
+True        NA           NA
+False       False        False
+False       NA           False
+NA          NA           NA
+==========  ===========  ============
+
+
+And for ``or``
+
+==========  ===========  ============
+left value  right value  output value
+==========  ===========  ============
+True        True         True
+True        False        True
+True        NA           True
+False       False        False
+False       NA           NA
+NA          NA           NA
+==========  ===========  ============
+
+And for ``xor``
+
+==========  ===========  ============
+left value  right value  output value
+==========  ===========  ============
+True        True         False
+True        False        True
+True        NA           NA
+False       False        False
+False       NA           NA
+NA          NA           NA
+==========  ===========  ============
+
+When an ``NA`` is present in an operation, the output value is ``NA`` only if
+the result cannot be determined soley based on the other input. For example,
+``True | NA`` is ``True``, because both ``True | True`` and ``True | False``
+are ``True``. In that case, we don't actually need to consider the value
+of the ``NA``.
+
+On the other hand, ``True & NA`` is ``NA``. The result depends on whether
+the ``NA`` really is ``True`` or ``False``, since ``True & True`` is ``True``,
+but ``True & False`` is ``False``, so we can't determine the output.
+
+
+This differs from how ``np.nan`` behaves in logical operations. Pandas treated
+``np.nan`` is *always false in the output*.
+
+In ``or``
+
+.. ipython:: python
+
+   pd.Series([True, False, np.nan], dtype="object") | True
+   pd.Series([True, False, np.nan], dtype="boolean") | True
+
+In ``and``
+
+   pd.Series([True, False, np.nan], dtype="object") & True
+   pd.Series([True, False, np.nan], dtype="boolean") & True

doc/source/getting_started/basics.rst

@@ -1950,6 +1950,7 @@ sparse              :class:`SparseDtype`      (none)             :class:`arrays.
 intervals           :class:`IntervalDtype`    :class:`Interval`  :class:`arrays.IntervalArray` :ref:`advanced.intervalindex`
 nullable integer    :class:`Int64Dtype`, ...  (none)             :class:`arrays.IntegerArray`  :ref:`integer_na`
 Strings             :class:`StringDtype`      :class:`str`       :class:`arrays.StringArray`   :ref:`text`
+Boolean (with NA)   :class:`BooleanDtype`     :class:`bool`      :class:`arrays.BooleanArray`  :ref:`api.arrays.bool`
 =================== ========================= ================== ============================= =============================
 
 Pandas has two ways to store strings.

doc/source/index.rst.template

@@ -73,6 +73,7 @@ See the :ref:`overview` for more detail about what's in the library.
   * :doc:`user_guide/missing_data`
   * :doc:`user_guide/categorical`
   * :doc:`user_guide/integer_na`
+  * :doc:`user_guide/boolean`
   * :doc:`user_guide/visualization`
   * :doc:`user_guide/computation`
   * :doc:`user_guide/groupby`

doc/source/reference/arrays.rst

@@ -25,6 +25,7 @@ Nullable Integer    :class:`Int64Dtype`, ...  (none)             :ref:`api.array
 Categorical         :class:`CategoricalDtype` (none)             :ref:`api.arrays.categorical`
 Sparse              :class:`SparseDtype`      (none)             :ref:`api.arrays.sparse`
 Strings             :class:`StringDtype`      :class:`str`       :ref:`api.arrays.string`
+Boolean (with NA)   :class:`BooleanDtype`     :class:`bool`      :ref:`api.arrays.bool`
 =================== ========================= ================== =============================
 
 Pandas and third-party libraries can extend NumPy's type system (see :ref:`extending.extension-types`).
@@ -485,6 +486,28 @@ The ``Series.str`` accessor is available for ``Series`` backed by a :class:`arra
 See :ref:`api.series.str` for more.
 
 
+.. _api.arrays.bool:
+
+Boolean data with missing values
+--------------------------------
+
+The boolean dtype (with the alias ``"boolean"``) provides support for storing
+boolean data (True, False values) with missing values, which is not possible
+with a bool :class:`numpy.ndarray`.
+
+.. autosummary::
+   :toctree: api/
+   :template: autosummary/class_without_autosummary.rst
+
+   arrays.BooleanArray
+
+.. autosummary::
+   :toctree: api/
+   :template: autosummary/class_without_autosummary.rst
+
+   BooleanDtype
+
+
 .. Dtype attributes which are manually listed in their docstrings: including
 .. it here to make sure a docstring page is built for them
 

doc/source/whatsnew/v1.0.0.rst

@@ -102,6 +102,30 @@ String accessor methods returning integers will return a value with :class:`Int6
 We recommend explicitly using the ``string`` data type when working with strings.
 See :ref:`text.types` for more.
 
+.. _whatsnew_100.boolean:
+
+Boolean data type with missing values support
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We've added :class:`BooleanDtype` / :class:`~arrays.BooleanArray`, an extension
+type dedicated to boolean data that can hold missing values. With the default
+``'bool`` data type based on a numpy bool array, the column can only hold
+True or False values and not missing values. This new :class:`BooleanDtype`
+can store missing values as well by keeping track of this in a separate mask.
+(:issue:`29555`)
+
+.. ipython:: python
+
+   pd.Series([True, False, None], dtype=pd.BooleanDtype())
+
+You can use the alias ``"boolean"`` as well.
+
+.. ipython:: python
+
+   s = pd.Series([True, False, None], dtype="boolean")
+   s
+
+
 .. _whatsnew_1000.enhancements.other:
 
 Other enhancements

pandas/__init__.py

@@ -67,6 +67,7 @@
     IntervalDtype,
     DatetimeTZDtype,
     StringDtype,
+    BooleanDtype,
     # missing
     isna,
     isnull,

pandas/arrays/__init__.py

@@ -4,6 +4,7 @@
 See :ref:`extending.extension-types` for more.
 """
 from pandas.core.arrays import (
+    BooleanArray,
     Categorical,
     DatetimeArray,
     IntegerArray,
@@ -16,6 +17,7 @@
 )
 
 __all__ = [
+    "BooleanArray",
     "Categorical",
     "DatetimeArray",
     "IntegerArray",

pandas/conftest.py

@@ -293,6 +293,20 @@ def compare_operators_no_eq_ne(request):
     return request.param
 
 
+@pytest.fixture(
+    params=["__and__", "__rand__", "__or__", "__ror__", "__xor__", "__rxor__"]
+)
+def all_logical_operators(request):
+    """
+    Fixture for dunder names for common logical operations
+
+    * |
+    * &
+    * ^
+    """
+    return request.param
+
+
 @pytest.fixture(params=[None, "gzip", "bz2", "zip", "xz"])
 def compression(request):
     """

pandas/core/api.py

@@ -12,6 +12,7 @@
 
 from pandas.core.algorithms import factorize, unique, value_counts
 from pandas.core.arrays import Categorical
+from pandas.core.arrays.boolean import BooleanDtype
 from pandas.core.arrays.integer import (
     Int8Dtype,
     Int16Dtype,

pandas/core/arrays/__init__.py

@@ -4,6 +4,7 @@
     ExtensionScalarOpsMixin,
     try_cast_to_ea,
 )
+from .boolean import BooleanArray  # noqa: F401
 from .categorical import Categorical  # noqa: F401
 from .datetimes import DatetimeArray  # noqa: F401
 from .integer import IntegerArray, integer_array  # noqa: F401

pandas/core/arrays/base.py

@@ -1088,6 +1088,15 @@ def _add_comparison_ops(cls):
         cls.__le__ = cls._create_comparison_method(operator.le)
         cls.__ge__ = cls._create_comparison_method(operator.ge)
 
+    @classmethod
+    def _add_logical_ops(cls):
+        cls.__and__ = cls._create_logical_method(operator.and_)
+        cls.__rand__ = cls._create_logical_method(ops.rand_)
+        cls.__or__ = cls._create_logical_method(operator.or_)
+        cls.__ror__ = cls._create_logical_method(ops.ror_)
+        cls.__xor__ = cls._create_logical_method(operator.xor)
+        cls.__rxor__ = cls._create_logical_method(ops.rxor)
+
 
 class ExtensionScalarOpsMixin(ExtensionOpsMixin):
     """