Skip to content

BUG: Series.any() and .all() don't return bool values if dtype=object #30416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

BUG: Series.any() and .all() don't return bool values if dtype=object #30416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 42 additions & 4 deletions pandas/core/nanops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,8 +421,27 @@ def nanany(values, axis=None, skipna: bool = True, mask=None):
>>> nanops.nanany(s)
False
"""
values, _, _, _, _ = _get_values(values, skipna, fill_value=False, mask=mask)
return values.any(axis)
values, _, dtype, _, _ = _get_values(values, skipna, fill_value=False, mask=mask)

# GH #12863
# Checking if the `axis` is None because numpy
# doesn't handle ``any`` and ``all`` on
# object arrays correctly. see
# https://github.com/numpy/numpy/issues/4352

# TODO: Find a less code-smelly way of doing this
if is_object_dtype(dtype) and axis is None:
output = values.any()
else:
output = values.any(axis)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can just return directly from here.


if isinstance(output, bool):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when is this NOT true?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's exactly the bug, for example (from the original issue).

master:

>>> import pandas as pd
>>> s = pd.Series(index=range(5), data=['a', 'b', 'c', 'd', 'e'], dtype=object)
>>> s.any()
'a'
>>> s.all()
'e'

This branch (with brute force function):

>>> import pandas as pd
>>> s = pd.Series(index=range(5), data=['a', 'b', 'c', 'd', 'e'], dtype=object)
>>> s.any()
True
>>> s.all()
True

return output

try:
return any(values)
except ValueError:
return values.any()


def nanall(values, axis=None, skipna: bool = True, mask=None):
Expand Down Expand Up @@ -453,8 +472,27 @@ def nanall(values, axis=None, skipna: bool = True, mask=None):
>>> nanops.nanall(s)
False
"""
values, _, _, _, _ = _get_values(values, skipna, fill_value=True, mask=mask)
return values.all(axis)
values, _, dtype, _, _ = _get_values(values, skipna, fill_value=True, mask=mask)

# GH #12863
# Checking if the `axis` is None because numpy
# doesn't handle ``any`` and ``all`` on
# object arrays correctly. see
# https://github.com/numpy/numpy/issues/4352

# TODO: Find a less code-smelly way of doing this
if is_object_dtype(dtype) and axis is None:
output = values.all()
else:
output = values.all(axis)

if isinstance(output, bool):
return output

try:
return all(values)
except ValueError:
return values.all()


@disallow("M8")
Expand Down
29 changes: 23 additions & 6 deletions pandas/tests/reductions/test_reductions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,19 +811,24 @@ def test_all_any(self):
assert not bool_series.all()
assert bool_series.any()

# Alternative types, with implicit 'object' dtype.
s = Series(["abc", True])
assert "abc" == s.any() # 'abc' || True => 'abc'

def test_all_any_params(self):
# Check skipna, with implicit 'object' dtype.
s1 = Series([np.nan, True])
s2 = Series([np.nan, False])
assert s1.all(skipna=False) # nan && True => True

# GH #12863
assert s1.all(skipna=True)
assert np.isnan(s2.any(skipna=False)) # nan || False => nan
assert s1.any(skipna=True)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pls add the GH ref here too


assert s1.all(skipna=False)
assert s1.any(skipna=False)

assert not s2.all(skipna=True)
assert not s2.any(skipna=True)

assert not s2.all(skipna=False)
assert s2.any(skipna=False)

# Check level.
s = pd.Series([False, False, True, True, False, True], index=[0, 0, 1, 1, 2, 2])
tm.assert_series_equal(s.all(level=0), Series([False, True, False]))
Expand All @@ -841,6 +846,18 @@ def test_all_any_params(self):
with pytest.raises(NotImplementedError):
s.all(bool_only=True)

def test_all_any_object_dtype(self):
# GH 12863

s1 = Series(["abc", True])
s2 = Series(["abc", False])

assert s1.all()
assert s1.any()

assert not s2.all()
assert s2.any()

def test_timedelta64_analytics(self):

# index min/max
Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/series/test_apply.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,8 @@ def test_non_callable_aggregates(self):
("sum", "abc"),
("max", "c"),
("min", "a"),
("all", "c"), # see GH12863
("any", "a"),
("all", True),
("any", True),
],
),
),
Expand Down
4 changes: 4 additions & 0 deletions pandas/tests/test_nanops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,10 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs):
value = value.astype("f8")
return func(value, **kwargs)

# GH #12863
# Disabled until https://github.com/numpy/numpy/issues/4352
# is fixed
@pytest.mark.xfail(reason="disabled")
@pytest.mark.parametrize(
"nan_op,np_op", [(nanops.nanany, np.any), (nanops.nanall, np.all)]
)
Expand Down