Skip to content

COMPAT: np 1.18 wants explicit dtype=object) #30035

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 11 commits into from
Closed

COMPAT: np 1.18 wants explicit dtype=object) #30035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bd1b261
COMPAT: np 1.18 wants explicit dtype=object)
jbrockmendel Dec 4, 2019
af573eb
suppress more
jbrockmendel Dec 4, 2019
fa5df88
Merge branch 'master' of https://github.com/pandas-dev/pandas into ci…
jbrockmendel Dec 4, 2019
1c6c7ff
reenable npdev build
jbrockmendel Dec 4, 2019
8f10b6a
Merge branch 'master' of https://github.com/pandas-dev/pandas into ci…
jbrockmendel Dec 4, 2019
4be57a6
Merge branch 'master' of https://github.com/pandas-dev/pandas into ci…
jbrockmendel Dec 8, 2019
77c592f
Merge branch 'master' of https://github.com/pandas-dev/pandas into ci…
jbrockmendel Dec 16, 2019
1ba69b1
fix merge mixup
jbrockmendel Dec 16, 2019
5ebff2e
post-merge fixups
jbrockmendel Dec 16, 2019
52ae9df
Merge branch 'master' of https://github.com/pandas-dev/pandas into ci…
jbrockmendel Dec 17, 2019
3584dab
revert kludges
jbrockmendel Dec 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions pandas/core/internals/blocks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1764,6 +1764,9 @@ def get_values(self, dtype=None):
return values

def to_dense(self):
if self.dtype.kind == "O":
# See https://github.com/numpy/numpy/issues/15041
return np.asarray(self.values, dtype=object)
return np.asarray(self.values)
Comment on lines +1767 to 1770
Copy link
Contributor

@eric-wieser eric-wieser Dec 4, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this is documented as being 1d only, this should be:

x = np.empty(len(self.values), dtype=self.dtype)
x[:] = self.values
return x

which will work correctly even if values == [[1, 2], [3, 4]]


def to_native_types(self, slicer=None, na_rep="nan", quoting=None, **kwargs):
Expand Down
5 changes: 5 additions & 0 deletions pandas/io/formats/format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1485,6 +1485,11 @@ def _format_strings(self) -> List[str]:
if is_categorical_dtype(values.dtype):
# Categorical is special for now, so that we can preserve tzinfo
array = values._internal_get_values()
elif values.dtype.kind == "O":
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

prefer to use is_object_dtype

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i didnt check whether that works on JSONArray, just that values.dtype == object doesnt

# numpy>=1.18 wants object dtype passed explicitly
# Note: dtype.kind check works for json extension tests, while
# dtype == object check does not.
array = np.asarray(values, dtype=object)
else:
array = np.asarray(values)

Expand Down
6 changes: 5 additions & 1 deletion pandas/tests/extension/base/interface.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@ def test_memory_usage(self, data):
assert result == s.nbytes

def test_array_interface(self, data):
result = np.array(data)
if hasattr(data, "dtype") and data.dtype.kind == "O":
# e.g. JSONArray
result = np.array(data, dtype=object)
else:
result = np.array(data)
assert result[0] == data[0]

result = np.array(data, dtype=object)
Expand Down
10 changes: 6 additions & 4 deletions pandas/tests/extension/test_numpy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def data_missing(allow_in_pandas, dtype):
if dtype.numpy_dtype == "object":
if _np_version_under1p16:
raise pytest.skip("Skipping for NumPy <1.16")
return PandasArray(np.array([np.nan, (1,)]))
return PandasArray(np.array([np.nan, (1,)], dtype=object))
return PandasArray(np.array([np.nan, 1.0]))


Expand All @@ -78,7 +78,7 @@ def data_for_sorting(allow_in_pandas, dtype):
if dtype.numpy_dtype == "object":
# Use an empty tuple for first element, then remove,
# to disable np.array's shape inference.
return PandasArray(np.array([(), (2,), (3,), (1,)])[1:])
return PandasArray(np.array([(), (2,), (3,), (1,)], dtype=object)[1:])
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TomAugspurger does the comment above about shape inference have any bearing on the inference here?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think using object dtype is still correct, and I think the comment is still correct.

If we want to get rid of the empty tuple / shape stuff, we could probably allocate the array with the right shape and set the values later

In [6]: a = np.empty((3,), dtype=object)

In [7]: a[:] = (2,), (3,), (1,)

return PandasArray(np.array([1, 2, 0]))


Expand All @@ -90,7 +90,7 @@ def data_missing_for_sorting(allow_in_pandas, dtype):
A < B and NA missing.
"""
if dtype.numpy_dtype == "object":
return PandasArray(np.array([(1,), np.nan, (0,)]))
return PandasArray(np.array([(1,), np.nan, (0,)], dtype=object))
return PandasArray(np.array([1, np.nan, 0]))


Expand All @@ -106,7 +106,9 @@ def data_for_grouping(allow_in_pandas, dtype):
a, b, c = (1,), (2,), (3,)
else:
a, b, c = np.arange(3)
return PandasArray(np.array([b, b, np.nan, np.nan, a, a, b, c]))
return PandasArray(
np.array([b, b, np.nan, np.nan, a, a, b, c], dtype=dtype.numpy_dtype)
)


@pytest.fixture
Expand Down