Skip to content

Address latest pandas-related upstream test failures #9081

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jun 10, 2024
Merged

Address latest pandas-related upstream test failures #9081

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
891fd6e
Address pandas-related upstream test failures
spencerkclark Jun 9, 2024
bd875d3
Address more warnings
spencerkclark Jun 9, 2024
616c179
Don't lose coverage for pandas < 3
spencerkclark Jun 9, 2024
1a3bdf6
Address one more warning
spencerkclark Jun 9, 2024
334d118
Fix accidental change from MS to ME
spencerkclark Jun 9, 2024
36a005a
Use datetime64[ns] arrays
spencerkclark Jun 10, 2024
85c95a1
Switch to @pytest.mark.filterwarnings
spencerkclark Jun 10, 2024
d181b97
Merge branch 'main' into upstream-failures-2024-06-09
spencerkclark Jun 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion xarray/coding/cftime_offsets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,21 @@
from xarray.core.types import InclusiveOptions, SideOptions


def _nanosecond_precision_timestamp(*args, **kwargs):
# As of pandas version 3.0, pd.to_datetime(Timestamp(...)) will try to
# infer the appropriate datetime precision. Until xarray supports
# non-nanosecond precision times, we will use this constructor wrapper to
# explicitly create nanosecond-precision Timestamp objects.
return pd.Timestamp(*args, **kwargs).as_unit("ns")


def get_date_type(calendar, use_cftime=True):
"""Return the cftime date type for a given calendar name."""
if cftime is None:
raise ImportError("cftime is required for dates with non-standard calendars")
else:
if _is_standard_calendar(calendar) and not use_cftime:
return pd.Timestamp
return _nanosecond_precision_timestamp

calendars = {
"noleap": cftime.DatetimeNoLeap,
Expand Down
2 changes: 1 addition & 1 deletion xarray/tests/test_backends.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ def test_roundtrip_string_encoded_characters(self) -> None:
assert actual["x"].encoding["_Encoding"] == "ascii"

def test_roundtrip_numpy_datetime_data(self) -> None:
times = pd.to_datetime(["2000-01-01", "2000-01-02", "NaT"])
times = pd.to_datetime(["2000-01-01", "2000-01-02", "NaT"], unit="ns")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pandas.to_datetime will infer the precision from the input in pandas 3, so we explicitly specify the desired precision now.

expected = Dataset({"t": ("t", times), "t0": times[0]})
kwargs = {"encoding": {"t0": {"units": "days since 1950-01-01"}}}
with self.roundtrip(expected, save_kwargs=kwargs) as actual:
Expand Down
9 changes: 6 additions & 3 deletions xarray/tests/test_coding_times.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,11 +538,14 @@ def test_infer_datetime_units(freq, units) -> None:
["dates", "expected"],
[
(
pd.to_datetime(["1900-01-01", "1900-01-02", "NaT"]),
pd.to_datetime(["1900-01-01", "1900-01-02", "NaT"], unit="ns"),
"days since 1900-01-01 00:00:00",
),
(pd.to_datetime(["NaT", "1900-01-01"]), "days since 1900-01-01 00:00:00"),
(pd.to_datetime(["NaT"]), "days since 1970-01-01 00:00:00"),
(
pd.to_datetime(["NaT", "1900-01-01"], unit="ns"),
"days since 1900-01-01 00:00:00",
),
(pd.to_datetime(["NaT"], unit="ns"), "days since 1970-01-01 00:00:00"),
],
)
def test_infer_datetime_units_with_NaT(dates, expected) -> None:
Expand Down
19 changes: 16 additions & 3 deletions xarray/tests/test_combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from __future__ import annotations

from datetime import datetime
from itertools import product

import numpy as np
Expand Down Expand Up @@ -229,8 +228,22 @@ def test_lexicographic_sort_string_coords(self):
assert concat_dims == ["simulation"]

def test_datetime_coords(self):
ds0 = Dataset({"time": [datetime(2000, 3, 6), datetime(2001, 3, 7)]})
ds1 = Dataset({"time": [datetime(1999, 1, 1), datetime(1999, 2, 4)]})
ds0 = Dataset(
{
"time": [
np.datetime64("2000-03-06", "ns"),
np.datetime64("2000-03-07", "ns"),
]
}
)
ds1 = Dataset(
{
"time": [
np.datetime64("1999-01-01", "ns"),
np.datetime64("1999-02-04", "ns"),
]
}
)

expected = {(0,): ds1, (1,): ds0}
actual, concat_dims = _infer_concat_order_from_coords([ds0, ds1])
Expand Down
8 changes: 6 additions & 2 deletions xarray/tests/test_dataarray.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3647,7 +3647,9 @@ def test_to_and_from_dict_with_time_dim(self) -> None:
t = pd.date_range("20130101", periods=10)
lat = [77.7, 83.2, 76]
da = DataArray(x, {"t": t, "lat": lat}, dims=["t", "lat"])
roundtripped = DataArray.from_dict(da.to_dict())
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Converting non-nanosecond")
roundtripped = DataArray.from_dict(da.to_dict())
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

da.to_dict() produces datetime.datetime objects, which under pandas 3 lead to a conversion warning in the DataArray constructor.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we have this pattern in multiple modules, it might be worth adding the code as a special context manager to xarray.tests.__init__. Something like this might work (I didn't check):

from contextlib import contextmanager
import warnings

@contextmanager
def ignore_warnings(category=None, pattern=None):
    if category is None and pattern is None:
        raise ValueError("need at least one of category and pattern")

    try:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", message=pattern, category=category)
            yield
    finally:
        pass

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks—I ended up switching to marking these tests with @pytest.mark.filterwarnings("ignore:Converting non-nanosecond"), since that is a pattern we use elsewhere in the tests already.

assert_identical(da, roundtripped)

def test_to_and_from_dict_with_nan_nat(self) -> None:
Expand All @@ -3657,7 +3659,9 @@ def test_to_and_from_dict_with_nan_nat(self) -> None:
t[2] = np.nan
lat = [77.7, 83.2, 76]
da = DataArray(y, {"t": t, "lat": lat}, dims=["t", "lat"])
roundtripped = DataArray.from_dict(da.to_dict())
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Converting non-nanosecond")
roundtripped = DataArray.from_dict(da.to_dict())
assert_identical(da, roundtripped)

def test_to_dict_with_numpy_attrs(self) -> None:
Expand Down
5 changes: 2 additions & 3 deletions xarray/tests/test_groupby.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,7 @@ def test_groupby_da_datetime() -> None:
times = pd.date_range("2000-01-01", periods=4)
foo = xr.DataArray([1, 2, 3, 4], coords=dict(time=times), dims="time")
# create test index
dd = times.to_pydatetime()
reference_dates = [dd[0], dd[2]]
reference_dates = [times[0], times[2]]
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I can tell, whether reference_dates started as datetime.datetime objects or np.datetime64[ns] values was not material to this test, so I removed the conversion to datetime.datetime to avoid the conversion warning under pandas 3 (the times would previously get converted back to datetime64[ns] values in the DataArray constructor).

labels = reference_dates[0:1] * 2 + reference_dates[1:2] * 2
ind = xr.DataArray(
labels, coords=dict(time=times), dims="time", name="reference_date"
Expand Down Expand Up @@ -1881,7 +1880,7 @@ def test_resample_first(self) -> None:
array = Dataset({"time": times})["time"]
actual = array.resample(time="1D").last()
expected_times = pd.to_datetime(
["2000-01-01T18", "2000-01-02T18", "2000-01-03T06"]
["2000-01-01T18", "2000-01-02T18", "2000-01-03T06"], unit="ns"
)
expected = DataArray(expected_times, [("time", times[::4])], name="time")
assert_identical(expected, actual)
Expand Down
7 changes: 3 additions & 4 deletions xarray/tests/test_plot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import math
from collections.abc import Generator, Hashable
from copy import copy
from datetime import date, datetime, timedelta
from datetime import date, timedelta
from typing import Any, Callable, Literal

import numpy as np
Expand Down Expand Up @@ -2912,9 +2912,8 @@ def setUp(self) -> None:
"""
month = np.arange(1, 13, 1)
data = np.sin(2 * np.pi * month / 12.0)

darray = DataArray(data, dims=["time"])
darray.coords["time"] = np.array([datetime(2017, m, 1) for m in month])
times = pd.date_range(start="2017-01-01", freq="ME", periods=12)
darray = DataArray(data, dims=["time"], coords=[times])
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use of datetime.datetime objects was immaterial to this test, so we use pd.date_range to produce the dates instead to avoid the non-nanosecond conversion warning.


self.darray = darray

Expand Down
9 changes: 6 additions & 3 deletions xarray/tests/test_variable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
assert_equal,
assert_identical,
assert_no_warnings,
has_pandas_3,
raise_if_dask_computes,
requires_bottleneck,
requires_cupy,
Expand Down Expand Up @@ -256,7 +257,9 @@ def test_index_and_concat_datetime(self):
# regression test for #125
date_range = pd.date_range("2011-09-01", periods=10)
for dates in [date_range, date_range.values, date_range.to_pydatetime()]:
expected = self.cls("t", dates)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="Converting non-nanosecond")
expected = self.cls("t", dates)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is needed since dates sometimes consists of datetime.datetime objects, which leads to a conversion warning under pandas 3.

for times in [
[expected[i] for i in range(10)],
[expected[i : (i + 1)] for i in range(10)],
Expand Down Expand Up @@ -2942,8 +2945,8 @@ def test_from_pint_wrapping_dask(self, Var):
(np.array([np.datetime64("2000-01-01", "ns")]), False),
(np.array([np.datetime64("2000-01-01", "s")]), True),
(pd.date_range("2000", periods=1), False),
(datetime(2000, 1, 1), False),
(np.array([datetime(2000, 1, 1)]), False),
(datetime(2000, 1, 1), has_pandas_3),
(np.array([datetime(2000, 1, 1)]), has_pandas_3),
Comment on lines +2947 to +2948
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With pandas 3, pd.Series(datetime.datetime(...)) will produce a Series with np.datetime64[us] values instead of np.datetime64[ns] values, so this conversion now warns.

(pd.date_range("2000", periods=1, tz=pytz.timezone("America/New_York")), False),
(
pd.Series(
Expand Down
Loading