Skip to content

BUG: pivot_table losing tz #32558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 14, 2020
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ Reshaping
- :meth:`Series.append` will now raise a ``TypeError`` when passed a DataFrame or a sequence containing Dataframe (:issue:`31413`)
- :meth:`DataFrame.replace` and :meth:`Series.replace` will raise a ``TypeError`` if ``to_replace`` is not an expected type. Previously the ``replace`` would fail silently (:issue:`18634`)
- Bug in :meth:`DataFrame.apply` where callback was called with :class:`Series` parameter even though ``raw=True`` requested. (:issue:`32423`)
- Bug in :meth:`DataFrame.pivot_table` losing timezone information when creating a :class:`MultiIndex` level from a column with timezone-aware dtype (:issue:`32558`)


Sparse
Expand Down
1 change: 1 addition & 0 deletions pandas/core/indexes/multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -565,6 +565,7 @@ def from_product(cls, iterables, sortorder=None, names=lib.no_default):
if names is lib.no_default:
names = [getattr(it, "name", None) for it in iterables]

# codes are all ndarrays, so cartesian_product is lossless
codes = cartesian_product(codes)
return MultiIndex(levels, codes, sortorder=sortorder, names=names)

Expand Down
25 changes: 17 additions & 8 deletions pandas/core/reshape/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

from pandas.core.dtypes.common import is_list_like

import pandas.core.common as com


def cartesian_product(X):
"""
Expand Down Expand Up @@ -51,9 +49,20 @@ def cartesian_product(X):
# if any factor is empty, the cartesian product is empty
b = np.zeros_like(cumprodX)

return [
np.tile(
np.repeat(np.asarray(com.values_from_object(x)), b[i]), np.product(a[i])
)
for i, x in enumerate(X)
]
return [_tile_compat(np.repeat(x, b[i]), np.product(a[i])) for i, x in enumerate(X)]


def _tile_compat(arr, num: int):
"""
Index compat for np.tile.

Notes
-----
Does not support multi-dimensional `num`.
"""
if isinstance(arr, np.ndarray):
return np.tile(arr, num)

# Otherwise we have an Index
taker = np.tile(np.arange(len(arr)), num)
return arr.take(taker)
8 changes: 8 additions & 0 deletions pandas/tests/reshape/test_pivot.py
Original file line number Diff line number Diff line change
Expand Up @@ -1026,6 +1026,14 @@ def test_pivot_table_multiindex_only(self, cols):

tm.assert_frame_equal(result, expected)

def test_pivot_table_retains_tz(self):
dti = date_range("2016-01-01", periods=3, tz="Europe/Amsterdam")
df = DataFrame({"A": np.random.randn(3), "B": np.random.randn(3), "C": dti})
result = df.pivot_table(index=["B", "C"], dropna=False)

# check tz retention
assert result.index.levels[1].equals(dti)

def test_pivot_integer_columns(self):
# caused by upstream bug in unstack

Expand Down
16 changes: 16 additions & 0 deletions pandas/tests/reshape/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,22 @@ def test_datetimeindex(self):
tm.assert_index_equal(result1, expected1)
tm.assert_index_equal(result2, expected2)

def test_tzaware_retained(self):
x = date_range("2000-01-01", periods=2, tz="US/Pacific")
y = np.array([3, 4])
result1, result2 = cartesian_product([x, y])

expected = x.repeat(2)
tm.assert_index_equal(result1, expected)

def test_tzaware_retained_categorical(self):
x = date_range("2000-01-01", periods=2, tz="US/Pacific").astype("category")
y = np.array([3, 4])
result1, result2 = cartesian_product([x, y])

expected = x.repeat(2)
tm.assert_index_equal(result1, expected)

def test_empty(self):
# product of empty factors
X = [[], [0, 1], []]
Expand Down