BUG: Assorted DatetimeIndex bugfixes

doc/source/whatsnew/v0.24.0.rst

@@ -1139,6 +1139,7 @@ Deprecations
 - :func:`pandas.types.is_datetimetz` is deprecated in favor of `pandas.types.is_datetime64tz` (:issue:`23917`)
 - Creating a :class:`TimedeltaIndex` or :class:`DatetimeIndex` by passing range arguments `start`, `end`, and `periods` is deprecated in favor of :func:`timedelta_range` and :func:`date_range` (:issue:`23919`)
 - Passing a string alias like ``'datetime64[ns, UTC]'`` as the `unit` parameter to :class:`DatetimeTZDtype` is deprecated. Use :class:`DatetimeTZDtype.construct_from_string` instead (:issue:`23990`).
+- Passing a `time_rule` to `pandas.tseries.offsets.generate_range` is deprecated and will raise a ``TypeError`` in a future version.  Pass an ``offset`` instead (:issue:`24157`)
 
 .. _whatsnew_0240.deprecations.datetimelike_int_ops:
 
@@ -1420,6 +1421,7 @@ Indexing
 - Bug in :func:`Index.union` and :func:`Index.intersection` where name of the ``Index`` of the result was not computed correctly for certain cases (:issue:`9943`, :issue:`9862`)
 - Bug in :class:`Index` slicing with boolean :class:`Index` may raise ``TypeError`` (:issue:`22533`)
 - Bug in ``PeriodArray.__setitem__`` when accepting slice and list-like value (:issue:`23978`)
+- Bug in :class:`DatetimeIndex`, :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would lose their ``freq`` attribute (:issue:`21282`)
 
 Missing
 ^^^^^^^

pandas/core/arrays/datetimelike.py

@@ -327,6 +327,10 @@ def __getitem__(self, key):
                              "numpy.newaxis (`None`) and integer or boolean "
                              "arrays are valid indices")
 
+        if key is Ellipsis:
+            # GH#21282
+            return self.copy()
+
         getitem = self._data.__getitem__
         if is_int:
             val = getitem(key)
@@ -547,9 +551,17 @@ def _validate_frequency(cls, index, freq, **kwargs):
         if index.size == 0 or inferred == freq.freqstr:
             return None
 
-        on_freq = cls._generate_range(start=index[0], end=None,
-                                      periods=len(index), freq=freq, **kwargs)
-        if not np.array_equal(index.asi8, on_freq.asi8):
+        try:
+            on_freq = cls._generate_range(start=index[0], end=None,
+                                          periods=len(index), freq=freq,
+                                          **kwargs)
+            assert np.array_equal(index.asi8, on_freq.asi8)
+        except (ValueError, AssertionError) as e:
+            if "non-fixed" in str(e):
+                # non-fixed frequencies are not meaningful for timedelta64;
+                #  we retain that error message
+                raise e
+            # GH#11587 if index[0] is NaT _generate_range will raise ValueError
             raise ValueError('Inferred frequency {infer} from passed values '
                              'does not conform to passed frequency {passed}'
                              .format(infer=inferred, passed=freq.freqstr))

pandas/core/arrays/datetimes.py

@@ -14,9 +14,9 @@
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import (
-    _INT64_DTYPE, _NS_DTYPE, is_datetime64_dtype, is_datetime64tz_dtype,
-    is_extension_type, is_float_dtype, is_int64_dtype, is_object_dtype,
-    is_period_dtype, is_string_dtype, is_timedelta64_dtype)
+    _INT64_DTYPE, _NS_DTYPE, is_categorical_dtype, is_datetime64_dtype,
+    is_datetime64tz_dtype, is_extension_type, is_float_dtype, is_int64_dtype,
+    is_object_dtype, is_period_dtype, is_string_dtype, is_timedelta64_dtype)
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
 from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
 from pandas.core.dtypes.missing import isna
@@ -264,6 +264,8 @@ def _generate_range(cls, start, end, periods, freq, tz=None,
             if closed is not None:
                 raise ValueError("Closed has to be None if not both of start"
                                  "and end are defined")
+        if start is NaT or end is NaT:
+            raise ValueError("Neither `start` nor `end` can be NaT")
 
         left_closed, right_closed = dtl.validate_endpoints(closed)
 
@@ -1647,6 +1649,19 @@ def maybe_convert_dtype(data, copy):
         raise TypeError("Passing PeriodDtype data is invalid.  "
                         "Use `data.to_timestamp()` instead")
 
+    elif is_categorical_dtype(data):
+        # TODO: cases where we need to do another pass through this func,
+        #  e.g. the categories are timedelta64s
+        if isna(data).any():
+            # slow-path
+            data = list(data)
+            data = np.array(data, dtype=np.object_)
+            copy = False
+        else:
+            # TODO: does this always make a copy?  If so, set copy=False
+            # GH#18664 preserve tz in going DTI->Categorical->DTI
+            data = data.categories[data.codes]
+
     elif is_extension_type(data) and not is_datetime64tz_dtype(data):
         # Includes categorical
         # TODO: We have no tests for these

pandas/tests/indexes/datetimes/test_construction.py

@@ -14,12 +14,51 @@
 from pandas import (
     DatetimeIndex, Index, Timestamp, date_range, datetime, offsets,
     to_datetime)
-from pandas.core.arrays import period_array
+from pandas.core.arrays import (
+    DatetimeArrayMixin as DatetimeArray, period_array)
 import pandas.util.testing as tm
 
 
 class TestDatetimeIndex(object):
 
+    @pytest.mark.parametrize('dt_cls', [DatetimeIndex, DatetimeArray])
+    def test_freq_validation_with_nat(self, dt_cls):
+        # GH#11587 make sure we get a useful error message when generate_range
+        #  raises
+        msg = ("Inferred frequency None from passed values does not conform "
+               "to passed frequency D")
+        with pytest.raises(ValueError, match=msg):
+            dt_cls([pd.NaT, pd.Timestamp('2011-01-01')], freq='D')
+        with pytest.raises(ValueError, match=msg):
+            dt_cls([pd.NaT, pd.Timestamp('2011-01-01').value],
+                   freq='D')
+
+    def test_categorical_preserves_tz(self):
+        # GH#18664 retain tz when going DTI-->Categorical-->DTI
+        # TODO: parametrize over DatetimeIndex/DatetimeArray
+        #  once CategoricalIndex(DTA) works
+
+        dti = pd.DatetimeIndex(
+            [pd.NaT, '2015-01-01', '1999-04-06 15:14:13', '2015-01-01'],
+            tz='US/Eastern')
+        ci = pd.CategoricalIndex(dti)
+        carr = pd.Categorical(dti)
+        cser = pd.Series(ci)
+
+        for obj in [ci, carr, cser]:
+            result = pd.DatetimeIndex(obj)
+            tm.assert_index_equal(result, dti)
+
+        # no-NaT case has a fastpath
+        dti2 = dti[1:]
+        ci2 = pd.CategoricalIndex(dti2)
+        carr2 = pd.Categorical(dti2)
+        cser2 = pd.Series(ci2)
+
+        for obj in [ci2, carr2, cser2]:
+            result = pd.DatetimeIndex(obj)
+            tm.assert_index_equal(result, dti2)
+
     def test_dti_with_period_data_raises(self):
         # GH#23675
         data = pd.PeriodIndex(['2016Q1', '2016Q2'], freq='Q')

pandas/tests/indexes/datetimes/test_date_range.py

@@ -80,6 +80,14 @@ def test_date_range_timestamp_equiv_preserve_frequency(self):
 
 
 class TestDateRanges(TestData):
+    def test_date_range_nat(self):
+        # GH#11587
+        msg = "Neither `start` nor `end` can be NaT"
+        with pytest.raises(ValueError, match=msg):
+            date_range(start='2016-01-01', end=pd.NaT, freq='D')
+        with pytest.raises(ValueError, match=msg):
+            date_range(start=pd.NaT, end='2016-01-01', freq='D')
+
     def test_date_range_out_of_bounds(self):
         # GH#14187
         with pytest.raises(OutOfBoundsDatetime):
@@ -533,12 +541,14 @@ class TestGenRangeGeneration(object):
 
     def test_generate(self):
         rng1 = list(generate_range(START, END, offset=BDay()))
-        rng2 = list(generate_range(START, END, time_rule='B'))
+        with tm.assert_produces_warning(FutureWarning):
+            rng2 = list(generate_range(START, END, time_rule='B'))
         assert rng1 == rng2
 
     def test_generate_cday(self):
         rng1 = list(generate_range(START, END, offset=CDay()))
-        rng2 = list(generate_range(START, END, time_rule='C'))
+        with tm.assert_produces_warning(FutureWarning):
+            rng2 = list(generate_range(START, END, time_rule='C'))
         assert rng1 == rng2
 
     def test_1(self):

pandas/tests/indexes/datetimes/test_indexing.py

@@ -16,6 +16,15 @@
 
 
 class TestGetItem(object):
+    def test_ellipsis(self):
+        # GH#21282
+        idx = pd.date_range('2011-01-01', '2011-01-31', freq='D',
+                            tz='Asia/Tokyo', name='idx')
+
+        result = idx[...]
+        assert result.equals(idx)
+        assert result is not idx
+
     def test_getitem(self):
         idx1 = pd.date_range('2011-01-01', '2011-01-31', freq='D', name='idx')
         idx2 = pd.date_range('2011-01-01', '2011-01-31', freq='D',

pandas/tests/indexes/period/test_indexing.py

@@ -13,6 +13,14 @@
 
 
 class TestGetItem(object):
+    def test_ellipsis(self):
+        # GH#21282
+        idx = period_range('2011-01-01', '2011-01-31', freq='D',
+                           name='idx')
+
+        result = idx[...]
+        assert result.equals(idx)
+        assert result is not idx
 
     def test_getitem(self):
         idx1 = pd.period_range('2011-01-01', '2011-01-31', freq='D',

pandas/tests/indexes/timedeltas/test_indexing.py

@@ -9,6 +9,14 @@
 
 
 class TestGetItem(object):
+    def test_ellipsis(self):
+        # GH#21282
+        idx = timedelta_range('1 day', '31 day', freq='D', name='idx')
+
+        result = idx[...]
+        assert result.equals(idx)
+        assert result is not idx
+
     def test_getitem(self):
         idx1 = timedelta_range('1 day', '31 day', freq='D', name='idx')
 

pandas/tests/tseries/offsets/test_offsets.py

@@ -49,21 +49,25 @@ class WeekDay(object):
 ####
 
 
+def test_time_rule_deprecated():
+    start = datetime(2007, 10, 1)
+    end = datetime(2012, 4, 9)
+
+    with tm.assert_produces_warning(FutureWarning):
+        # Note: generate_range returns a generator, and the warning is not
+        #  issued until the first __next__ call
+        list(offsets.generate_range(start=start, end=end, time_rule="W"))
+
+
 def test_to_m8():
     valb = datetime(2007, 10, 1)
     valu = _to_m8(valb)
     assert isinstance(valu, np.datetime64)
-    # assert valu == np.datetime64(datetime(2007,10,1))
 
-    # def test_datetime64_box():
-    #    valu = np.datetime64(datetime(2007,10,1))
-    #    valb = _dt_box(valu)
-    #    assert type(valb) == datetime
-    #    assert valb == datetime(2007,10,1)
 
-    #####
-    # DateOffset Tests
-    #####
+#####
+# DateOffset Tests
+#####
 
 
 class Base(object):

pandas/tseries/offsets.py

@@ -2,6 +2,7 @@
 from datetime import date, datetime, timedelta
 import functools
 import operator
+import warnings
 
 from dateutil.easter import easter
 import numpy as np
@@ -2487,6 +2488,9 @@ def generate_range(start=None, end=None, periods=None,
 
     """
     if time_rule is not None:
+        warnings.warn("`time_rule` is deprecated and will be removed in a "
+                      "future version.  Use `offset` instead.",
+                      FutureWarning, stacklevel=2)
         from pandas.tseries.frequencies import get_offset
 
         offset = get_offset(time_rule)