Merge remote-tracking branch 'upstream/master' into datetime-type-inference

Author: dsaxton

pandas/_libs/lib.pyx

@@ -67,7 +67,11 @@ cimport pandas._libs.util as util
 from pandas._libs.util cimport is_nan, UINT64_MAX, INT64_MAX, INT64_MIN
 
 from pandas._libs.tslib import array_to_datetime
-from pandas._libs.tslibs.nattype cimport NPY_NAT, c_NaT as NaT
+from pandas._libs.tslibs.nattype cimport (
+    NPY_NAT,
+    c_NaT as NaT,
+    checknull_with_nat,
+)
 from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
 from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
@@ -77,7 +81,6 @@ from pandas._libs.missing cimport (
     isnaobj,
     is_null_datetime64,
     is_null_timedelta64,
-    is_null_period,
     C_NA,
 )
 
@@ -1846,7 +1849,7 @@ cdef class PeriodValidator(TemporalValidator):
         return util.is_period_object(value)
 
     cdef inline bint is_valid_null(self, object value) except -1:
-        return is_null_period(value)
+        return checknull_with_nat(value)
 
 
 cpdef bint is_period_array(ndarray values):

pandas/_libs/missing.pxd

@@ -6,7 +6,6 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr)
 
 cdef bint is_null_datetime64(v)
 cdef bint is_null_timedelta64(v)
-cdef bint is_null_period(v)
 
 cdef class C_NAType:
     pass

pandas/_libs/missing.pyx

@@ -40,6 +40,7 @@ cpdef bint checknull(object val):
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
 
     Parameters
     ----------
@@ -278,12 +279,6 @@ cdef inline bint is_null_timedelta64(v):
     return False
 
 
-cdef inline bint is_null_period(v):
-    # determine if we have a null for a Period (or integer versions),
-    # excluding np.datetime64('nat') and np.timedelta64('nat')
-    return checknull_with_nat(v)
-
-
 # -----------------------------------------------------------------------------
 # Implementation of NA singleton
 

pandas/tests/groupby/test_nth.py

@@ -140,6 +140,18 @@ def test_first_last_nth_dtypes(df_mixed_floats):
     assert f.dtype == "int64"
 
 
+def test_first_last_nth_nan_dtype():
+    # GH 33591
+    df = pd.DataFrame({"data": ["A"], "nans": pd.Series([np.nan], dtype=object)})
+
+    grouped = df.groupby("data")
+    expected = df.set_index("data").nans
+    tm.assert_series_equal(grouped.nans.first(), expected)
+    tm.assert_series_equal(grouped.nans.last(), expected)
+    tm.assert_series_equal(grouped.nans.nth(-1), expected)
+    tm.assert_series_equal(grouped.nans.nth(0), expected)
+
+
 def test_first_strings_timestamps():
     # GH 11244
     test = pd.DataFrame(

pandas/tests/indexes/common.py

@@ -393,6 +393,9 @@ def test_numpy_repeat(self):
     @pytest.mark.parametrize("klass", [list, tuple, np.array, Series])
     def test_where(self, klass):
         i = self.create_index()
+        if isinstance(i, (pd.DatetimeIndex, pd.TimedeltaIndex)):
+            # where does not preserve freq
+            i._set_freq(None)
 
         cond = [True] * len(i)
         result = i.where(klass(cond))

pandas/tests/indexes/datetimelike.py

@@ -81,8 +81,8 @@ def test_map_dictlike(self, mapper):
         expected = index + index.freq
 
         # don't compare the freqs
-        if isinstance(expected, pd.DatetimeIndex):
-            expected._data.freq = None
+        if isinstance(expected, (pd.DatetimeIndex, pd.TimedeltaIndex)):
+            expected._set_freq(None)
 
         result = index.map(mapper(expected, index))
         tm.assert_index_equal(result, expected)

pandas/tests/indexes/datetimes/test_astype.py

@@ -88,6 +88,7 @@ def test_astype_with_tz(self):
         result = idx.astype("datetime64[ns, US/Eastern]")
         expected = date_range("20170101 03:00:00", periods=4, tz="US/Eastern")
         tm.assert_index_equal(result, expected)
+        assert result.freq == expected.freq
 
         # GH 18951: tz-naive to tz-aware
         idx = date_range("20170101", periods=4)

pandas/tests/indexes/datetimes/test_constructors.py

@@ -131,6 +131,7 @@ def test_construction_with_alt(self, kwargs, tz_aware_fixture):
     def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture):
         tz = tz_aware_fixture
         i = pd.date_range("20130101", periods=5, freq="H", tz=tz)
+        i._set_freq(None)
         kwargs = {key: attrgetter(val)(i) for key, val in kwargs.items()}
 
         if "tz" in kwargs:
@@ -703,7 +704,9 @@ def test_constructor_start_end_with_tz(self, tz):
         end = Timestamp("2013-01-02 06:00:00", tz="America/Los_Angeles")
         result = date_range(freq="D", start=start, end=end, tz=tz)
         expected = DatetimeIndex(
-            ["2013-01-01 06:00:00", "2013-01-02 06:00:00"], tz="America/Los_Angeles"
+            ["2013-01-01 06:00:00", "2013-01-02 06:00:00"],
+            tz="America/Los_Angeles",
+            freq="D",
         )
         tm.assert_index_equal(result, expected)
         # Especially assert that the timezone is consistent for pytz

pandas/tests/indexes/datetimes/test_date_range.py

@@ -218,7 +218,7 @@ def test_date_range_normalize(self):
         rng = date_range(snap, periods=n, normalize=False, freq="2D")
 
         offset = timedelta(2)
-        values = DatetimeIndex([snap + i * offset for i in range(n)])
+        values = DatetimeIndex([snap + i * offset for i in range(n)], freq=offset)
 
         tm.assert_index_equal(rng, values)
 
@@ -413,7 +413,7 @@ def test_construct_over_dst(self):
             pre_dst,
             pst_dst,
         ]
-        expected = DatetimeIndex(expect_data)
+        expected = DatetimeIndex(expect_data, freq="H")
         result = date_range(start="2010-11-7", periods=3, freq="H", tz="US/Pacific")
         tm.assert_index_equal(result, expected)
 
@@ -427,7 +427,8 @@ def test_construct_with_different_start_end_string_format(self):
                 Timestamp("2013-01-01 00:00:00+09:00"),
                 Timestamp("2013-01-01 01:00:00+09:00"),
                 Timestamp("2013-01-01 02:00:00+09:00"),
-            ]
+            ],
+            freq="H",
         )
         tm.assert_index_equal(result, expected)
 
@@ -442,7 +443,7 @@ def test_range_bug(self):
         result = date_range("2011-1-1", "2012-1-31", freq=offset)
 
         start = datetime(2011, 1, 1)
-        expected = DatetimeIndex([start + i * offset for i in range(5)])
+        expected = DatetimeIndex([start + i * offset for i in range(5)], freq=offset)
         tm.assert_index_equal(result, expected)
 
     def test_range_tz_pytz(self):
@@ -861,6 +862,7 @@ def test_bdays_and_open_boundaries(self, closed):
         bday_end = "2018-07-27"  # Friday
         expected = pd.date_range(bday_start, bday_end, freq="D")
         tm.assert_index_equal(result, expected)
+        # Note: we do _not_ expect the freqs to match here
 
     def test_bday_near_overflow(self):
         # GH#24252 avoid doing unnecessary addition that _would_ overflow
@@ -910,15 +912,19 @@ def test_daterange_bug_456(self):
 
     def test_cdaterange(self):
         result = bdate_range("2013-05-01", periods=3, freq="C")
-        expected = DatetimeIndex(["2013-05-01", "2013-05-02", "2013-05-03"])
+        expected = DatetimeIndex(["2013-05-01", "2013-05-02", "2013-05-03"], freq="C")
         tm.assert_index_equal(result, expected)
+        assert result.freq == expected.freq
 
     def test_cdaterange_weekmask(self):
         result = bdate_range(
             "2013-05-01", periods=3, freq="C", weekmask="Sun Mon Tue Wed Thu"
         )
-        expected = DatetimeIndex(["2013-05-01", "2013-05-02", "2013-05-05"])
+        expected = DatetimeIndex(
+            ["2013-05-01", "2013-05-02", "2013-05-05"], freq=result.freq
+        )
         tm.assert_index_equal(result, expected)
+        assert result.freq == expected.freq
 
         # raise with non-custom freq
         msg = (
@@ -930,8 +936,11 @@ def test_cdaterange_weekmask(self):
 
     def test_cdaterange_holidays(self):
         result = bdate_range("2013-05-01", periods=3, freq="C", holidays=["2013-05-01"])
-        expected = DatetimeIndex(["2013-05-02", "2013-05-03", "2013-05-06"])
+        expected = DatetimeIndex(
+            ["2013-05-02", "2013-05-03", "2013-05-06"], freq=result.freq
+        )
         tm.assert_index_equal(result, expected)
+        assert result.freq == expected.freq
 
         # raise with non-custom freq
         msg = (
@@ -949,8 +958,11 @@ def test_cdaterange_weekmask_and_holidays(self):
             weekmask="Sun Mon Tue Wed Thu",
             holidays=["2013-05-01"],
         )
-        expected = DatetimeIndex(["2013-05-02", "2013-05-05", "2013-05-06"])
+        expected = DatetimeIndex(
+            ["2013-05-02", "2013-05-05", "2013-05-06"], freq=result.freq
+        )
         tm.assert_index_equal(result, expected)
+        assert result.freq == expected.freq
 
         # raise with non-custom freq
         msg = (

pandas/tests/indexes/period/test_astype.py

@@ -143,18 +143,24 @@ def test_astype_array_fallback(self):
     def test_period_astype_to_timestamp(self):
         pi = PeriodIndex(["2011-01", "2011-02", "2011-03"], freq="M")
 
-        exp = DatetimeIndex(["2011-01-01", "2011-02-01", "2011-03-01"])
-        tm.assert_index_equal(pi.astype("datetime64[ns]"), exp)
+        exp = DatetimeIndex(["2011-01-01", "2011-02-01", "2011-03-01"], freq="MS")
+        res = pi.astype("datetime64[ns]")
+        tm.assert_index_equal(res, exp)
+        assert res.freq == exp.freq
 
         exp = DatetimeIndex(["2011-01-31", "2011-02-28", "2011-03-31"])
         exp = exp + Timedelta(1, "D") - Timedelta(1, "ns")
-        tm.assert_index_equal(pi.astype("datetime64[ns]", how="end"), exp)
+        res = pi.astype("datetime64[ns]", how="end")
+        tm.assert_index_equal(res, exp)
+        assert res.freq == exp.freq
 
         exp = DatetimeIndex(["2011-01-01", "2011-02-01", "2011-03-01"], tz="US/Eastern")
         res = pi.astype("datetime64[ns, US/Eastern]")
-        tm.assert_index_equal(pi.astype("datetime64[ns, US/Eastern]"), exp)
+        tm.assert_index_equal(res, exp)
+        assert res.freq == exp.freq
 
         exp = DatetimeIndex(["2011-01-31", "2011-02-28", "2011-03-31"], tz="US/Eastern")
         exp = exp + Timedelta(1, "D") - Timedelta(1, "ns")
         res = pi.astype("datetime64[ns, US/Eastern]", how="end")
         tm.assert_index_equal(res, exp)
+        assert res.freq == exp.freq

pandas/tests/indexes/period/test_to_timestamp.py

@@ -60,8 +60,9 @@ def test_to_timestamp_quarterly_bug(self):
         pindex = PeriodIndex(year=years, quarter=quarters)
 
         stamps = pindex.to_timestamp("D", "end")
-        expected = DatetimeIndex([x.to_timestamp("D", "end") for x in pindex], freq="Q")
+        expected = DatetimeIndex([x.to_timestamp("D", "end") for x in pindex])
         tm.assert_index_equal(stamps, expected)
+        assert stamps.freq == expected.freq
 
     def test_to_timestamp_pi_mult(self):
         idx = PeriodIndex(["2011-01", "NaT", "2011-02"], freq="2M", name="idx")