Merge remote-tracking branch 'upstream/master' into typing

Author: simonjayhawkins

.github/FUNDING.yml

@@ -1,2 +1,3 @@
 custom: https://pandas.pydata.org/donate.html
+github: [numfocus]
 tidelift: pypi/pandas

asv_bench/benchmarks/gil.py

@@ -37,7 +37,7 @@ def wrapper(fname):
         return wrapper
 
 
-from .pandas_vb_common import BaseIO  # noqa: E402 isort:skip
+from .pandas_vb_common import BaseIO  # isort:skip
 
 
 class ParallelGroupbyMethods:

asv_bench/benchmarks/offset.py

@@ -3,7 +3,7 @@
 import pandas as pd
 
 try:
-    import pandas.tseries.holiday  # noqa
+    import pandas.tseries.holiday
 except ImportError:
     pass
 

doc/source/whatsnew/v1.0.0.rst

@@ -314,7 +314,7 @@ Datetimelike
 - Bug in :func:`pandas.core.groupby.generic.SeriesGroupBy.apply` raising ``ValueError`` when a column in the original DataFrame is a datetime and the column labels are not standard integers (:issue:`28247`)
 - Bug in :func:`pandas._config.localization.get_locales` where the ``locales -a`` encodes the locales list as windows-1252 (:issue:`23638`, :issue:`24760`, :issue:`27368`)
 - Bug in :meth:`Series.var` failing to raise ``TypeError`` when called with ``timedelta64[ns]`` dtype (:issue:`28289`)
--
+- Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`)
 
 Timedelta
 ^^^^^^^^^

pandas/_libs/parsers.pyx

@@ -843,11 +843,6 @@ cdef class TextReader:
 
             passed_count = len(header[0])
 
-            # if passed_count > field_count:
-            #     raise ParserError('Column names have %d fields, '
-            #                        'data has %d fields'
-            #                        % (passed_count, field_count))
-
             if (self.has_usecols and self.allow_leading_cols and
                     not callable(self.usecols)):
                 nuse = len(self.usecols)

pandas/_libs/reduction.pyx

@@ -82,11 +82,9 @@ cdef class Reducer:
         else:
 
             # we passed a series-like
-            if hasattr(dummy, 'values'):
-
-                typ = type(dummy)
-                index = getattr(dummy, 'index', None)
-                dummy = dummy.values
+            typ = type(dummy)
+            index = dummy.index
+            dummy = dummy.values
 
             if dummy.dtype != self.arr.dtype:
                 raise ValueError('Dummy array must be same dtype')
@@ -99,10 +97,10 @@ cdef class Reducer:
         cdef:
             char* dummy_buf
             ndarray arr, result, chunk
-            Py_ssize_t i, incr
+            Py_ssize_t i
             flatiter it
             bint has_labels
-            object res, name, labels, index
+            object res, name, labels
             object cached_typ = None
 
         arr = self.arr
@@ -112,7 +110,6 @@ cdef class Reducer:
         labels = self.labels
         has_labels = labels is not None
         has_index = self.index is not None
-        incr = self.increment
 
         result = np.empty(self.nresults, dtype='O')
         it = <flatiter>PyArray_IterNew(result)
@@ -193,10 +190,10 @@ cdef class _BaseGrouper:
         return values, index
 
     cdef inline _update_cached_objs(self, object cached_typ, object cached_ityp,
-                                    Slider islider, Slider vslider, object name):
+                                    Slider islider, Slider vslider):
         if cached_typ is None:
             cached_ityp = self.ityp(islider.buf)
-            cached_typ = self.typ(vslider.buf, index=cached_ityp, name=name)
+            cached_typ = self.typ(vslider.buf, index=cached_ityp, name=self.name)
         else:
             # See the comment in indexes/base.py about _index_data.
             # We need this for EA-backed indexes that have a reference
@@ -205,7 +202,7 @@ cdef class _BaseGrouper:
             cached_ityp._engine.clear_mapping()
             object.__setattr__(cached_typ._data._block, 'values', vslider.buf)
             object.__setattr__(cached_typ, '_index', cached_ityp)
-            object.__setattr__(cached_typ, 'name', name)
+            object.__setattr__(cached_typ, 'name', self.name)
 
         return cached_typ, cached_ityp
 
@@ -224,6 +221,7 @@ cdef class SeriesBinGrouper(_BaseGrouper):
     def __init__(self, object series, object f, object bins, object dummy):
 
         assert dummy is not None  # always obj[:0]
+        assert len(bins) > 0  # otherwise we get IndexError in get_result
 
         self.bins = bins
         self.f = f
@@ -254,7 +252,7 @@ cdef class SeriesBinGrouper(_BaseGrouper):
             object res
             bint initialized = 0
             Slider vslider, islider
-            object name, cached_typ = None, cached_ityp = None
+            object cached_typ = None, cached_ityp = None
 
         counts = np.zeros(self.ngroups, dtype=np.int64)
 
@@ -268,7 +266,6 @@ cdef class SeriesBinGrouper(_BaseGrouper):
 
         group_size = 0
         n = len(self.arr)
-        name = self.name
 
         vslider = Slider(self.arr, self.dummy_arr)
         islider = Slider(self.index, self.dummy_index)
@@ -283,7 +280,7 @@ cdef class SeriesBinGrouper(_BaseGrouper):
                 vslider.set_length(group_size)
 
                 cached_typ, cached_ityp = self._update_cached_objs(
-                    cached_typ, cached_ityp, islider, vslider, name)
+                    cached_typ, cached_ityp, islider, vslider)
 
                 cached_ityp._engine.clear_mapping()
                 res = self.f(cached_typ)
@@ -356,13 +353,12 @@ cdef class SeriesGrouper(_BaseGrouper):
             object res
             bint initialized = 0
             Slider vslider, islider
-            object name, cached_typ = None, cached_ityp = None
+            object cached_typ = None, cached_ityp = None
 
         labels = self.labels
         counts = np.zeros(self.ngroups, dtype=np.int64)
         group_size = 0
         n = len(self.arr)
-        name = self.name
 
         vslider = Slider(self.arr, self.dummy_arr)
         islider = Slider(self.index, self.dummy_index)
@@ -386,7 +382,7 @@ cdef class SeriesGrouper(_BaseGrouper):
                     vslider.set_length(group_size)
 
                     cached_typ, cached_ityp = self._update_cached_objs(
-                        cached_typ, cached_ityp, islider, vslider, name)
+                        cached_typ, cached_ityp, islider, vslider)
 
                     cached_ityp._engine.clear_mapping()
                     res = self.f(cached_typ)

pandas/_libs/src/parser/tokenizer.h

@@ -162,9 +162,9 @@ typedef struct parser_t {
     int64_t skip_footer;
     // pick one, depending on whether the converter requires GIL
     double (*double_converter_nogil)(const char *, char **,
-                                     char, char, char, int);
+                                     char, char, char, int, int *, int *);
     double (*double_converter_withgil)(const char *, char **,
-                                       char, char, char, int);
+                                       char, char, char, int, int *, int *);
 
     // error handling
     char *warn_msg;

pandas/_libs/tslibs/c_timestamp.pyx

@@ -201,7 +201,7 @@ cdef class _Timestamp(datetime):
         """
         return np.datetime64(self.value, 'ns')
 
-    def to_numpy(self, dtype=None, copy=False):
+    def to_numpy(self, dtype=None, copy=False) -> np.datetime64:
         """
         Convert the Timestamp to a NumPy datetime64.
 
@@ -369,18 +369,18 @@ cdef class _Timestamp(datetime):
         return out[0]
 
     @property
-    def _repr_base(self):
+    def _repr_base(self) -> str:
         return '{date} {time}'.format(date=self._date_repr,
                                       time=self._time_repr)
 
     @property
-    def _date_repr(self):
+    def _date_repr(self) -> str:
         # Ideal here would be self.strftime("%Y-%m-%d"), but
         # the datetime strftime() methods require year >= 1900
         return '%d-%.2d-%.2d' % (self.year, self.month, self.day)
 
     @property
-    def _time_repr(self):
+    def _time_repr(self) -> str:
         result = '%.2d:%.2d:%.2d' % (self.hour, self.minute, self.second)
 
         if self.nanosecond != 0:
@@ -391,7 +391,7 @@ cdef class _Timestamp(datetime):
         return result
 
     @property
-    def _short_repr(self):
+    def _short_repr(self) -> str:
         # format a Timestamp with only _date_repr if possible
         # otherwise _repr_base
         if (self.hour == 0 and
@@ -403,7 +403,7 @@ cdef class _Timestamp(datetime):
         return self._repr_base
 
     @property
-    def asm8(self):
+    def asm8(self) -> np.datetime64:
         """
         Return numpy datetime64 format in nanoseconds.
         """

pandas/_libs/tslibs/nattype.pyx

@@ -230,16 +230,16 @@ cdef class _NaT(datetime):
         return NotImplemented
 
     @property
-    def asm8(self):
+    def asm8(self) -> np.datetime64:
         return np.datetime64(NPY_NAT, 'ns')
 
-    def to_datetime64(self):
+    def to_datetime64(self) -> np.datetime64:
         """
         Return a numpy.datetime64 object with 'ns' precision.
         """
         return np.datetime64('NaT', 'ns')
 
-    def to_numpy(self, dtype=None, copy=False):
+    def to_numpy(self, dtype=None, copy=False) -> np.datetime64:
         """
         Convert the Timestamp to a NumPy datetime64.
 
@@ -265,7 +265,7 @@ cdef class _NaT(datetime):
     def __str__(self) -> str:
         return 'NaT'
 
-    def isoformat(self, sep='T'):
+    def isoformat(self, sep='T') -> str:
         # This allows Timestamp(ts.isoformat()) to always correctly roundtrip.
         return 'NaT'
 

pandas/_libs/tslibs/timedeltas.pyx

@@ -841,15 +841,15 @@ cdef class _Timedelta(timedelta):
         """
         return timedelta(microseconds=int(self.value) / 1000)
 
-    def to_timedelta64(self):
+    def to_timedelta64(self) -> np.timedelta64:
         """
         Return a numpy.timedelta64 object with 'ns' precision.
         """
         return np.timedelta64(self.value, 'ns')
 
-    def to_numpy(self, dtype=None, copy=False):
+    def to_numpy(self, dtype=None, copy=False) -> np.timedelta64:
         """
-        Convert the Timestamp to a NumPy timedelta64.
+        Convert the Timedelta to a NumPy timedelta64.
 
         .. versionadded:: 0.25.0
 
@@ -920,7 +920,7 @@ cdef class _Timedelta(timedelta):
         return self.value
 
     @property
-    def asm8(self):
+    def asm8(self) -> np.timedelta64:
         """
         Return a numpy timedelta64 array scalar view.
 
@@ -955,7 +955,7 @@ cdef class _Timedelta(timedelta):
         return np.int64(self.value).view('m8[ns]')
 
     @property
-    def resolution_string(self):
+    def resolution_string(self) -> str:
         """
         Return a string representing the lowest timedelta resolution.
 
@@ -1095,7 +1095,7 @@ cdef class _Timedelta(timedelta):
         self._ensure_components()
         return self._ns
 
-    def _repr_base(self, format=None):
+    def _repr_base(self, format=None) -> str:
         """
 
         Parameters
@@ -1148,10 +1148,10 @@ cdef class _Timedelta(timedelta):
     def __str__(self) -> str:
         return self._repr_base(format='long')
 
-    def __bool__(self):
+    def __bool__(self) -> bool:
         return self.value != 0
 
-    def isoformat(self):
+    def isoformat(self) -> str:
         """
         Format Timedelta as ISO 8601 Duration like
         ``P[n]Y[n]M[n]DT[n]H[n]M[n]S``, where the ``[n]`` s are replaced by the

pandas/_libs/tslibs/timestamps.pyx

@@ -36,7 +36,6 @@ from pandas._libs.tslibs.tzconversion import (
 # Constants
 _zero_time = datetime_time(0, 0)
 _no_input = object()
-PY36 = sys.version_info >= (3, 6)
 
 # ----------------------------------------------------------------------
 
@@ -982,9 +981,8 @@ default 'raise'
         else:
             kwargs = {'year': dts.year, 'month': dts.month, 'day': dts.day,
                       'hour': dts.hour, 'minute': dts.min, 'second': dts.sec,
-                      'microsecond': dts.us, 'tzinfo': _tzinfo}
-            if PY36:
-                kwargs['fold'] = fold
+                      'microsecond': dts.us, 'tzinfo': _tzinfo,
+                      'fold': fold}
             ts_input = datetime(**kwargs)
 
         ts = convert_datetime_to_tsobject(ts_input, _tzinfo)

pandas/compat/__init__.py

@@ -12,7 +12,6 @@
 import sys
 import warnings
 
-PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
 PY38 = sys.version_info >= (3, 8)
 PYPY = platform.python_implementation() == "PyPy"