Merge branch 'master' of github.com:pandas-dev/pandas into document-mask-indexing

Author: psteinb

asv_bench/benchmarks/indexing.py

@@ -158,9 +158,9 @@ def time_boolean_rows_boolean(self):
 class DataFrameNumericIndexing:
     def setup(self):
         self.idx_dupe = np.array(range(30)) * 99
-        self.df = DataFrame(np.random.randn(10000, 5))
+        self.df = DataFrame(np.random.randn(100000, 5))
         self.df_dup = concat([self.df, 2 * self.df, 3 * self.df])
-        self.bool_indexer = [True] * 5000 + [False] * 5000
+        self.bool_indexer = [True] * 50000 + [False] * 50000
 
     def time_iloc_dups(self):
         self.df_dup.iloc[self.idx_dupe]

doc/source/whatsnew/index.rst

@@ -24,6 +24,7 @@ Version 1.0
 .. toctree::
    :maxdepth: 2
 
+   v1.0.5
    v1.0.4
    v1.0.3
    v1.0.2

doc/source/whatsnew/v1.0.4.rst

@@ -45,4 +45,4 @@ Bug fixes
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.0.3..v1.0.4|HEAD
+.. contributors:: v1.0.3..v1.0.4

doc/source/whatsnew/v1.0.5.rst

@@ -0,0 +1,31 @@
+
+.. _whatsnew_105:
+
+What's new in 1.0.5 (June XX, 2020)
+-----------------------------------
+
+These are the changes in pandas 1.0.5. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_105.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+-
+-
+
+.. _whatsnew_105.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+-
+-
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.0.4..v1.0.5|HEAD

doc/source/whatsnew/v1.1.0.rst

@@ -394,6 +394,8 @@ Backwards incompatible API changes
 - :meth:`Series.to_timestamp` now raises a ``TypeError`` if the axis is not a :class:`PeriodIndex`. Previously an ``AttributeError`` was raised (:issue:`33327`)
 - :meth:`Series.to_period` now raises a ``TypeError`` if the axis is not a :class:`DatetimeIndex`. Previously an ``AttributeError`` was raised (:issue:`33327`)
 - :func: `pandas.api.dtypes.is_string_dtype` no longer incorrectly identifies categorical series as string.
+- :func:`read_excel` no longer takes ``**kwds`` arguments. This means that passing in keyword ``chunksize`` now raises a ``TypeError``
+  (previously raised a ``NotImplementedError``), while passing in keyword ``encoding`` now raises a ``TypeError`` (:issue:`34464`)
 
 ``MultiIndex.get_indexer`` interprets `method` argument differently
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -727,6 +729,7 @@ Performance improvements
 - Performance improvement in arithmetic operations between two :class:`DataFrame` objects (:issue:`32779`)
 - Performance improvement in :class:`pandas.core.groupby.RollingGroupby` (:issue:`34052`)
 - Performance improvement in arithmetic operations (sub, add, mul, div) for MultiIndex (:issue:`34297`)
+- Performance improvement in `DataFrame[bool_indexer]` when `bool_indexer` is a list (:issue:`33924`)
 
 .. ---------------------------------------------------------------------------
 
@@ -759,6 +762,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.to_period` not infering the frequency when called with no arguments (:issue:`33358`)
 - Bug in :meth:`DatetimeIndex.tz_localize` incorrectly retaining ``freq`` in some cases where the original freq is no longer valid (:issue:`30511`)
 - Bug in :meth:`DatetimeIndex.intersection` losing ``freq`` and timezone in some cases (:issue:`33604`)
+- Bug in :meth:`DatetimeIndex.get_indexer` where incorrect output would be returned for mixed datetime-like targets (:issue:`33741`)
 - Bug in :class:`DatetimeIndex` addition and subtraction with some types of :class:`DateOffset` objects incorrectly retaining an invalid ``freq`` attribute (:issue:`33779`)
 - Bug in :class:`DatetimeIndex` where setting the ``freq`` attribute on an index could silently change the ``freq`` attribute on another index viewing the same data (:issue:`33552`)
 - :meth:`DataFrame.min`/:meth:`DataFrame.max` not returning consistent result with :meth:`Series.min`/:meth:`Series.max` when called on objects initialized with empty :func:`pd.to_datetime`
@@ -963,6 +967,7 @@ Sparse
 - Creating a :class:`SparseArray` from timezone-aware dtype will issue a warning before dropping timezone information, instead of doing so silently (:issue:`32501`)
 - Bug in :meth:`arrays.SparseArray.from_spmatrix` wrongly read scipy sparse matrix (:issue:`31991`)
 - Bug in :meth:`Series.sum` with ``SparseArray`` raises ``TypeError`` (:issue:`25777`)
+- Bug where :class:`DataFrame` containing :class:`SparseArray` filled with ``NaN`` when indexed by a list-like (:issue:`27781`, :issue:`29563`)
 - The repr of :class:`SparseDtype` now includes the repr of its ``fill_value`` attribute. Previously it used ``fill_value``'s  string representation (:issue:`34352`)
 
 ExtensionArray
@@ -994,6 +999,7 @@ Other
 - Bug in :meth:`DataFrame.plot.scatter` caused an error when plotting variable marker sizes (:issue:`32904`)
 - :class:`IntegerArray` now implements the ``sum`` operation (:issue:`33172`)
 - Bug in :class:`Tick` comparisons raising ``TypeError`` when comparing against timedelta-like objects (:issue:`34088`)
+- Bug in :class:`Tick` multiplication raising ``TypeError`` when multiplying by a float (:issue:`34486`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/hashtable_func_helper.pxi.in

@@ -84,7 +84,8 @@ cpdef value_count_{{dtype}}({{c_type}}[:] values, bint dropna):
         int64_t[:] result_counts
         {{endif}}
 
-        Py_ssize_t k
+        # Don't use Py_ssize_t, since table.n_buckets is unsigned
+        khiter_t k
 
     table = kh_init_{{ttype}}()
     {{if dtype == 'object'}}
@@ -132,7 +133,8 @@ def duplicated_{{dtype}}(const {{c_type}}[:] values, object keep='first'):
         {{if dtype != 'object'}}
         {{dtype}}_t value
         {{endif}}
-        Py_ssize_t k, i, n = len(values)
+        Py_ssize_t i, n = len(values)
+        khiter_t k
         kh_{{ttype}}_t *table = kh_init_{{ttype}}()
         ndarray[uint8_t, ndim=1, cast=True] out = np.empty(n, dtype='bool')
 
@@ -222,7 +224,8 @@ def ismember_{{dtype}}(const {{c_type}}[:] arr, {{c_type}}[:] values):
     boolean ndarry len of (arr)
     """
     cdef:
-        Py_ssize_t i, n, k
+        Py_ssize_t i, n
+        khiter_t k
         int ret = 0
         ndarray[uint8_t] result
         {{c_type}} val
@@ -295,7 +298,8 @@ def mode_{{dtype}}({{ctype}}[:] values, bint dropna):
     cdef:
         int count, max_count = 1
         int j = -1  # so you can do +=
-        Py_ssize_t k
+        # Don't use Py_ssize_t, since table.n_buckets is unsigned
+        khiter_t k
         kh_{{table_type}}_t *table
         ndarray[{{ctype}}] modes
 

pandas/_libs/lib.pyx

@@ -1380,8 +1380,10 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
         return "mixed-integer"
 
     elif PyDateTime_Check(val):
-        if is_datetime_array(values):
+        if is_datetime_array(values, skipna=skipna):
             return "datetime"
+        elif is_date_array(values, skipna=skipna):
+            return "date"
 
     elif PyDate_Check(val):
         if is_date_array(values, skipna=skipna):
@@ -1752,10 +1754,10 @@ cdef class DatetimeValidator(TemporalValidator):
         return is_null_datetime64(value)
 
 
-cpdef bint is_datetime_array(ndarray values):
+cpdef bint is_datetime_array(ndarray values, bint skipna=True):
     cdef:
         DatetimeValidator validator = DatetimeValidator(len(values),
-                                                        skipna=True)
+                                                        skipna=skipna)
     return validator.validate(values)
 
 

pandas/_libs/src/parser/tokenizer.c

@@ -709,7 +709,7 @@ int skip_this_line(parser_t *self, int64_t rownum) {
 }
 
 int tokenize_bytes(parser_t *self,
-                   size_t line_limit, int64_t start_lines) {
+                   size_t line_limit, uint64_t start_lines) {
     int64_t i;
     uint64_t slen;
     int should_skip;
@@ -1348,7 +1348,7 @@ int parser_trim_buffers(parser_t *self) {
 
 int _tokenize_helper(parser_t *self, size_t nrows, int all) {
     int status = 0;
-    int64_t start_lines = self->lines;
+    uint64_t start_lines = self->lines;
 
     if (self->state == FINISHED) {
         return 0;

pandas/_libs/tslibs/__init__.py

@@ -8,6 +8,7 @@
     "OutOfBoundsDatetime",
     "IncompatibleFrequency",
     "Period",
+    "Resolution",
     "Timedelta",
     "delta_to_nanoseconds",
     "ints_to_pytimedelta",
@@ -20,6 +21,7 @@
 from .nattype import NaT, NaTType, iNaT, is_null_datetimelike, nat_strings
 from .np_datetime import OutOfBoundsDatetime
 from .period import IncompatibleFrequency, Period
+from .resolution import Resolution
 from .timedeltas import Timedelta, delta_to_nanoseconds, ints_to_pytimedelta
 from .timestamps import Timestamp
 from .tzconversion import tz_convert_single

pandas/_libs/tslibs/conversion.pxd

@@ -1,6 +1,6 @@
-from cpython.datetime cimport datetime
+from cpython.datetime cimport datetime, tzinfo
 
-from numpy cimport int64_t, int32_t
+from numpy cimport int64_t, int32_t, ndarray
 
 from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct
 
@@ -24,3 +24,5 @@ cdef int64_t get_datetime64_nanos(object val) except? -1
 
 cpdef datetime localize_pydatetime(datetime dt, object tz)
 cdef int64_t cast_from_unit(object ts, str unit) except? -1
+
+cpdef ndarray[int64_t] normalize_i8_timestamps(const int64_t[:] stamps, tzinfo tz)

pandas/_libs/tslibs/conversion.pyx

@@ -763,7 +763,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def normalize_i8_timestamps(int64_t[:] stamps, object tz):
+cpdef ndarray[int64_t] normalize_i8_timestamps(const int64_t[:] stamps, tzinfo tz):
     """
     Normalize each of the (nanosecond) timezone aware timestamps in the given
     array by rounding down to the beginning of the day (i.e. midnight).
@@ -774,31 +774,6 @@ def normalize_i8_timestamps(int64_t[:] stamps, object tz):
     stamps : int64 ndarray
     tz : tzinfo or None
 
-    Returns
-    -------
-    result : int64 ndarray of converted of normalized nanosecond timestamps
-    """
-    cdef:
-        int64_t[:] result
-
-    result = _normalize_local(stamps, tz)
-
-    return result.base  # .base to access underlying np.ndarray
-
-
-@cython.wraparound(False)
-@cython.boundscheck(False)
-cdef int64_t[:] _normalize_local(const int64_t[:] stamps, tzinfo tz):
-    """
-    Normalize each of the (nanosecond) timestamps in the given array by
-    rounding down to the beginning of the day (i.e. midnight) for the
-    given timezone `tz`.
-
-    Parameters
-    ----------
-    stamps : int64 ndarray
-    tz : tzinfo
-
     Returns
     -------
     result : int64 ndarray of converted of normalized nanosecond timestamps
@@ -813,7 +788,16 @@ cdef int64_t[:] _normalize_local(const int64_t[:] stamps, tzinfo tz):
         npy_datetimestruct dts
         int64_t delta, local_val
 
-    if is_tzlocal(tz):
+    if tz is None or is_utc(tz):
+        with nogil:
+            for i in range(n):
+                if stamps[i] == NPY_NAT:
+                    result[i] = NPY_NAT
+                    continue
+                local_val = stamps[i]
+                dt64_to_dtstruct(local_val, &dts)
+                result[i] = _normalized_stamp(&dts)
+    elif is_tzlocal(tz):
         for i in range(n):
             if stamps[i] == NPY_NAT:
                 result[i] = NPY_NAT
@@ -843,7 +827,7 @@ cdef int64_t[:] _normalize_local(const int64_t[:] stamps, tzinfo tz):
                 dt64_to_dtstruct(stamps[i] + deltas[pos[i]], &dts)
                 result[i] = _normalized_stamp(&dts)
 
-    return result
+    return result.base  # `.base` to access underlying ndarray
 
 
 cdef inline int64_t _normalized_stamp(npy_datetimestruct *dts) nogil: