Merge remote-tracking branch 'upstream/master' into grp-desc-perf

Author: WillAyd

README.md

@@ -89,7 +89,7 @@
 
 
 
-## What is it
+## What is it?
 
 **pandas** is a Python package providing fast, flexible, and expressive data
 structures designed to make working with "relational" or "labeled" data both

asv_bench/benchmarks/series_methods.py

@@ -38,6 +38,64 @@ def time_isin(self, dtypes):
         self.s.isin(self.values)
 
 
+class IsInFloat64(object):
+
+    def setup(self):
+        self.small = Series([1, 2], dtype=np.float64)
+        self.many_different_values = np.arange(10**6, dtype=np.float64)
+        self.few_different_values = np.zeros(10**7, dtype=np.float64)
+        self.only_nans_values = np.full(10**7, np.nan, dtype=np.float64)
+
+    def time_isin_many_different(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.many_different_values)
+
+    def time_isin_few_different(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.few_different_values)
+
+    def time_isin_nan_values(self):
+        # runtime is dominated by creation of the lookup-table
+        self.small.isin(self.few_different_values)
+
+
+class IsInForObjects(object):
+
+    def setup(self):
+        self.s_nans = Series(np.full(10**4, np.nan)).astype(np.object)
+        self.vals_nans = np.full(10**4, np.nan).astype(np.object)
+        self.s_short = Series(np.arange(2)).astype(np.object)
+        self.s_long = Series(np.arange(10**5)).astype(np.object)
+        self.vals_short = np.arange(2).astype(np.object)
+        self.vals_long = np.arange(10**5).astype(np.object)
+        # because of nans floats are special:
+        self.s_long_floats = Series(np.arange(10**5,
+                                    dtype=np.float)).astype(np.object)
+        self.vals_long_floats = np.arange(10**5,
+                                          dtype=np.float).astype(np.object)
+
+    def time_isin_nans(self):
+        # if nan-objects are different objects,
+        # this has the potential to trigger O(n^2) running time
+        self.s_nans.isin(self.vals_nans)
+
+    def time_isin_short_series_long_values(self):
+        # running time dominated by the preprocessing
+        self.s_short.isin(self.vals_long)
+
+    def time_isin_long_series_short_values(self):
+        # running time dominated by look-up
+        self.s_long.isin(self.vals_short)
+
+    def time_isin_long_series_long_values(self):
+        # no dominating part
+        self.s_long.isin(self.vals_long)
+
+    def time_isin_long_series_long_values_floats(self):
+        # no dominating part
+        self.s_long_floats.isin(self.vals_long_floats)
+
+
 class NSort(object):
 
     goal_time = 0.2

doc/source/api.rst

@@ -545,6 +545,7 @@ These can be accessed like ``Series.dt.<property>``.
 
    Series.dt.date
    Series.dt.time
+   Series.dt.timetz
    Series.dt.year
    Series.dt.month
    Series.dt.day
@@ -1739,6 +1740,7 @@ Time/Date Components
    DatetimeIndex.nanosecond
    DatetimeIndex.date
    DatetimeIndex.time
+   DatetimeIndex.timetz
    DatetimeIndex.dayofyear
    DatetimeIndex.weekofyear
    DatetimeIndex.week

doc/source/install.rst

@@ -225,7 +225,7 @@ Dependencies
 
 * `setuptools <https://setuptools.readthedocs.io/en/latest/>`__: 24.2.0 or higher
 * `NumPy <http://www.numpy.org>`__: 1.9.0 or higher
-* `python-dateutil <//https://dateutil.readthedocs.io/en/stable/>`__: 2.5.0 or higher
+* `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__: 2.5.0 or higher
 * `pytz <http://pytz.sourceforge.net/>`__
 
 .. _install.recommended_dependencies:

doc/source/timeseries.rst

@@ -724,6 +724,7 @@ There are several time/date properties that one can access from ``Timestamp`` or
     nanosecond,"The nanoseconds of the datetime"
     date,"Returns datetime.date (does not contain timezone information)"
     time,"Returns datetime.time (does not contain timezone information)"
+    timetz,"Returns datetime.time as local time with timezone information"
     dayofyear,"The ordinal day of year"
     weekofyear,"The week ordinal of the year"
     week,"The week ordinal of the year"

doc/source/whatsnew/v0.23.5.txt

@@ -20,6 +20,9 @@ and bug fixes. We recommend that all users upgrade to this version.
 Fixed Regressions
 ~~~~~~~~~~~~~~~~~
 
+- Constructing a DataFrame with an index argument that wasn't already an
+  instance of :class:`~pandas.core.Index` was broken in `4efb39f
+  <https://github.com/pandas-dev/pandas/commit/4efb39f01f5880122fa38d91e12d217ef70fad9e>`_ (:issue:`22227`).
 -
 -
 
@@ -30,10 +33,14 @@ Bug Fixes
 
 **Groupby/Resample/Rolling**
 
--
+- Bug in :meth:`DataFrame.resample` when resampling ``NaT`` in ``TimeDeltaIndex`` (:issue:`13223`).
 -
 
 **Missing**
 
 -
 -
+
+**I/O**
+
+- Bug in :func:`read_csv` that caused it to raise ``OverflowError`` when trying to use 'inf' as ``na_value`` with integer index column (:issue:`17128`)

doc/source/whatsnew/v0.24.0.txt

@@ -181,6 +181,7 @@ Other Enhancements
   The default compression for ``to_csv``, ``to_json``, and ``to_pickle`` methods has been updated to ``'infer'`` (:issue:`22004`).
 - :func:`to_timedelta` now supports iso-formated timedelta strings (:issue:`21877`)
 - :class:`Series` and :class:`DataFrame` now support :class:`Iterable` in constructor (:issue:`2193`)
+- :class:`DatetimeIndex` gained :attr:`DatetimeIndex.timetz` attribute. Returns local time with timezone information. (:issue:`21358`)
 
 .. _whatsnew_0240.api_breaking:
 
@@ -477,6 +478,7 @@ Deprecations
 - :meth:`MultiIndex.to_hierarchical` is deprecated and will be removed in a future version  (:issue:`21613`)
 - :meth:`Series.ptp` is deprecated. Use ``numpy.ptp`` instead (:issue:`21614`)
 - :meth:`Series.compress` is deprecated. Use ``Series[condition]`` instead (:issue:`18262`)
+- The signature of :meth:`Series.to_csv` has been uniformed to that of doc:meth:`DataFrame.to_csv`: the name of the first argument is now 'path_or_buf', the order of subsequent arguments has changed, the 'header' argument now defaults to True. (:issue:`19715`)
 - :meth:`Categorical.from_codes` has deprecated providing float values for the ``codes`` argument. (:issue:`21767`)
 - :func:`pandas.read_table` is deprecated. Instead, use :func:`pandas.read_csv` passing ``sep='\t'`` if necessary (:issue:`21948`)
 
@@ -618,13 +620,14 @@ Missing
 
 - Bug in :func:`DataFrame.fillna` where a ``ValueError`` would raise when one column contained a ``datetime64[ns, tz]`` dtype (:issue:`15522`)
 - Bug in :func:`Series.hasnans` that could be incorrectly cached and return incorrect answers if null elements are introduced after an initial call (:issue:`19700`)
+- :func:`Series.isin` now treats all nans as equal also for `np.object`-dtype. This behavior is consistent with the behavior for float64 (:issue:`22119`)
 
 MultiIndex
 ^^^^^^^^^^
 
-- Removed compatibility for MultiIndex pickles prior to version 0.8.0; compatibility with MultiIndex pickles from version 0.13 forward is maintained (:issue:`21654`)
--
--
+- Removed compatibility for :class:`MultiIndex` pickles prior to version 0.8.0; compatibility with :class:`MultiIndex` pickles from version 0.13 forward is maintained (:issue:`21654`)
+- :meth:`MultiIndex.get_loc_level` (and as a consequence, ``.loc`` on a :class:``MultiIndex``ed object) will now raise a ``KeyError``, rather than returning an empty ``slice``, if asked a label which is present in the ``levels`` but is unused (:issue:`22221`)
+- Fix ``TypeError`` in Python 3 when creating :class:`MultiIndex` in which some levels have mixed types, e.g. when some labels are tuples (:issue:`15457`)
 
 I/O
 ^^^
@@ -668,6 +671,7 @@ Reshaping
 - :func:`pandas.core.groupby.GroupBy.rank` now raises a ``ValueError`` when an invalid value is passed for argument ``na_option`` (:issue:`22124`)
 - Bug in :func:`get_dummies` with Unicode attributes in Python 2 (:issue:`22084`)
 - Bug in :meth:`DataFrame.replace` raises ``RecursionError`` when replacing empty lists (:issue:`22083`)
+- Bug in :meth:`Series.replace` and meth:`DataFrame.replace` when dict is used as the `to_replace` value and one key in the dict is is another key's value, the results were inconsistent between using integer key and using string key (:issue:`20656`)
 -
 
 Build Changes

pandas/_libs/algos_common_helper.pxi.in

@@ -523,7 +523,7 @@ def put2d_{{name}}_{{dest_type}}(ndarray[{{c_type}}, ndim=2, cast=True] values,
         Py_ssize_t i, j, k
 
     k = len(values)
-    for j from 0 <= j < k:
+    for j in range(k):
         i = indexer[j]
         out[i] = values[j, loc]
 

pandas/_libs/algos_take_helper.pxi.in

@@ -46,7 +46,7 @@ def get_dispatch(dtypes):
     fv = fill_value
 
     %(nogil_str)s
-    %(tab)sfor i from 0 <= i < n:
+    %(tab)sfor i in range(n):
     %(tab)s    idx = indexer[i]
     %(tab)s    if idx == -1:
     %(tab)s        out[i] = fv
@@ -74,24 +74,24 @@ def get_dispatch(dtypes):
             values.strides[1] == sizeof(%(c_type_out)s) and
             sizeof(%(c_type_out)s) * n >= 256):
 
-            for i from 0 <= i < n:
+            for i in range(n):
                 idx = indexer[i]
                 if idx == -1:
-                    for j from 0 <= j < k:
+                    for j in range(k):
                         out[i, j] = fv
                 else:
                     v = &values[idx, 0]
                     o = &out[i, 0]
                     memmove(o, v, <size_t>(sizeof(%(c_type_out)s) * k))
             return
 
-    for i from 0 <= i < n:
+    for i in range(n):
         idx = indexer[i]
         if idx == -1:
-            for j from 0 <= j < k:
+            for j in range(k):
                 out[i, j] = fv
         else:
-            for j from 0 <= j < k:
+            for j in range(k):
                 out[i, j] = %(preval)svalues[idx, j]%(postval)s
 """
 
@@ -108,8 +108,8 @@ def get_dispatch(dtypes):
 
     fv = fill_value
 
-    for i from 0 <= i < n:
-        for j from 0 <= j < k:
+    for i in range(n):
+        for j in range(k):
             idx = indexer[j]
             if idx == -1:
                 out[i, j] = fv
@@ -246,13 +246,13 @@ def take_2d_multi_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=2] values,
     k = len(idx1)
 
     fv = fill_value
-    for i from 0 <= i < n:
+    for i in range(n):
         idx = idx0[i]
         if idx == -1:
-            for j from 0 <= j < k:
+            for j in range(k):
                 out[i, j] = fv
         else:
-            for j from 0 <= j < k:
+            for j in range(k):
                 if idx1[j] == -1:
                     out[i, j] = fv
                 else:

pandas/_libs/hashtable_func_helper.pxi.in

@@ -161,18 +161,18 @@ def duplicated_{{dtype}}({{scalar}}[:] values, object keep='first'):
         {{endif}}
     elif keep == 'first':
         {{if dtype == 'object'}}
-        for i from 0 <= i < n:
+        for i in range(n):
             kh_put_{{ttype}}(table, <PyObject*> values[i], &ret)
             out[i] = ret == 0
         {{else}}
         with nogil:
-            for i from 0 <= i < n:
+            for i in range(n):
                 kh_put_{{ttype}}(table, values[i], &ret)
                 out[i] = ret == 0
         {{endif}}
     else:
         {{if dtype == 'object'}}
-        for i from 0 <= i < n:
+        for i in range(n):
             value = values[i]
             k = kh_get_{{ttype}}(table, <PyObject*> value)
             if k != table.n_buckets:
@@ -185,7 +185,7 @@ def duplicated_{{dtype}}({{scalar}}[:] values, object keep='first'):
                 out[i] = 0
         {{else}}
         with nogil:
-            for i from 0 <= i < n:
+            for i in range(n):
                 value = values[i]
                 k = kh_get_{{ttype}}(table, value)
                 if k != table.n_buckets:
@@ -210,10 +210,10 @@ def duplicated_{{dtype}}({{scalar}}[:] values, object keep='first'):
 @cython.boundscheck(False)
 {{if dtype == 'object'}}
 
-def ismember_{{dtype}}(ndarray[{{scalar}}] arr, ndarray[{{scalar}}] values, bint hasnans=0):
+def ismember_{{dtype}}(ndarray[{{scalar}}] arr, ndarray[{{scalar}}] values):
 {{else}}
 
-def ismember_{{dtype}}({{scalar}}[:] arr, {{scalar}}[:] values, bint hasnans=0):
+def ismember_{{dtype}}({{scalar}}[:] arr, {{scalar}}[:] values):
 {{endif}}
 
     """
@@ -224,7 +224,6 @@ def ismember_{{dtype}}({{scalar}}[:] arr, {{scalar}}[:] values, bint hasnans=0):
     ----------
     arr : {{dtype}} ndarray
     values : {{dtype}} ndarray
-    hasnans : bint, optional
 
     Returns
     -------
@@ -259,19 +258,13 @@ def ismember_{{dtype}}({{scalar}}[:] arr, {{scalar}}[:] values, bint hasnans=0):
     for i in range(n):
         val = arr[i]
         k = kh_get_{{ttype}}(table, <PyObject*> val)
-        if k != table.n_buckets:
-            result[i] = 1
-        else:
-            result[i] = hasnans and val != val
+        result[i] = (k != table.n_buckets)
     {{else}}
     with nogil:
         for i in range(n):
             val = arr[i]
             k = kh_get_{{ttype}}(table, val)
-            if k != table.n_buckets:
-                result[i] = 1
-            else:
-                result[i] = hasnans and val != val
+            result[i] = (k != table.n_buckets)
     {{endif}}
 
     kh_destroy_{{ttype}}(table)