Merge branch 'master' into feature/flake8-rst

Author: FHaase

ci/azure-36-locale_slow.yaml

@@ -14,7 +14,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl=2.5.5
+  - openpyxl
   - psycopg2
   - pymysql
   - pytables

ci/azure-37-locale.yaml

@@ -13,7 +13,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl=2.5.5
+  - openpyxl
   - psycopg2
   - pymysql
   - pytables

ci/azure-macos-35.yaml

@@ -12,7 +12,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy=1.12.0
-  - openpyxl=2.5.5
+  - openpyxl
   - pytables
   - python=3.5*
   - pytz

ci/azure-windows-27.yaml

@@ -13,7 +13,7 @@ dependencies:
   - matplotlib=2.0.1
   - numexpr
   - numpy=1.12*
-  - openpyxl=2.5.5
+  - openpyxl
   - pytables
   - python=2.7.*
   - pytz

ci/azure-windows-36.yaml

@@ -11,7 +11,7 @@ dependencies:
   - matplotlib
   - numexpr
   - numpy=1.14*
-  - openpyxl=2.5.5
+  - openpyxl
   - parquet-cpp
   - pyarrow
   - pytables

ci/circle-36-locale.yaml

@@ -13,7 +13,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl=2.5.5
+  - openpyxl
   - psycopg2
   - pymysql
   - pytables

ci/requirements-optional-conda.txt

@@ -12,7 +12,7 @@ lxml
 matplotlib>=2.0.0
 nbsphinx
 numexpr>=2.6.1
-openpyxl=2.5.5
+openpyxl
 pyarrow
 pymysql
 pytables>=3.4.2

ci/requirements-optional-pip.txt

@@ -14,7 +14,7 @@ lxml
 matplotlib>=2.0.0
 nbsphinx
 numexpr>=2.6.1
-openpyxl==2.5.5
+openpyxl
 pyarrow
 pymysql
 tables

ci/travis-27.yaml

@@ -35,7 +35,7 @@ dependencies:
   - s3fs
   - scipy
   - sqlalchemy=0.9.6
-  - xarray=0.8.0
+  - xarray=0.9.6
   - xlrd=0.9.2
   - xlsxwriter=0.5.2
   - xlwt=0.7.5

ci/travis-36-doc.yaml

@@ -22,7 +22,7 @@ dependencies:
   - notebook
   - numexpr
   - numpy=1.13*
-  - openpyxl=2.5.5
+  - openpyxl
   - pandoc
   - pyqt
   - pytables

ci/travis-36-slow.yaml

@@ -10,7 +10,7 @@ dependencies:
   - matplotlib
   - numexpr
   - numpy
-  - openpyxl=2.5.5
+  - openpyxl
   - patsy
   - psycopg2
   - pymysql

ci/travis-36.yaml

@@ -22,7 +22,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl=2.5.5
+  - openpyxl
   - psycopg2
   - pyarrow
   - pymysql

doc/source/api.rst

@@ -515,7 +515,6 @@ Reshaping, sorting
    Series.repeat
    Series.squeeze
    Series.view
-   Series.sortlevel
 
 
 Combining / joining / merging

doc/source/whatsnew/v0.24.0.txt

@@ -180,6 +180,29 @@ array, but rather an ``ExtensionArray``:
 This is the same behavior as ``Series.values`` for categorical data. See
 :ref:`whatsnew_0240.api_breaking.interval_values` for more.
 
+.. _whatsnew_0240.enhancements.rename_axis:
+
+Renaming names in a MultiIndex
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:func:`DataFrame.rename_axis` now supports ``index`` and ``columns`` arguments
+and :func:`Series.rename_axis` supports ``index`` argument (:issue:`19978`)
+
+This change allows a dictionary to be passed so that some of the names 
+of a ``MultiIndex`` can be changed.
+
+Example:
+
+.. ipython:: python
+
+    mi = pd.MultiIndex.from_product([list('AB'), list('CD'), list('EF')],
+                                    names=['AB', 'CD', 'EF'])
+    df = pd.DataFrame([i for i in range(len(mi))], index=mi, columns=['N'])
+    df
+    df.rename_axis(index={'CD': 'New'})
+
+See the :ref:`advanced docs on renaming<advanced.index_names>` for more details.
+
 .. _whatsnew_0240.enhancements.other:
 
 Other Enhancements
@@ -201,7 +224,6 @@ Other Enhancements
 - :func:`~DataFrame.to_csv`, :func:`~Series.to_csv`, :func:`~DataFrame.to_json`, and :func:`~Series.to_json` now support ``compression='infer'`` to infer compression based on filename extension (:issue:`15008`).
   The default compression for ``to_csv``, ``to_json``, and ``to_pickle`` methods has been updated to ``'infer'`` (:issue:`22004`).
 - :func:`to_timedelta` now supports iso-formated timedelta strings (:issue:`21877`)
-- Comparing :class:`Timedelta` with unknown types now return ``NotImplemented`` instead of ``False`` (:issue:`20829`)
 - :class:`Series` and :class:`DataFrame` now support :class:`Iterable` in constructor (:issue:`2193`)
 - :class:`DatetimeIndex` gained :attr:`DatetimeIndex.timetz` attribute. Returns local time with timezone information. (:issue:`21358`)
 - :meth:`round`, :meth:`ceil`, and meth:`floor` for :class:`DatetimeIndex` and :class:`Timestamp` now support an ``ambiguous`` argument for handling datetimes that are rounded to ambiguous times (:issue:`18946`)
@@ -910,6 +932,8 @@ Other API Changes
   has an improved ``KeyError`` message, and will not fail on duplicate column names with ``drop=True``. (:issue:`22484`)
 - Slicing a single row of a DataFrame with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`)
 - :class:`DateOffset` attribute `_cacheable` and method `_should_cache` have been removed (:issue:`23118`)
+- Comparing :class:`Timedelta` to be less or greater than unknown types now raises a ``TypeError`` instead of returning ``False`` (:issue:`20829`)
+- :meth:`Index.hasnans` and :meth:`Series.hasnans` now always return a python boolean. Previously, a python or a numpy boolean could be returned, depending on circumstances (:issue:`23294`).
 
 .. _whatsnew_0240.deprecations:
 
@@ -929,6 +953,7 @@ Deprecations
 - :func:`DatetimeIndex.shift` and :func:`PeriodIndex.shift` now accept ``periods`` argument instead of ``n`` for consistency with :func:`Index.shift` and :func:`Series.shift`. Using ``n`` throws a deprecation warning (:issue:`22458`, :issue:`22912`)
 - The ``fastpath`` keyword of the different Index constructors is deprecated (:issue:`23110`).
 - :meth:`Timestamp.tz_localize`, :meth:`DatetimeIndex.tz_localize`, and :meth:`Series.tz_localize` have deprecated the ``errors`` argument in favor of the ``nonexistent`` argument (:issue:`8917`)
+- The class ``FrozenNDArray`` has been deprecated. When unpickling, ``FrozenNDArray`` will be unpickled to ``np.ndarray`` once this class is removed (:issue:`9031`)
 
 .. _whatsnew_0240.prior_deprecations:
 
@@ -941,13 +966,22 @@ Removal of prior version deprecations/changes
 - Removal of the previously deprecated module ``pandas.core.datetools`` (:issue:`14105`, :issue:`14094`)
 - Strings passed into :meth:`DataFrame.groupby` that refer to both column and index levels will raise a ``ValueError`` (:issue:`14432`)
 - :meth:`Index.repeat` and :meth:`MultiIndex.repeat` have renamed the ``n`` argument to ``repeats`` (:issue:`14645`)
+- The ``Series`` constructor and ``.astype`` method will now raise a ``ValueError`` if timestamp dtypes are passed in without a unit (e.g. ``np.datetime64``) for the ``dtype`` parameter (:issue:`15987`)
 - Removal of the previously deprecated ``as_indexer`` keyword completely from ``str.match()`` (:issue:`22356`, :issue:`6581`)
+- The modules ``pandas.types``, ``pandas.computation``, and ``pandas.util.decorators`` have been removed (:issue:`16157`, :issue:`16250`)
 - Removed the ``pandas.formats.style`` shim for :class:`pandas.io.formats.style.Styler` (:issue:`16059`)
+- :func:`pandas.pnow`, :func:`pandas.match`, :func:`pandas.groupby`, :func:`pd.get_store`, ``pd.Expr``, and ``pd.Term`` have been removed (:issue:`15538`, :issue:`15940`)
 - :meth:`Categorical.searchsorted` and :meth:`Series.searchsorted` have renamed the ``v`` argument to ``value`` (:issue:`14645`)
+- ``pandas.parser``, ``pandas.lib``, and ``pandas.tslib`` have been removed (:issue:`15537`)
 - :meth:`TimedeltaIndex.searchsorted`, :meth:`DatetimeIndex.searchsorted`, and :meth:`PeriodIndex.searchsorted` have renamed the ``key`` argument to ``value`` (:issue:`14645`)
+- :meth:`DataFrame.consolidate` and :meth:`Series.consolidate` have been removed (:issue:`15501`)
 - Removal of the previously deprecated module ``pandas.json`` (:issue:`19944`)
+- The module ``pandas.tools`` has been removed (:issue:`15358`, :issue:`16005`)
 - :meth:`SparseArray.get_values` and :meth:`SparseArray.to_dense` have dropped the ``fill`` parameter (:issue:`14686`)
+- :meth:`DataFrame.sortlevel` and :meth:`Series.sortlevel` have been removed (:issue:`15099`)
 - :meth:`SparseSeries.to_dense` has dropped the ``sparse_only`` parameter (:issue:`14686`)
+- :meth:`DataFrame.astype` and :meth:`Series.astype` have renamed the ``raise_on_error`` argument to ``errors`` (:issue:`14967`)
+- ``is_sequence``, ``is_any_int_dtype``, and ``is_floating_dtype`` have been removed from ``pandas.api.types`` (:issue:`16163`, :issue:`16189`)
 
 .. _whatsnew_0240.performance:
 
@@ -1020,7 +1054,10 @@ Datetimelike
 - Bug in :func:`to_datetime` with an :class:`Index` argument that would drop the ``name`` from the result (:issue:`21697`)
 - Bug in :class:`PeriodIndex` where adding or subtracting a :class:`timedelta` or :class:`Tick` object produced incorrect results (:issue:`22988`)
 - Bug in :func:`date_range` when decrementing a start date to a past end date by a negative frequency (:issue:`23270`)
+- Bug in :meth:`Series.min` which would return ``NaN`` instead of ``NaT`` when called on a series of ``NaT`` (:issue:`23282`)
 - Bug in :func:`DataFrame.combine` with datetimelike values raising a TypeError (:issue:`23079`)
+- Bug in :func:`date_range` with frequency of ``Day`` or higher where dates sufficiently far in the future could wrap around to the past instead of raising ``OutOfBoundsDatetime`` (:issue:`14187`)
+- Bug in :class:`PeriodIndex` with attribute ``freq.n`` greater than 1 where adding a :class:`DateOffset` object would return incorrect results (:issue:`23215`)
 
 Timedelta
 ^^^^^^^^^
@@ -1032,7 +1069,8 @@ Timedelta
 - Bug in :class:`TimedeltaIndex` incorrectly allowing indexing with ``Timestamp`` object (:issue:`20464`)
 - Fixed bug where subtracting :class:`Timedelta` from an object-dtyped array would raise ``TypeError`` (:issue:`21980`)
 - Fixed bug in adding a :class:`DataFrame` with all-`timedelta64[ns]` dtypes to a :class:`DataFrame` with all-integer dtypes returning incorrect results instead of raising ``TypeError`` (:issue:`22696`)
-
+- Bug in :class:`TimedeltaIndex` where adding a timezone-aware datetime scalar incorrectly returned a timezone-naive :class:`DatetimeIndex` (:issue:`23215`)
+- Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`23215`)
 
 Timezones
 ^^^^^^^^^
@@ -1062,7 +1100,7 @@ Offsets
 
 - Bug in :class:`FY5253` where date offsets could incorrectly raise an ``AssertionError`` in arithmetic operatons (:issue:`14774`)
 - Bug in :class:`DateOffset` where keyword arguments ``week`` and ``milliseconds`` were accepted and ignored.  Passing these will now raise ``ValueError`` (:issue:`19398`)
--
+- Bug in adding :class:`DateOffset` with :class:`DataFrame` or :class:`PeriodIndex` incorrectly raising ``TypeError`` (:issue:`23215`)
 
 Numeric
 ^^^^^^^
@@ -1079,6 +1117,7 @@ Numeric
 - Bug in :meth:`DataFrame.astype` to extension dtype may raise ``AttributeError`` (:issue:`22578`)
 - Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype arithmetic operations with ``ndarray`` with integer dtype incorrectly treating the narray as ``timedelta64[ns]`` dtype (:issue:`23114`)
 - Bug in :meth:`Series.rpow` with object dtype ``NaN`` for ``1 ** NA`` instead of ``1`` (:issue:`22922`).
+- :meth:`Series.agg` can now handle numpy NaN-aware methods like :func:`numpy.nansum` (:issue:`19629`)
 
 Strings
 ^^^^^^^
@@ -1094,6 +1133,7 @@ Interval
 - Bug in the ``IntervalIndex`` repr where a trailing comma was missing after the list of intervals (:issue:`20611`)
 - Bug in :class:`Interval` where scalar arithmetic operations did not retain the ``closed`` value (:issue:`22313`)
 - Bug in :class:`IntervalIndex` where indexing with datetime-like values raised a ``KeyError`` (:issue:`20636`)
+- Bug in ``IntervalTree`` where data containing ``NaN`` triggered a warning and resulted in incorrect indexing queries with :class:`IntervalIndex` (:issue:`23352`)
 
 Indexing
 ^^^^^^^^

pandas/__init__.py

@@ -45,37 +45,11 @@
 from pandas.core.computation.api import *
 from pandas.core.reshape.api import *
 
-# deprecate tools.plotting, plot_params and scatter_matrix on the top namespace
-import pandas.tools.plotting
-plot_params = pandas.plotting._style._Options(deprecated=True)
-# do not import deprecate to top namespace
-scatter_matrix = pandas.util._decorators.deprecate(
-    'pandas.scatter_matrix', pandas.plotting.scatter_matrix, '0.20.0',
-    'pandas.plotting.scatter_matrix')
-
 from pandas.util._print_versions import show_versions
 from pandas.io.api import *
 from pandas.util._tester import test
 import pandas.testing
 
-# extension module deprecations
-from pandas.util._depr_module import _DeprecatedModule
-
-parser = _DeprecatedModule(deprmod='pandas.parser',
-                           removals=['na_values'],
-                           moved={'CParserError': 'pandas.errors.ParserError'})
-lib = _DeprecatedModule(deprmod='pandas.lib', deprmodto=False,
-                        moved={'Timestamp': 'pandas.Timestamp',
-                               'Timedelta': 'pandas.Timedelta',
-                               'NaT': 'pandas.NaT',
-                               'infer_dtype': 'pandas.api.types.infer_dtype'})
-tslib = _DeprecatedModule(deprmod='pandas.tslib',
-                          moved={'Timestamp': 'pandas.Timestamp',
-                                 'Timedelta': 'pandas.Timedelta',
-                                 'NaT': 'pandas.NaT',
-                                 'NaTType': 'type(pandas.NaT)',
-                                 'OutOfBoundsDatetime': 'pandas.errors.OutOfBoundsDatetime'})
-
 # use the closest tagged version if possible
 from ._version import get_versions
 v = get_versions()

pandas/_libs/intervaltree.pxi.in

@@ -72,6 +72,12 @@ cdef class IntervalTree(IntervalMixin):
 
         self.closed = closed
 
+        # GH 23352: ensure no nan in nodes
+        mask = ~np.isnan(self.left)
+        self.left = self.left[mask]
+        self.right = self.right[mask]
+        indices = indices[mask]
+
         node_cls = NODE_CLASSES[str(self.dtype), closed]
         self.root = node_cls(self.left, self.right, indices, leaf_size)
 

pandas/_libs/tslibs/offsets.pyx

@@ -344,8 +344,8 @@ class _BaseOffset(object):
         return {name: kwds[name] for name in kwds if kwds[name] is not None}
 
     def __add__(self, other):
-        if getattr(other, "_typ", None) in ["datetimeindex",
-                                            "series", "period"]:
+        if getattr(other, "_typ", None) in ["datetimeindex", "periodindex",
+                                            "series", "period", "dataframe"]:
             # defer to the other class's implementation
             return other + self
         try:

pandas/_libs/tslibs/timedeltas.pyx

@@ -1213,6 +1213,12 @@ class Timedelta(_Timedelta):
                 return other.delta * self
             return NotImplemented
 
+        elif util.is_nan(other):
+            # i.e. np.nan, but also catch np.float64("NaN") which would
+            #  otherwise get caught by the hasattr(other, "dtype") branch
+            #  incorrectly return a np.timedelta64 object.
+            return NaT
+
         elif hasattr(other, 'dtype'):
             # ndarray-like
             return other * self.to_timedelta64()
@@ -1240,6 +1246,12 @@ class Timedelta(_Timedelta):
             # convert to Timedelta below
             pass
 
+        elif util.is_nan(other):
+            # i.e. np.nan, but also catch np.float64("NaN") which would
+            #  otherwise get caught by the hasattr(other, "dtype") branch
+            #  incorrectly return a np.timedelta64 object.
+            return NaT
+
         elif hasattr(other, 'dtype'):
             return self.to_timedelta64() / other
 

pandas/_version.py

@@ -12,6 +12,7 @@
 import re
 import subprocess
 import sys
+
 from pandas.compat import PY3
 
 

pandas/compat/pickle_compat.py

@@ -60,6 +60,17 @@ def load_reduce(self):
         ('pandas.core.arrays', 'SparseArray'),
 
     # 15477
+    #
+    # TODO: When FrozenNDArray is removed, add
+    # the following lines for compat:
+    #
+    # ('pandas.core.base', 'FrozenNDArray'):
+    #     ('numpy', 'ndarray'),
+    # ('pandas.core.indexes.frozen', 'FrozenNDArray'):
+    #     ('numpy', 'ndarray'),
+    #
+    # Afterwards, remove the current entry
+    # for `pandas.core.base.FrozenNDArray`.
     ('pandas.core.base', 'FrozenNDArray'):
         ('pandas.core.indexes.frozen', 'FrozenNDArray'),
     ('pandas.core.base', 'FrozenList'):