Merge branch 'master' into master

Author: KristianHolsheimer

.pep8speaks.yml

@@ -11,10 +11,10 @@ pycodestyle:
     max-line-length: 79
     ignore:
         - W503,  # line break before binary operator
+        - W504,  # line break after binary operator
         - E402,  # module level import not at top of file
         - E722,  # do not use bare except
         - E731,  # do not assign a lambda expression, use a def
-        - E741,  # ambiguous variable name 'l'
         - C406,  # Unnecessary list literal - rewrite as a dict literal.
         - C408,  # Unnecessary dict call - rewrite as a literal.
         - C409   # Unnecessary list passed to tuple() - rewrite as a tuple literal.

asv_bench/benchmarks/indexing.py

@@ -2,10 +2,9 @@
 
 import numpy as np
 import pandas.util.testing as tm
-from pandas import (Series, DataFrame, MultiIndex, Int64Index, Float64Index,
-                    IntervalIndex, CategoricalIndex,
+from pandas import (Series, DataFrame, Panel, MultiIndex, Int64Index,
+                    Float64Index, IntervalIndex, CategoricalIndex,
                     IndexSlice, concat, date_range)
-from .pandas_vb_common import Panel
 
 
 class NumericSeriesIndexing(object):

asv_bench/benchmarks/join_merge.py

@@ -11,8 +11,6 @@
 except ImportError:
     from pandas import ordered_merge as merge_ordered
 
-from .pandas_vb_common import Panel
-
 
 class Append(object):
 

asv_bench/benchmarks/panel_ctor.py

@@ -3,8 +3,6 @@
 
 from pandas import DataFrame, Panel, DatetimeIndex, date_range
 
-from .pandas_vb_common import Panel
-
 
 class DifferentIndexes(object):
     goal_time = 0.2

asv_bench/benchmarks/panel_methods.py

@@ -3,8 +3,6 @@
 import numpy as np
 from pandas import Panel
 
-from .pandas_vb_common import Panel
-
 
 class PanelMethods(object):
 

asv_bench/benchmarks/period.py

@@ -119,3 +119,6 @@ def time_align(self):
 
     def time_intersection(self):
         self.index[:750].intersection(self.index[250:])
+
+    def time_unique(self):
+        self.index.unique()

ci/azure-macos-35.yaml

@@ -8,10 +8,10 @@ dependencies:
   - html5lib
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib=2.2.0
   - nomkl
   - numexpr
-  - numpy=1.10.4
+  - numpy=1.12.0
   - openpyxl=2.5.5
   - pytables
   - python=3.5*

ci/azure-windows-27.yaml

@@ -10,7 +10,7 @@ dependencies:
   - html5lib
   - jinja2=2.8
   - lxml
-  - matplotlib
+  - matplotlib=2.0.1
   - numexpr
   - numpy=1.12*
   - openpyxl=2.5.5

ci/azure/macos.yml

@@ -9,7 +9,7 @@ jobs:
   strategy:
     maxParallel: 11
     matrix:
-      py35_np_110:
+      py35_np_120:
         ENV_FILE: ci/azure-macos-35.yaml
         CONDA_PY: "35"
         CONDA_ENV: pandas

ci/azure/windows-py27.yml

@@ -9,7 +9,7 @@ jobs:
   strategy:
     maxParallel: 11
     matrix:
-      py36_np14:
+      py36_np121:
         ENV_FILE: ci/azure-windows-27.yaml
         CONDA_PY: "27"
         CONDA_ENV: pandas

ci/circle-27-compat.yaml

@@ -3,18 +3,18 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - bottleneck=1.0.0
+  - bottleneck=1.2.0
   - cython=0.28.2
   - jinja2=2.8
-  - numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
-  - numpy=1.9.3
+  - numexpr=2.6.1
+  - numpy=1.12.0
   - openpyxl=2.5.5
   - psycopg2
-  - pytables=3.2.2
+  - pytables=3.4.2
   - python-dateutil=2.5.0
   - python=2.7*
   - pytz=2013b
-  - scipy=0.14.0
+  - scipy=0.18.1
   - sqlalchemy=0.7.8
   - xlrd=0.9.2
   - xlsxwriter=0.5.2

ci/code_checks.sh

@@ -56,6 +56,11 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    # Imports - Check formatting using isort see setup.cfg for settings
+    MSG='Check import format using isort ' ; echo $MSG
+    isort --recursive --check-only pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### PATTERNS ###
@@ -118,7 +123,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Doctests frame.py' ; echo $MSG
     pytest --doctest-modules -v pandas/core/frame.py \
-        -k"-axes -combine -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata"
+        -k"-axes -combine -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_stata"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests series.py' ; echo $MSG

ci/environment-dev.yaml

@@ -8,6 +8,7 @@ dependencies:
   - flake8
   - flake8-comprehensions
   - hypothesis>=3.58.0
+  - isort
   - moto
   - pytest>=3.6
   - python-dateutil>=2.5.0

ci/requirements-optional-conda.txt

@@ -1,6 +1,6 @@
 beautifulsoup4>=4.2.1
 blosc
-bottleneck
+bottleneck>=1.2.0
 fastparquet
 feather-format
 gcsfs
@@ -9,17 +9,17 @@ ipython>=5.6.0
 ipykernel
 jinja2
 lxml
-matplotlib
+matplotlib>=2.0.0
 nbsphinx
-numexpr
+numexpr>=2.6.1
 openpyxl=2.5.5
 pyarrow
 pymysql
-pytables
+pytables>=3.4.2
 pytest-cov
 pytest-xdist
 s3fs
-scipy
+scipy>=0.18.1
 seaborn
 sqlalchemy
 statsmodels

ci/requirements-optional-pip.txt

@@ -2,7 +2,7 @@
 # Do not modify directly
 beautifulsoup4>=4.2.1
 blosc
-bottleneck
+bottleneck>=1.2.0
 fastparquet
 feather-format
 gcsfs
@@ -11,17 +11,17 @@ ipython>=5.6.0
 ipykernel
 jinja2
 lxml
-matplotlib
+matplotlib>=2.0.0
 nbsphinx
-numexpr
+numexpr>=2.6.1
 openpyxl==2.5.5
 pyarrow
 pymysql
 tables
 pytest-cov
 pytest-xdist
 s3fs
-scipy
+scipy>=0.18.1
 seaborn
 sqlalchemy
 statsmodels

ci/requirements_dev.txt

@@ -5,6 +5,7 @@ NumPy
 flake8
 flake8-comprehensions
 hypothesis>=3.58.0
+isort
 moto
 pytest>=3.6
 python-dateutil>=2.5.0

ci/travis-27-locale.yaml

@@ -3,11 +3,11 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - bottleneck=1.0.0
+  - bottleneck=1.2.0
   - cython=0.28.2
   - lxml
-  - matplotlib=1.4.3
-  - numpy=1.9.3
+  - matplotlib=2.0.0
+  - numpy=1.12.0
   - openpyxl=2.4.0
   - python-dateutil
   - python-blosc

ci/travis-27.yaml

@@ -14,7 +14,7 @@ dependencies:
   - jemalloc=4.5.0.post
   - jinja2=2.8
   - lxml
-  - matplotlib
+  - matplotlib=2.2.2
   - mock
   - nomkl
   - numexpr

ci/travis-36.yaml

@@ -14,6 +14,7 @@ dependencies:
   - geopandas
   - html5lib
   - ipython
+  - isort
   - jinja2
   - lxml
   - matplotlib

ci/travis-37-numpydev.yaml

@@ -13,5 +13,5 @@ dependencies:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"
-    - "numpy"
+    - "numpy<=1.16.0.dev0+20181015190246"
     - "scipy"

doc/source/api.rst

@@ -245,6 +245,15 @@ Top-level evaluation
 
    eval
 
+Hashing
+~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   util.hash_array
+   util.hash_pandas_object
+
 Testing
 ~~~~~~~
 

doc/source/cookbook.rst

@@ -1228,36 +1228,40 @@ Correlation
 
 The `method` argument within `DataFrame.corr` can accept a callable in addition to the named correlation types.  Here we compute the `distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>`__ matrix for a `DataFrame` object.
 
-.. ipython:: python
-
-    def distcorr(x, y):
-        n = len(x)
-        a = np.zeros(shape=(n, n))
-        b = np.zeros(shape=(n, n))
-
-        for i in range(n):
-            for j in range(i + 1, n):
-                a[i, j] = abs(x[i] - x[j])
-                b[i, j] = abs(y[i] - y[j])
-
-        a += a.T
-        b += b.T
-
-        a_bar = np.vstack([np.nanmean(a, axis=0)] * n)
-        b_bar = np.vstack([np.nanmean(b, axis=0)] * n)
-
-        A = a - a_bar - a_bar.T + np.full(shape=(n, n), fill_value=a_bar.mean())
-        B = b - b_bar - b_bar.T + np.full(shape=(n, n), fill_value=b_bar.mean())
-
-        cov_ab = np.sqrt(np.nansum(A * B)) / n
-        std_a = np.sqrt(np.sqrt(np.nansum(A**2)) / n)
-        std_b = np.sqrt(np.sqrt(np.nansum(B**2)) / n)
-
-        return cov_ab / std_a / std_b
-
-    df = pd.DataFrame(np.random.normal(size=(100, 3)))
+.. code-block:: python
 
-    df.corr(method=distcorr)
+    >>> def distcorr(x, y):
+    ...     n = len(x)
+    ...     a = np.zeros(shape=(n, n))
+    ...     b = np.zeros(shape=(n, n))
+    ...
+    ...     for i in range(n):
+    ...         for j in range(i + 1, n):
+    ...             a[i, j] = abs(x[i] - x[j])
+    ...             b[i, j] = abs(y[i] - y[j])
+    ...
+    ...     a += a.T
+    ...     b += b.T
+    ...
+    ...     a_bar = np.vstack([np.nanmean(a, axis=0)] * n)
+    ...     b_bar = np.vstack([np.nanmean(b, axis=0)] * n)
+    ...
+    ...     A = a - a_bar - a_bar.T + np.full(shape=(n, n), fill_value=a_bar.mean())
+    ...     B = b - b_bar - b_bar.T + np.full(shape=(n, n), fill_value=b_bar.mean())
+    ...
+    ...     cov_ab = np.sqrt(np.nansum(A * B)) / n
+    ...     std_a = np.sqrt(np.sqrt(np.nansum(A**2)) / n)
+    ...     std_b = np.sqrt(np.sqrt(np.nansum(B**2)) / n)
+    ...
+    ...     return cov_ab / std_a / std_b
+    ...
+    >>> df = pd.DataFrame(np.random.normal(size=(100, 3)))
+    ...
+    >>> df.corr(method=distcorr)
+              0         1         2
+    0  1.000000  0.171368  0.145302
+    1  0.171368  1.000000  0.189919
+    2  0.145302  0.189919  1.000000
 
 Timedeltas
 ----------

doc/source/install.rst

@@ -225,7 +225,7 @@ Dependencies
 ------------
 
 * `setuptools <https://setuptools.readthedocs.io/en/latest/>`__: 24.2.0 or higher
-* `NumPy <http://www.numpy.org>`__: 1.9.0 or higher
+* `NumPy <http://www.numpy.org>`__: 1.12.0 or higher
 * `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__: 2.5.0 or higher
 * `pytz <http://pytz.sourceforge.net/>`__
 
@@ -236,11 +236,11 @@ Recommended Dependencies
 
 * `numexpr <https://github.com/pydata/numexpr>`__: for accelerating certain numerical operations.
   ``numexpr`` uses multiple cores as well as smart chunking and caching to achieve large speedups.
-  If installed, must be Version 2.4.6 or higher.
+  If installed, must be Version 2.6.1 or higher.
 
 * `bottleneck <https://github.com/kwgoodman/bottleneck>`__: for accelerating certain types of ``nan``
   evaluations. ``bottleneck`` uses specialized cython routines to achieve large speedups. If installed,
-  must be Version 1.0.0 or higher.
+  must be Version 1.2.0 or higher.
 
 .. note::
 
@@ -255,9 +255,9 @@ Optional Dependencies
 
 * `Cython <http://www.cython.org>`__: Only necessary to build development
   version. Version 0.28.2 or higher.
-* `SciPy <http://www.scipy.org>`__: miscellaneous statistical functions, Version 0.14.0 or higher
+* `SciPy <http://www.scipy.org>`__: miscellaneous statistical functions, Version 0.18.1 or higher
 * `xarray <http://xarray.pydata.org>`__: pandas like handling for > 2 dims, needed for converting Panels to xarray objects. Version 0.7.0 or higher is recommended.
-* `PyTables <http://www.pytables.org>`__: necessary for HDF5-based storage. Version 3.0.0 or higher required, Version 3.2.1 or higher highly recommended.
+* `PyTables <http://www.pytables.org>`__: necessary for HDF5-based storage, Version 3.4.2 or higher
 * `Feather Format <https://github.com/wesm/feather>`__: necessary for feather-based storage, version 0.3.1 or higher.
 * `Apache Parquet <https://parquet.apache.org/>`__, either `pyarrow <http://arrow.apache.org/docs/python/>`__ (>= 0.4.1) or `fastparquet <https://fastparquet.readthedocs.io/en/latest>`__ (>= 0.0.6) for parquet-based storage. The `snappy <https://pypi.org/project/python-snappy>`__ and `brotli <https://pypi.org/project/brotlipy>`__ are available for compression support.
 * `SQLAlchemy <http://www.sqlalchemy.org>`__: for SQL database support. Version 0.8.1 or higher recommended. Besides SQLAlchemy, you also need a database specific driver. You can find an overview of supported drivers for each SQL dialect in the `SQLAlchemy docs <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`__. Some common drivers are:
@@ -266,7 +266,7 @@ Optional Dependencies
     * `pymysql <https://github.com/PyMySQL/PyMySQL>`__: for MySQL.
     * `SQLite <https://docs.python.org/3/library/sqlite3.html>`__: for SQLite, this is included in Python's standard library by default.
 
-* `matplotlib <http://matplotlib.org/>`__: for plotting, Version 1.4.3 or higher.
+* `matplotlib <http://matplotlib.org/>`__: for plotting, Version 2.0.0 or higher.
 * For Excel I/O:
 
     * `xlrd/xlwt <http://www.python-excel.org/>`__: Excel reading (xlrd) and writing (xlwt)