Merge branch 'master' into integer-array-from-bool

Author: vladserkoff

asv_bench/benchmarks/io/stata.py

@@ -12,12 +12,12 @@ class Stata(BaseIO):
 
     def setup(self, convert_dates):
         self.fname = '__test__.dta'
-        N = 100000
-        C = 5
+        N = self.N = 100000
+        C = self.C = 5
         self.df = DataFrame(np.random.randn(N, C),
                             columns=['float{}'.format(i) for i in range(C)],
                             index=date_range('20000101', periods=N, freq='H'))
-        self.df['object'] = tm.makeStringIndex(N)
+        self.df['object'] = tm.makeStringIndex(self.N)
         self.df['int8_'] = np.random.randint(np.iinfo(np.int8).min,
                                              np.iinfo(np.int8).max - 27, N)
         self.df['int16_'] = np.random.randint(np.iinfo(np.int16).min,
@@ -36,4 +36,14 @@ def time_write_stata(self, convert_dates):
         self.df.to_stata(self.fname, self.convert_dates)
 
 
+class StataMissing(Stata):
+    def setup(self, convert_dates):
+        super(StataMissing, self).setup(convert_dates)
+        for i in range(10):
+            missing_data = np.random.randn(self.N)
+            missing_data[missing_data < 0] = np.nan
+            self.df['missing_{0}'.format(i)] = missing_data
+        self.df.to_stata(self.fname, self.convert_dates)
+
+
 from ..pandas_vb_common import setup  # noqa: F401

ci/azure/posix.yml

@@ -15,6 +15,18 @@ jobs:
           PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
+        py35_compat:
+          ENV_FILE: ci/deps/azure-35-compat.yaml
+          CONDA_PY: "35"
+          PATTERN: "not slow and not network"
+
+        py36_locale_slow_old_np:
+          ENV_FILE: ci/deps/azure-36-locale.yaml
+          CONDA_PY: "36"
+          PATTERN: "slow"
+          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          EXTRA_APT: "language-pack-zh-hans"
+
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"

ci/azure/windows.yml

@@ -12,6 +12,10 @@ jobs:
         ENV_FILE: ci/deps/azure-windows-36.yaml
         CONDA_PY: "36"
 
+      py37_np141:
+        ENV_FILE: ci/deps/azure-windows-37.yaml
+        CONDA_PY: "37"
+
   steps:
     - task: CondaEnvironment@1
       inputs:

ci/code_checks.sh

@@ -175,9 +175,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
-        ! grep -n '--exclude=*.'{svg,c,cpp,html} -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
     else
-        ! grep -n '--exclude=*.'{svg,c,cpp,html}  -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi

ci/deps/azure-35-compat.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4==4.4.1
+  - bottleneck=1.2.0
+  - cython=0.28.2
+  - hypothesis>=3.58.0
+  - jinja2=2.8
+  - numexpr=2.6.1
+  - numpy=1.12.0
+  - openpyxl=2.4.0
+  - pytables=3.4.2
+  - python-dateutil=2.5.0
+  - python=3.5*
+  - pytz=2015.4
+  - scipy=0.18.1
+  - xlrd=1.0.0
+  - xlsxwriter=0.7.7
+  - xlwt=1.0.0
+  # universal
+  - pytest-xdist
+  - pytest-mock
+  - isort
+  - pip:
+    # for python 3.5, pytest>=4.0.2 is not available in conda
+    - pytest>=4.0.2
+    - html5lib==1.0b2

ci/deps/azure-36-locale.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4==4.5.1
+  - bottleneck=1.2.0
+  - cython=0.28.2
+  - lxml
+  - matplotlib=2.0.0
+  - numpy=1.12.0
+  - openpyxl=2.4.0
+  - python-dateutil
+  - python-blosc
+  - python=3.6
+  - pytz=2016.10
+  - scipy
+  - sqlalchemy=1.1.4
+  - xlrd=1.0.0
+  - xlsxwriter=0.9.4
+  - xlwt=1.2.0
+  # universal
+  - pytest>=4.0.2
+  - pytest-xdist
+  - pytest-mock
+  - hypothesis>=3.58.0
+  - isort
+  - pip:
+    - html5lib==1.0b2

ci/deps/azure-macos-35.yaml

@@ -12,7 +12,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy=1.12.0
-  - openpyxl=2.5.5
+  - openpyxl
   - pyarrow
   - pytables
   - python=3.5*

ci/deps/azure-windows-37.yaml

@@ -0,0 +1,33 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - bottleneck
+  - gcsfs
+  - html5lib
+  - jinja2
+  - lxml
+  - matplotlib=3.0.1
+  - numexpr
+  - numpy=1.14.*
+  - openpyxl
+  - pytables
+  - python=3.7.*
+  - python-dateutil
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - cython>=0.28.2
+  - pytest>=4.0.2
+  - pytest-xdist
+  - pytest-mock
+  - moto
+  - hypothesis>=3.58.0
+  - isort

ci/deps/travis-36-locale.yaml

@@ -15,7 +15,7 @@ dependencies:
   - numpy
   - openpyxl
   - psycopg2
-  - pymysql
+  - pymysql=0.7.9
   - pytables
   - python-dateutil
   - python=3.6*

doc/cheatsheet/README.txt

@@ -1,8 +1,8 @@
 The Pandas Cheat Sheet was created using Microsoft Powerpoint 2013.
 To create the PDF version, within Powerpoint, simply do a "Save As"
-and pick "PDF' as the format.
+and pick "PDF" as the format.
 
-This cheat sheet was inspired by the RstudioData Wrangling Cheatsheet[1], written by Irv Lustig, Princeton Consultants[2].
+This cheat sheet was inspired by the RStudio Data Wrangling Cheatsheet[1], written by Irv Lustig, Princeton Consultants[2].
 
 [1]: https://www.rstudio.com/wp-content/uploads/2015/02/data-wrangling-cheatsheet.pdf
 [2]: http://www.princetonoptimization.com/

doc/source/install.rst

@@ -259,7 +259,7 @@ Optional Dependencies
 * `PyTables <http://www.pytables.org>`__: necessary for HDF5-based storage, Version 3.4.2 or higher
 * `pyarrow <http://arrow.apache.org/docs/python/>`__ (>= 0.9.0): necessary for feather-based storage.
 * `Apache Parquet <https://parquet.apache.org/>`__, either `pyarrow <http://arrow.apache.org/docs/python/>`__ (>= 0.7.0) or `fastparquet <https://fastparquet.readthedocs.io/en/latest>`__ (>= 0.2.1) for parquet-based storage. The `snappy <https://pypi.org/project/python-snappy>`__ and `brotli <https://pypi.org/project/brotlipy>`__ are available for compression support.
-* `SQLAlchemy <http://www.sqlalchemy.org>`__: for SQL database support. Version 1.0.8 or higher recommended. Besides SQLAlchemy, you also need a database specific driver. You can find an overview of supported drivers for each SQL dialect in the `SQLAlchemy docs <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`__. Some common drivers are:
+* `SQLAlchemy <http://www.sqlalchemy.org>`__: for SQL database support. Version 1.1.4 or higher recommended. Besides SQLAlchemy, you also need a database specific driver. You can find an overview of supported drivers for each SQL dialect in the `SQLAlchemy docs <http://docs.sqlalchemy.org/en/latest/dialects/index.html>`__. Some common drivers are:
 
     * `psycopg2 <http://initd.org/psycopg/>`__: for PostgreSQL
     * `pymysql <https://github.com/PyMySQL/PyMySQL>`__: for MySQL.

doc/source/whatsnew/v0.25.0.rst

@@ -123,13 +123,13 @@ If installed, we now require:
 +=================+=================+==========+
 | beautifulsoup4  | 4.4.1           |          |
 +-----------------+-----------------+----------+
-| openpyxl        | 2.2.6           |          |
+| openpyxl        | 2.4.0           |          |
 +-----------------+-----------------+----------+
-| pymysql         | 0.6.6           |          |
+| pymysql         | 0.7.9           |          |
 +-----------------+-----------------+----------+
 | pytz            | 2015.4          |          |
 +-----------------+-----------------+----------+
-| sqlalchemy      | 1.0.8           |          |
+| sqlalchemy      | 1.1.4           |          |
 +-----------------+-----------------+----------+
 | xlsxwriter      | 0.7.7           |          |
 +-----------------+-----------------+----------+
@@ -175,7 +175,8 @@ Performance Improvements
 - Improved performance of :meth:`Series.searchsorted`. The speedup is especially large when the dtype is
   int8/int16/int32 and the searched key is within the integer bounds for the dtype (:issue:`22034`)
 - Improved performance of :meth:`pandas.core.groupby.GroupBy.quantile` (:issue:`20405`)
-
+- Improved performance of :meth:`read_csv` by faster tokenizing and faster parsing of small float numbers (:issue:`25784`)
+- Improved performance of :meth:`read_csv` by faster parsing of N/A and boolean values (:issue:`25804`)
 
 .. _whatsnew_0250.bug_fixes:
 
@@ -208,9 +209,11 @@ Timedelta
 Timezones
 ^^^^^^^^^
 
+- Bug in :func:`DatetimeIndex.to_frame` where timezone aware data would be converted to timezone naive data (:issue:`25809`)
 - Bug in :func:`to_datetime` with ``utc=True`` and datetime strings that would apply previously parsed UTC offsets to subsequent arguments (:issue:`24992`)
 - Bug in :func:`Timestamp.tz_localize` and :func:`Timestamp.tz_convert` does not propagate ``freq`` (:issue:`25241`)
 - Bug in :func:`Series.at` where setting :class:`Timestamp` with timezone raises ``TypeError`` (:issue:`25506`)
+- Bug in :func:`DataFrame.update` when updating with timezone aware data would return timezone naive data (:issue:`25807`)
 
 Numeric
 ^^^^^^^
@@ -219,6 +222,7 @@ Numeric
 - Bug in :meth:`to_numeric` in which numbers were being coerced to float, even though ``errors`` was not ``coerce`` (:issue:`24910`)
 - Bug in error messages in :meth:`DataFrame.corr` and :meth:`Series.corr`. Added the possibility of using a callable. (:issue:`25729`)
 - Bug in :meth:`Series.divmod` and :meth:`Series.rdivmod` which would raise an (incorrect) ``ValueError`` rather than return a pair of :class:`Series` objects as result (:issue:`25557`)
+- Raises a helpful exception when a non-numeric index is sent to :meth:`interpolate` with methods which require numeric index. (:issue:`21662`)
 - Fixed bug where casting all-boolean array to integer extension array failed (:issue:`25211`)
 -
 -
@@ -242,7 +246,7 @@ Strings
 Interval
 ^^^^^^^^
 
--
+- Construction of :class:`Interval` is restricted to numeric, :class:`Timestamp` and :class:`Timedelta` endpoints (:issue:`23013`)
 -
 -
 
@@ -280,7 +284,7 @@ I/O
 - :meth:`DataFrame.to_html` now raises ``TypeError`` when using an invalid type for the ``classes`` parameter instead of ``AsseertionError`` (:issue:`25608`)
 - Bug in :meth:`DataFrame.to_string` and :meth:`DataFrame.to_latex` that would lead to incorrect output when the ``header`` keyword is used (:issue:`16718`)
 - Bug in :func:`read_csv` not properly interpreting the UTF8 encoded filenames on Windows on Python 3.6+ (:issue:`15086`)
--
+- Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`)
 
 
 Plotting

pandas/__init__.py

@@ -36,6 +36,9 @@
 
 from datetime import datetime
 
+from pandas._config import (get_option, set_option, reset_option,
+                            describe_option, option_context, options)
+
 # let init-time option registration happen
 import pandas.core.config_init
 

pandas/_config/__init__.py

@@ -2,3 +2,9 @@
 pandas._config is considered explicitly upstream of everything else in pandas,
 should have no intra-pandas dependencies.
 """
+__all__ = ["config", "get_option", "set_option", "reset_option",
+           "describe_option", "option_context", "options"]
+from pandas._config import config
+from pandas._config.config import (
+    describe_option, get_option, option_context, options, reset_option,
+    set_option)

pandas/core/config.py renamed to pandas/_config/config.py

@@ -695,7 +695,7 @@ def config_prefix(prefix):
 
     Example:
 
-    import pandas.core.config as cf
+    import pandas._config.config as cf
     with cf.config_prefix("display.font"):
         cf.register_option("color", "red")
         cf.register_option("size", " 5 pt")

pandas/_libs/interval.pyx

@@ -20,8 +20,10 @@ cnp.import_array()
 cimport pandas._libs.util as util
 
 from pandas._libs.hashtable cimport Int64Vector, Int64VectorData
+from pandas._libs.tslibs.util cimport is_integer_object, is_float_object
 
 from pandas._libs.tslibs import Timestamp
+from pandas._libs.tslibs.timedeltas import Timedelta
 from pandas._libs.tslibs.timezones cimport tz_compare
 
 
@@ -104,12 +106,7 @@ cdef class IntervalMixin(object):
     @property
     def length(self):
         """Return the length of the Interval"""
-        try:
-            return self.right - self.left
-        except TypeError:
-            # length not defined for some types, e.g. string
-            msg = 'cannot compute length between {left!r} and {right!r}'
-            raise TypeError(msg.format(left=self.left, right=self.right))
+        return self.right - self.left
 
     def _check_closed_matches(self, other, name='other'):
         """Check if the closed attribute of `other` matches.
@@ -250,6 +247,10 @@ cdef class Interval(IntervalMixin):
     def __init__(self, left, right, str closed='right'):
         # note: it is faster to just do these checks than to use a special
         # constructor (__cinit__/__new__) to avoid them
+
+        self._validate_endpoint(left)
+        self._validate_endpoint(right)
+
         if closed not in _VALID_CLOSED:
             msg = "invalid option for 'closed': {closed}".format(closed=closed)
             raise ValueError(msg)
@@ -266,6 +267,14 @@ cdef class Interval(IntervalMixin):
         self.right = right
         self.closed = closed
 
+    def _validate_endpoint(self, endpoint):
+        # GH 23013
+        if not (is_integer_object(endpoint) or is_float_object(endpoint) or
+                isinstance(endpoint, (Timestamp, Timedelta))):
+            msg = ("Only numeric, Timestamp and Timedelta endpoints "
+                   "are allowed when constructing an Interval.")
+            raise ValueError(msg)
+
     def __hash__(self):
         return hash((self.left, self.right, self.closed))
 

pandas/_libs/khash.pxd

@@ -56,6 +56,17 @@ cdef extern from "khash_python.h":
 
     bint kh_exist_str(kh_str_t*, khiter_t) nogil
 
+    ctypedef struct kh_str_starts_t:
+        kh_str_t *table
+        int starts[256]
+
+    kh_str_starts_t* kh_init_str_starts() nogil
+    khint_t kh_put_str_starts_item(kh_str_starts_t* table, char* key,
+                                   int* ret) nogil
+    khint_t kh_get_str_starts_item(kh_str_starts_t* table, char* key) nogil
+    void kh_destroy_str_starts(kh_str_starts_t*) nogil
+    void kh_resize_str_starts(kh_str_starts_t*, khint_t) nogil
+
     ctypedef struct kh_int64_t:
         khint_t n_buckets, size, n_occupied, upper_bound
         uint32_t *flags