Fetch latest whatsnew edit from upstream master

Author: Marco Gorelli

doc/source/user_guide/io.rst

@@ -28,6 +28,7 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :delim: ;
 
     text;`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__;:ref:`read_csv<io.read_csv_table>`;:ref:`to_csv<io.store_in_csv>`
+    text;`TXT <https://www.oracle.com/webfolder/technetwork/data-quality/edqhelp/Content/introduction/getting_started/configuring_fixed_width_text_file_formats.htm>`__;:ref:`read_fwf<io.fwf_reader>`
     text;`JSON <https://www.json.org/>`__;:ref:`read_json<io.json_reader>`;:ref:`to_json<io.json_writer>`
     text;`HTML <https://en.wikipedia.org/wiki/HTML>`__;:ref:`read_html<io.read_html>`;:ref:`to_html<io.html>`
     text; Local clipboard;:ref:`read_clipboard<io.clipboard>`;:ref:`to_clipboard<io.clipboard>`
@@ -1372,6 +1373,7 @@ should pass the ``escapechar`` option:
    print(data)
    pd.read_csv(StringIO(data), escapechar='\\')
 
+.. _io.fwf_reader:
 .. _io.fwf:
 
 Files with fixed width columns

doc/source/whatsnew/v0.25.1.rst

@@ -31,7 +31,7 @@ Categorical
 Datetimelike
 ^^^^^^^^^^^^
 - Bug in :func:`to_datetime` where passing a timezone-naive :class:`DatetimeArray` or :class:`DatetimeIndex` and ``utc=True`` would incorrectly return a timezone-naive result (:issue:`27733`)
--
+- Bug in :meth:`Period.to_timestamp` where a :class:`Period` outside the :class:`Timestamp` implementation bounds (roughly 1677-09-21 to 2262-04-11) would return an incorrect :class:`Timestamp` instead of raising ``OutOfBoundsDatetime`` (:issue:`19643`)
 -
 -
 
@@ -53,7 +53,7 @@ Numeric
 ^^^^^^^
 - Bug in :meth:`Series.interpolate` when using a timezone aware :class:`DatetimeIndex` (:issue:`27548`)
 - Bug when printing negative floating point complex numbers would raise an ``IndexError`` (:issue:`27484`)
--
+- Bug where :class:`DataFrame` arithmetic operators such as :meth:`DataFrame.mul` with a :class:`Series` with axis=1 would raise an ``AttributeError`` on :class:`DataFrame` larger than the minimum threshold to invoke numexpr (:issue:`27636`)
 -
 
 Conversion
@@ -84,6 +84,7 @@ Indexing
 - Bug in partial-string indexing returning a NumPy array rather than a ``Series`` when indexing with a scalar like ``.loc['2015']`` (:issue:`27516`)
 - Break reference cycle involving :class:`Index` and other index classes to allow garbage collection of index objects without running the GC. (:issue:`27585`, :issue:`27840`)
 - Fix regression in assigning values to a single column of a DataFrame with a ``MultiIndex`` columns (:issue:`27841`).
+- Fix regression in ``.ix`` fallback with an ``IntervalIndex`` (:issue:`27865`).
 - When using :meth:`DataFrame.explode`, don't duplicate entire exploded column when joining back with original frame (:issue:`28005`).
 
 Missing
@@ -102,7 +103,6 @@ MultiIndex
 
 I/O
 ^^^
-
 - Avoid calling ``S3File.s3`` when reading parquet, as this was removed in s3fs version 0.3.0 (:issue:`27756`)
 - Better error message when a negative header is passed in :func:`pandas.read_csv` (:issue:`27779`)
 -
@@ -159,6 +159,14 @@ Other
 -
 -
 
+I/O and LZMA
+~~~~~~~~~~~~
+
+Some users may unknowingly have an incomplete Python installation, which lacks the `lzma` module from the standard library. In this case, `import pandas` failed due to an `ImportError` (:issue: `27575`).
+Pandas will now warn, rather than raising an `ImportError` if the `lzma` module is not present. Any subsequent attempt to use `lzma` methods will raise a `RuntimeError`.
+A possible fix for the lack of the `lzma` module is to ensure you have the necessary libraries and then re-install Python.
+For example, on MacOS installing Python with `pyenv` may lead to an incomplete Python installation due to unmet system dependencies at compilation time (like `xz`). Compilation will succeed, but Python might fail at run time. The issue can be solved by installing the necessary dependencies and then re-installing Python.
+
 .. _whatsnew_0.251.contributors:
 
 Contributors

doc/source/whatsnew/v1.0.0.rst

@@ -158,7 +158,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`)
 -
 
 Plotting

pandas/_libs/parsers.pyx

@@ -2,7 +2,6 @@
 # See LICENSE for the license
 import bz2
 import gzip
-import lzma
 import os
 import sys
 import time
@@ -59,9 +58,12 @@ from pandas.core.arrays import Categorical
 from pandas.core.dtypes.concat import union_categoricals
 import pandas.io.common as icom
 
+from pandas.compat import _import_lzma, _get_lzma_file
 from pandas.errors import (ParserError, DtypeWarning,
                            EmptyDataError, ParserWarning)
 
+lzma = _import_lzma()
+
 # Import CParserError as alias of ParserError for backwards compatibility.
 # Ultimately, we want to remove this import. See gh-12665 and gh-14479.
 CParserError = ParserError
@@ -645,9 +647,9 @@ cdef class TextReader:
                                      'zip file %s', str(zip_names))
             elif self.compression == 'xz':
                 if isinstance(source, str):
-                    source = lzma.LZMAFile(source, 'rb')
+                    source = _get_lzma_file(lzma)(source, 'rb')
                 else:
-                    source = lzma.LZMAFile(filename=source)
+                    source = _get_lzma_file(lzma)(filename=source)
             else:
                 raise ValueError('Unrecognized compression type: %s' %
                                  self.compression)

pandas/_libs/tslibs/period.pyx

@@ -21,7 +21,8 @@ PyDateTime_IMPORT
 
 from pandas._libs.tslibs.np_datetime cimport (
     npy_datetimestruct, dtstruct_to_dt64, dt64_to_dtstruct,
-    pandas_datetime_to_datetimestruct, NPY_DATETIMEUNIT, NPY_FR_D)
+    pandas_datetime_to_datetimestruct, check_dts_bounds,
+    NPY_DATETIMEUNIT, NPY_FR_D)
 
 cdef extern from "src/datetime/np_datetime.h":
     int64_t npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
@@ -1011,7 +1012,7 @@ def dt64arr_to_periodarr(int64_t[:] dtarr, int freq, tz=None):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def periodarr_to_dt64arr(int64_t[:] periodarr, int freq):
+def periodarr_to_dt64arr(const int64_t[:] periodarr, int freq):
     """
     Convert array to datetime64 values from a set of ordinals corresponding to
     periods per period convention.
@@ -1024,9 +1025,8 @@ def periodarr_to_dt64arr(int64_t[:] periodarr, int freq):
 
     out = np.empty(l, dtype='i8')
 
-    with nogil:
-        for i in range(l):
-            out[i] = period_ordinal_to_dt64(periodarr[i], freq)
+    for i in range(l):
+        out[i] = period_ordinal_to_dt64(periodarr[i], freq)
 
     return out.base  # .base to access underlying np.ndarray
 
@@ -1179,14 +1179,15 @@ cpdef int64_t period_ordinal(int y, int m, int d, int h, int min,
     return get_period_ordinal(&dts, freq)
 
 
-cpdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq) nogil:
+cdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq) except? -1:
     cdef:
         npy_datetimestruct dts
 
     if ordinal == NPY_NAT:
         return NPY_NAT
 
     get_date_info(ordinal, freq, &dts)
+    check_dts_bounds(&dts)
     return dtstruct_to_dt64(&dts)
 
 

pandas/compat/__init__.py

@@ -10,6 +10,7 @@
 import platform
 import struct
 import sys
+import warnings
 
 PY35 = sys.version_info[:2] == (3, 5)
 PY36 = sys.version_info >= (3, 6)
@@ -65,3 +66,32 @@ def is_platform_mac():
 
 def is_platform_32bit():
     return struct.calcsize("P") * 8 < 64
+
+
+def _import_lzma():
+    """Attempts to import lzma, warning the user when lzma is not available.
+    """
+    try:
+        import lzma
+
+        return lzma
+    except ImportError:
+        msg = (
+            "Could not import the lzma module. "
+            "Your installed Python is incomplete. "
+            "Attempting to use lzma compression will result in a RuntimeError."
+        )
+        warnings.warn(msg)
+
+
+def _get_lzma_file(lzma):
+    """Returns the lzma method LZMAFile when the module was correctly imported.
+    Otherwise, raises a RuntimeError.
+    """
+    if lzma is None:
+        raise RuntimeError(
+            "lzma module not available. "
+            "A Python re-install with the proper "
+            "dependencies might be required to solve this issue."
+        )
+    return lzma.LZMAFile

pandas/core/arrays/sparse.py

@@ -39,6 +39,7 @@
 )
 from pandas.core.dtypes.dtypes import register_extension_dtype
 from pandas.core.dtypes.generic import (
+    ABCDataFrame,
     ABCIndexClass,
     ABCSeries,
     ABCSparseArray,
@@ -1735,13 +1736,15 @@ def sparse_unary_method(self):
 
     @classmethod
     def _create_arithmetic_method(cls, op):
-        def sparse_arithmetic_method(self, other):
-            op_name = op.__name__
+        op_name = op.__name__
 
-            if isinstance(other, (ABCSeries, ABCIndexClass)):
+        def sparse_arithmetic_method(self, other):
+            if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
                 # Rely on pandas to dispatch to us.
                 return NotImplemented
 
+            other = lib.item_from_zerodim(other)
+
             if isinstance(other, SparseArray):
                 return _sparse_array_op(self, other, op, op_name)
 

pandas/core/computation/expressions.py

@@ -76,16 +76,17 @@ def _can_use_numexpr(op, op_str, a, b, dtype_check):
 
         # required min elements (otherwise we are adding overhead)
         if np.prod(a.shape) > _MIN_ELEMENTS:
-
             # check for dtype compatibility
             dtypes = set()
             for o in [a, b]:
-                if hasattr(o, "dtypes"):
+                # Series implements dtypes, check for dimension count as well
+                if hasattr(o, "dtypes") and o.ndim > 1:
                     s = o.dtypes.value_counts()
                     if len(s) > 1:
                         return False
                     dtypes |= set(s.index.astype(str))
-                elif isinstance(o, np.ndarray):
+                # ndarray and Series Case
+                elif hasattr(o, "dtype"):
                     dtypes |= {o.dtype.name}
 
             # allowed are a superset

pandas/core/indexes/base.py

@@ -2325,7 +2325,10 @@ def __sub__(self, other):
         return Index(np.array(self) - other)
 
     def __rsub__(self, other):
-        return Index(other - np.array(self))
+        # wrap Series to ensure we pin name correctly
+        from pandas import Series
+
+        return Index(other - Series(self))
 
     def __and__(self, other):
         return self.intersection(other)

pandas/core/indexing.py

@@ -124,14 +124,17 @@ def __getitem__(self, key):
             key = tuple(com.apply_if_callable(x, self.obj) for x in key)
             try:
                 values = self.obj._get_value(*key)
-            except (KeyError, TypeError, InvalidIndexError):
+            except (KeyError, TypeError, InvalidIndexError, AttributeError):
                 # TypeError occurs here if the key has non-hashable entries,
                 #  generally slice or list.
                 # TODO(ix): most/all of the TypeError cases here are for ix,
                 #  so this check can be removed once ix is removed.
                 # The InvalidIndexError is only catched for compatibility
                 #  with geopandas, see
                 #  https://github.com/pandas-dev/pandas/issues/27258
+                # TODO: The AttributeError is for IntervalIndex which
+                #  incorrectly implements get_value, see
+                #  https://github.com/pandas-dev/pandas/issues/27865
                 pass
             else:
                 if is_scalar(values):

pandas/core/ops/__init__.py

@@ -17,9 +17,7 @@
 from pandas.core.dtypes.common import (
     ensure_object,
     is_bool_dtype,
-    is_categorical_dtype,
     is_datetime64_dtype,
-    is_datetime64tz_dtype,
     is_datetimelike_v_numeric,
     is_extension_array_dtype,
     is_integer_dtype,
@@ -32,6 +30,7 @@
     ABCDataFrame,
     ABCDatetimeArray,
     ABCDatetimeIndex,
+    ABCExtensionArray,
     ABCIndexClass,
     ABCSeries,
     ABCSparseSeries,
@@ -699,42 +698,17 @@ def wrapper(self, other, axis=None):
 
         if isinstance(other, ABCSeries) and not self._indexed_same(other):
             raise ValueError("Can only compare identically-labeled Series objects")
-        elif (
-            is_list_like(other)
-            and len(other) != len(self)
-            and not isinstance(other, (set, frozenset))
-        ):
-            raise ValueError("Lengths must match")
 
-        elif isinstance(other, (np.ndarray, ABCIndexClass, ABCSeries)):
+        elif isinstance(
+            other, (np.ndarray, ABCExtensionArray, ABCIndexClass, ABCSeries)
+        ):
             # TODO: make this treatment consistent across ops and classes.
             #  We are not catching all listlikes here (e.g. frozenset, tuple)
             #  The ambiguous case is object-dtype.  See GH#27803
             if len(self) != len(other):
                 raise ValueError("Lengths must match to compare")
 
-        if is_categorical_dtype(self):
-            # Dispatch to Categorical implementation; CategoricalIndex
-            # behavior is non-canonical GH#19513
-            res_values = dispatch_to_extension_op(op, self, other)
-
-        elif is_datetime64_dtype(self) or is_datetime64tz_dtype(self):
-            # Dispatch to DatetimeIndex to ensure identical
-            # Series/Index behavior
-            from pandas.core.arrays import DatetimeArray
-
-            res_values = dispatch_to_extension_op(op, DatetimeArray(self), other)
-
-        elif is_timedelta64_dtype(self):
-            from pandas.core.arrays import TimedeltaArray
-
-            res_values = dispatch_to_extension_op(op, TimedeltaArray(self), other)
-
-        elif is_extension_array_dtype(self) or (
-            is_extension_array_dtype(other) and not is_scalar(other)
-        ):
-            # Note: the `not is_scalar(other)` condition rules out
-            #  e.g. other == "category"
+        if should_extension_dispatch(self, other):
             res_values = dispatch_to_extension_op(op, self, other)
 
         elif is_scalar(other) and isna(other):
@@ -756,9 +730,12 @@ def wrapper(self, other, axis=None):
                 )
 
         result = self._constructor(res_values, index=self.index)
-        # rename is needed in case res_name is None and result.name
-        #  is not.
-        return finalizer(result).rename(res_name)
+        result = finalizer(result)
+
+        # Set the result's name after finalizer is called because finalizer
+        #  would set it back to self.name
+        result.name = res_name
+        return result
 
     wrapper.__name__ = op_name
     return wrapper

pandas/core/ops/array_ops.py

@@ -74,8 +74,9 @@ def masked_arith_op(x, y, op):
                 result[mask] = op(xrav[mask], yrav[mask])
 
     else:
-        assert is_scalar(y), type(y)
-        assert isinstance(x, np.ndarray), type(x)
+        if not is_scalar(y):
+            raise TypeError(type(y))
+
         # mask is only meaningful for x
         result = np.empty(x.size, dtype=x.dtype)
         mask = notna(xrav)