MERGE: resolve master merge conflicts by accepting master

Author: AlexKirko

doc/source/user_guide/io.rst

@@ -4683,6 +4683,7 @@ See the documentation for `pyarrow <https://arrow.apache.org/docs/python/>`__ an
 Write to a parquet file.
 
 .. ipython:: python
+   :okwarning:
 
    df.to_parquet('example_pa.parquet', engine='pyarrow')
    df.to_parquet('example_fp.parquet', engine='fastparquet')

doc/source/whatsnew/v1.0.0.rst

@@ -204,6 +204,7 @@ Other enhancements
 - Roundtripping DataFrames with nullable integer or string data types to parquet
   (:meth:`~DataFrame.to_parquet` / :func:`read_parquet`) using the `'pyarrow'` engine
   now preserve those data types with pyarrow >= 1.0.0 (:issue:`20612`).
+- The ``partition_cols`` argument in :meth:`DataFrame.to_parquet` now accepts a string (:issue:`27117`)
 
 Build Changes
 ^^^^^^^^^^^^^
@@ -432,6 +433,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | pytables        | 3.4.2           |         |
 +-----------------+-----------------+---------+
+| s3fs            | 0.3.0           |    X    |
++-----------------+-----------------+---------+
 | scipy           | 0.19.0          |         |
 +-----------------+-----------------+---------+
 | sqlalchemy      | 1.1.4           |         |
@@ -453,7 +456,7 @@ See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for mor
 Other API changes
 ^^^^^^^^^^^^^^^^^
 
-- Bumpded the minimum supported version of ``s3fs`` from 0.0.8 to 0.3.0 (:issue:`28616`)
+- Bumped the minimum supported version of ``s3fs`` from 0.0.8 to 0.3.0 (:issue:`28616`)
 - :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`)
 - :meth:`pandas.api.types.infer_dtype` will now return "integer-na" for integer and ``np.nan`` mix (:issue:`27283`)
 - :meth:`MultiIndex.from_arrays` will no longer infer names from arrays if ``names=None`` is explicitly provided (:issue:`27292`)
@@ -491,6 +494,7 @@ Deprecations
 - :meth:`Categorical.take_nd` is deprecated, use :meth:`Categorical.take` instead (:issue:`27745`)
 - The parameter ``numeric_only`` of :meth:`Categorical.min` and :meth:`Categorical.max` is deprecated and replaced with ``skipna`` (:issue:`25303`)
 - The parameter ``label`` in :func:`lreshape` has been deprecated and will be removed in a future version (:issue:`29742`)
+- ``pandas.core.index`` has been deprecated and will be removed in a future version, the public classes are available in the top-level namespace (:issue:`19711`)
 -
 
 .. _whatsnew_1000.prior_deprecations:
@@ -697,6 +701,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.strftime` and :meth:`Series.dt.strftime` where ``NaT`` was converted to the string ``'NaT'`` instead of ``np.nan`` (:issue:`29578`)
 - Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`)
 - Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`)
+- Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`)
 - Bug in :func:`pandas.to_datetime` when called with ``Series`` storing ``IntegerArray`` raising ``TypeError`` instead of returning ``Series`` (:issue:`30050`)
 
 Timedelta
@@ -872,7 +877,8 @@ Other
 - Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`)
 - Fix :class:`AbstractHolidayCalendar` to return correct results for
   years after 2030 (now goes up to 2200) (:issue:`27790`)
-- Fixed :class:`IntegerArray` returning ``NA`` rather than ``inf`` for operations dividing by 0 (:issue:`27398`)
+- Fixed :class:`IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by 0 (:issue:`27398`)
+- Fixed ``pow`` operations for :class:`IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`)
 - Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`)
 
 

pandas/_libs/internals.pyx

@@ -96,7 +96,7 @@ cdef class BlockPlacement:
             return self._as_array
 
     def isin(self, arr):
-        from pandas.core.index import Int64Index
+        from pandas.core.indexes.api import Int64Index
         return Int64Index(self.as_array, copy=False).isin(arr)
 
     @property

pandas/_libs/lib.pyx

@@ -598,7 +598,7 @@ def astype_str(arr: ndarray, skipna: bool=False) -> ndarray[object]:
 @cython.boundscheck(False)
 def clean_index_list(obj: list):
     """
-    Utility used in ``pandas.core.index.ensure_index``.
+    Utility used in ``pandas.core.indexes.api.ensure_index``.
     """
     cdef:
         Py_ssize_t i, n = len(obj)

pandas/_libs/skiplist.pxd

@@ -22,9 +22,27 @@ from pandas._libs.algos import is_monotonic
 
 from pandas._libs.util cimport numeric
 
-from pandas._libs.skiplist cimport (
-    skiplist_t, skiplist_init, skiplist_destroy, skiplist_get, skiplist_insert,
-    skiplist_remove)
+cdef extern from "../src/skiplist.h":
+    ctypedef struct node_t:
+        node_t **next
+        int *width
+        double value
+        int is_nil
+        int levels
+        int ref_count
+
+    ctypedef struct skiplist_t:
+        node_t *head
+        node_t **tmp_chain
+        int *tmp_steps
+        int size
+        int maxlevels
+
+    skiplist_t* skiplist_init(int) nogil
+    void skiplist_destroy(skiplist_t*) nogil
+    double skiplist_get(skiplist_t*, int, int*) nogil
+    int skiplist_insert(skiplist_t*, double) nogil
+    int skiplist_remove(skiplist_t*, double) nogil
 
 cdef:
     float32_t MINfloat32 = np.NINF

pandas/_libs/window/aggregations.pyx

@@ -2,6 +2,9 @@
 
 import numpy as np
 
+from pandas._libs import NaT, Period, Timedelta, Timestamp
+from pandas._libs.missing import NA
+
 from pandas.core.dtypes.dtypes import (
     CategoricalDtype,
     DatetimeTZDtype,
@@ -26,24 +29,23 @@
 from pandas.core.arrays.string_ import StringDtype
 from pandas.core.construction import array
 from pandas.core.groupby import Grouper, NamedAgg
-from pandas.core.index import (
+from pandas.core.indexes.api import (
     CategoricalIndex,
     DatetimeIndex,
     Float64Index,
     Index,
     Int64Index,
     IntervalIndex,
     MultiIndex,
-    NaT,
     PeriodIndex,
     RangeIndex,
     TimedeltaIndex,
     UInt64Index,
 )
-from pandas.core.indexes.datetimes import Timestamp, bdate_range, date_range
+from pandas.core.indexes.datetimes import bdate_range, date_range
 from pandas.core.indexes.interval import Interval, interval_range
-from pandas.core.indexes.period import Period, period_range
-from pandas.core.indexes.timedeltas import Timedelta, timedelta_range
+from pandas.core.indexes.period import period_range
+from pandas.core.indexes.timedeltas import timedelta_range
 from pandas.core.indexing import IndexSlice
 from pandas.core.series import Series
 from pandas.core.tools.datetimes import to_datetime
@@ -55,5 +57,3 @@
 
 # DataFrame needs to be imported after NamedAgg to avoid a circular import
 from pandas.core.frame import DataFrame  # isort:skip
-
-from pandas._libs.missing import NA

pandas/core/api.py

@@ -320,7 +320,6 @@ def apply_series_generator(self) -> Tuple[ResType, "Index"]:
         series_gen = self.series_generator
         res_index = self.result_index
 
-        i = None
         keys = []
         results = {}
         if self.ignore_failures:

pandas/core/apply.py

@@ -179,7 +179,6 @@ def _generate_range_overflow_safe_signed(
             # watch out for very special case in which we just slightly
             #  exceed implementation bounds, but when passing the result to
             #  np.arange will get a result slightly within the bounds
-            assert endpoint >= 0
             result = np.uint64(endpoint) + np.uint64(addend)
             i64max = np.uint64(np.iinfo(np.int64).max)
             assert result > i64max

pandas/core/arrays/_ranges.py

@@ -2109,7 +2109,6 @@ def _validate_dt64_dtype(dtype):
         dtype = pandas_dtype(dtype)
         if is_dtype_equal(dtype, np.dtype("M8")):
             # no precision, disallowed GH#24806
-            dtype = _NS_DTYPE
             msg = (
                 "Passing in 'datetime64' dtype with no precision is not allowed. "
                 "Please pass in 'datetime64[ns]' instead."

pandas/core/arrays/datetimes.py

@@ -78,7 +78,8 @@ def itemsize(self):
 
     @classmethod
     def construct_array_type(cls):
-        """Return the array type associated with this dtype
+        """
+        Return the array type associated with this dtype.
 
         Returns
         -------
@@ -718,13 +719,13 @@ def _create_arithmetic_method(cls, op):
         @unpack_zerodim_and_defer(op.__name__)
         def integer_arithmetic_method(self, other):
 
-            mask = None
+            omask = None
 
             if getattr(other, "ndim", 0) > 1:
                 raise NotImplementedError("can only perform ops with 1-d structures")
 
             if isinstance(other, IntegerArray):
-                other, mask = other._data, other._mask
+                other, omask = other._data, other._mask
 
             elif is_list_like(other):
                 other = np.asarray(other)
@@ -742,17 +743,28 @@ def integer_arithmetic_method(self, other):
                     raise TypeError("can only perform ops with numeric values")
 
             # nans propagate
-            if mask is None:
+            if omask is None:
                 mask = self._mask.copy()
             else:
-                mask = self._mask | mask
+                mask = self._mask | omask
 
-            # 1 ** np.nan is 1. So we have to unmask those.
             if op_name == "pow":
-                mask = np.where(self == 1, False, mask)
+                # 1 ** x is 1.
+                mask = np.where((self._data == 1) & ~self._mask, False, mask)
+                # x ** 0 is 1.
+                if omask is not None:
+                    mask = np.where((other == 0) & ~omask, False, mask)
+                else:
+                    mask = np.where(other == 0, False, mask)
 
             elif op_name == "rpow":
-                mask = np.where(other == 1, False, mask)
+                # 1 ** x is 1.
+                if omask is not None:
+                    mask = np.where((other == 1) & ~omask, False, mask)
+                else:
+                    mask = np.where(other == 1, False, mask)
+                # x ** 0 is 1.
+                mask = np.where((self._data == 0) & ~self._mask, False, mask)
 
             with np.errstate(all="ignore"):
                 result = op(self._data, other)

pandas/core/arrays/integer.py

@@ -79,7 +79,15 @@ def construct_from_string(cls, string):
                 f"Cannot construct a 'PandasDtype' from '{string}'"
             ) from err
 
+    @classmethod
     def construct_array_type(cls):
+        """
+        Return the array type associated with this dtype.
+
+        Returns
+        -------
+        type
+        """
         return PandasArray
 
     @property

pandas/core/arrays/numpy_.py

@@ -168,6 +168,13 @@ def __repr__(self) -> str:
 
     @classmethod
     def construct_array_type(cls):
+        """
+        Return the array type associated with this dtype.
+
+        Returns
+        -------
+        type
+        """
         from .array import SparseArray
 
         return SparseArray

pandas/core/arrays/sparse/dtype.py

@@ -5,7 +5,7 @@
 """
 from collections import OrderedDict
 
-from pandas.core.index import Index, MultiIndex
+from pandas.core.indexes.api import Index, MultiIndex
 from pandas.core.series import Series
 
 

pandas/core/arrays/sparse/scipy_sparse.py

@@ -1122,7 +1122,6 @@ def _validate_td64_dtype(dtype):
     dtype = pandas_dtype(dtype)
     if is_dtype_equal(dtype, np.dtype("timedelta64")):
         # no precision disallowed GH#24806
-        dtype = _TD_DTYPE
         msg = (
             "Passing in 'timedelta' dtype with no precision is not allowed. "
             "Please pass in 'timedelta64[ns]' instead."

pandas/core/arrays/timedeltas.py

@@ -309,7 +309,6 @@ def _aggregate(self, arg, *args, **kwargs):
         None if not required
         """
         is_aggregator = lambda x: isinstance(x, (list, tuple, dict))
-        is_nested_renamer = False
 
         _axis = kwargs.pop("_axis", None)
         if _axis is None:
@@ -398,24 +397,7 @@ def _agg(arg, func):
             keys = list(arg.keys())
             result = OrderedDict()
 
-            # nested renamer
-            if is_nested_renamer:
-                result = list(_agg(arg, _agg_1dim).values())
-
-                if all(isinstance(r, dict) for r in result):
-
-                    result, results = OrderedDict(), result
-                    for r in results:
-                        result.update(r)
-                    keys = list(result.keys())
-
-                else:
-
-                    if self._selection is not None:
-                        keys = None
-
-            # some selection on the object
-            elif self._selection is not None:
+            if self._selection is not None:
 
                 sl = set(self._selection_list)
 

pandas/core/base.py

@@ -49,7 +49,7 @@
 
 if TYPE_CHECKING:
     from pandas.core.series import Series  # noqa: F401
-    from pandas.core.index import Index  # noqa: F401
+    from pandas.core.indexes.api import Index  # noqa: F401
 
 
 def array(

pandas/core/construction.py

@@ -183,7 +183,7 @@ def names(self) -> Optional[List[str]]:
     @classmethod
     def construct_array_type(cls):
         """
-        Return the array type associated with this dtype
+        Return the array type associated with this dtype.
 
         Returns
         -------