Merge remote-tracking branch 'upstream/master'

Author: ccharlesgb

.github/FUNDING.yml

@@ -1 +1,2 @@
 custom: https://pandas.pydata.org/donate.html
+tidelift: pypi/pandas

.github/SECURITY.md

@@ -0,0 +1 @@
+To report a security vulnerability to pandas, please go to https://tidelift.com/security and see the instructions there.

ci/deps/azure-37-locale.yaml

@@ -17,7 +17,7 @@ dependencies:
   - openpyxl
   - pytables
   - python-dateutil
-  - python=3.7.*
+  - python=3.7.3
   - pytz
   - s3fs
   - scipy
@@ -26,8 +26,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - pip

ci/deps/azure-37-numpydev.yaml

@@ -2,11 +2,12 @@ name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.7.*
+  - python=3.7.3
   - pytz
   - Cython>=0.28.2
   # universal
-  - pytest>=4.0.2
+  # pytest < 5 until defaults has pytest-xdist>=1.29.0
+  - pytest>=4.0.2,<5.0
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/azure-macos-35.yaml

@@ -25,8 +25,8 @@ dependencies:
   - pip:
     - pyreadstat
     # universal
-    - pytest==4.5.0
-    - pytest-xdist
+    - pytest>=5.0.1
+    - pytest-xdist>=1.29.0
     - pytest-mock
     - hypothesis>=3.58.0
     # https://github.com/pandas-dev/pandas/issues/27421

ci/deps/azure-windows-36.yaml

@@ -23,8 +23,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/azure-windows-37.yaml

@@ -26,8 +26,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/travis-36-cov.yaml

@@ -39,8 +39,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-cov
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/travis-37.yaml

@@ -4,7 +4,7 @@ channels:
   - conda-forge
   - c3i_test
 dependencies:
-  - python=3.7.*
+  - python=3.7.3
   - botocore>=1.11
   - cython>=0.28.2
   - numpy
@@ -13,8 +13,8 @@ dependencies:
   - pyarrow
   - pytz
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - hypothesis>=3.58.0
   - s3fs

doc/source/reference/extensions.rst

@@ -44,6 +44,7 @@ objects.
       api.extensions.ExtensionArray.argsort
       api.extensions.ExtensionArray.astype
       api.extensions.ExtensionArray.copy
+      api.extensions.ExtensionArray.view
       api.extensions.ExtensionArray.dropna
       api.extensions.ExtensionArray.factorize
       api.extensions.ExtensionArray.fillna

doc/source/whatsnew/v0.25.1.rst

@@ -104,23 +104,23 @@ MultiIndex
 I/O
 ^^^
 
--
+- Avoid calling ``S3File.s3`` when reading parquet, as this was removed in s3fs version 0.3.0 (:issue:`27756`)
 -
 -
 
 Plotting
 ^^^^^^^^
 
 - Added a pandas_plotting_backends entrypoint group for registering plot backends. See :ref:`extending.plotting-backends` for more (:issue:`26747`).
--
+- Fix compatibility issue with matplotlib when passing a pandas ``Index`` to a plot call (:issue:`27775`).
 -
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.transform` where applying a timezone conversion lambda function would drop timezone information (:issue:`27496`)
 - Bug in windowing over read-only arrays (:issue:`27766`)
--
+- Fixed segfault in `pandas.core.groupby.DataFrameGroupBy.quantile` when an invalid quantile was passed (:issue:`27470`)
 -
 
 Reshaping

doc/source/whatsnew/v1.0.0.rst

@@ -163,7 +163,7 @@ I/O
 Plotting
 ^^^^^^^^
 
--
+- Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`)
 -
 
 Groupby/resample/rolling

environment.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   # required
   - numpy>=1.15
-  - python=3
+  - python=3.7.3
   - python-dateutil>=2.6.1
   - pytz
 

pandas/_libs/groupby.pyx

@@ -719,6 +719,11 @@ def group_quantile(ndarray[float64_t] out,
         ndarray[int64_t] counts, non_na_counts, sort_arr
 
     assert values.shape[0] == N
+
+    if not (0 <= q <= 1):
+        raise ValueError("'q' must be between 0 and 1. Got"
+                         " '{}' instead".format(q))
+
     inter_methods = {
         'linear': INTERPOLATION_LINEAR,
         'lower': INTERPOLATION_LOWER,

pandas/core/arrays/base.py

@@ -64,6 +64,7 @@ class ExtensionArray:
     shift
     take
     unique
+    view
     _concat_same_type
     _formatter
     _from_factorized
@@ -146,7 +147,7 @@ class ExtensionArray:
     If implementing NumPy's ``__array_ufunc__`` interface, pandas expects
     that
 
-    1. You defer by raising ``NotImplemented`` when any Series are present
+    1. You defer by returning ``NotImplemented`` when any Series are present
        in `inputs`. Pandas will extract the arrays and call the ufunc again.
     2. You define a ``_HANDLED_TYPES`` tuple as an attribute on the class.
        Pandas inspect this to determine whether the ufunc is valid for the
@@ -861,6 +862,27 @@ def copy(self) -> ABCExtensionArray:
         """
         raise AbstractMethodError(self)
 
+    def view(self, dtype=None) -> Union[ABCExtensionArray, np.ndarray]:
+        """
+        Return a view on the array.
+
+        Parameters
+        ----------
+        dtype : str, np.dtype, or ExtensionDtype, optional
+            Default None
+
+        Returns
+        -------
+        ExtensionArray
+        """
+        # NB:
+        # - This must return a *new* object referencing the same data, not self.
+        # - The only case that *must* be implemented is with dtype=None,
+        #   giving a view with the same dtype as self.
+        if dtype is not None:
+            raise NotImplementedError(dtype)
+        return self[:]
+
     # ------------------------------------------------------------------------
     # Printing
     # ------------------------------------------------------------------------

pandas/core/arrays/categorical.py

@@ -77,7 +77,7 @@
 )
 
 
-def _cat_compare_op(op):
+def _cat_compare_op(opname):
     def f(self, other):
         # On python2, you can usually compare any type to any type, and
         # Categoricals can be seen as a custom type, but having different
@@ -90,7 +90,7 @@ def f(self, other):
         other = lib.item_from_zerodim(other)
 
         if not self.ordered:
-            if op in ["__lt__", "__gt__", "__le__", "__ge__"]:
+            if opname in ["__lt__", "__gt__", "__le__", "__ge__"]:
                 raise TypeError(
                     "Unordered Categoricals can only compare equality or not"
                 )
@@ -117,7 +117,7 @@ def f(self, other):
                 other_codes = other._codes
 
             mask = (self._codes == -1) | (other_codes == -1)
-            f = getattr(self._codes, op)
+            f = getattr(self._codes, opname)
             ret = f(other_codes)
             if mask.any():
                 # In other series, the leads to False, so do that here too
@@ -127,38 +127,38 @@ def f(self, other):
         if is_scalar(other):
             if other in self.categories:
                 i = self.categories.get_loc(other)
-                ret = getattr(self._codes, op)(i)
+                ret = getattr(self._codes, opname)(i)
 
                 # check for NaN in self
                 mask = self._codes == -1
                 ret[mask] = False
                 return ret
             else:
-                if op == "__eq__":
+                if opname == "__eq__":
                     return np.repeat(False, len(self))
-                elif op == "__ne__":
+                elif opname == "__ne__":
                     return np.repeat(True, len(self))
                 else:
                     msg = (
                         "Cannot compare a Categorical for op {op} with a "
                         "scalar, which is not a category."
                     )
-                    raise TypeError(msg.format(op=op))
+                    raise TypeError(msg.format(op=opname))
         else:
 
             # allow categorical vs object dtype array comparisons for equality
             # these are only positional comparisons
-            if op in ["__eq__", "__ne__"]:
-                return getattr(np.array(self), op)(np.array(other))
+            if opname in ["__eq__", "__ne__"]:
+                return getattr(np.array(self), opname)(np.array(other))
 
             msg = (
                 "Cannot compare a Categorical for op {op} with type {typ}."
                 "\nIf you want to compare values, use 'np.asarray(cat) "
                 "<op> other'."
             )
-            raise TypeError(msg.format(op=op, typ=type(other)))
+            raise TypeError(msg.format(op=opname, typ=type(other)))
 
-    f.__name__ = op
+    f.__name__ = opname
 
     return f
 
@@ -516,19 +516,12 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             return self._set_dtype(dtype)
         return np.array(self, dtype=dtype, copy=copy)
 
-    @cache_readonly
-    def ndim(self) -> int:
-        """
-        Number of dimensions of the Categorical
-        """
-        return self._codes.ndim
-
     @cache_readonly
     def size(self) -> int:
         """
         return the len of myself
         """
-        return len(self)
+        return self._codes.size
 
     @cache_readonly
     def itemsize(self) -> int:
@@ -1763,18 +1756,10 @@ def ravel(self, order="C"):
         )
         return np.array(self)
 
-    def view(self):
-        """
-        Return a view of myself.
-
-        For internal compatibility with numpy arrays.
-
-        Returns
-        -------
-        view : Categorical
-           Returns `self`!
-        """
-        return self
+    def view(self, dtype=None):
+        if dtype is not None:
+            raise NotImplementedError(dtype)
+        return self._constructor(values=self._codes, dtype=self.dtype, fastpath=True)
 
     def to_dense(self):
         """

pandas/core/arrays/datetimelike.py

@@ -544,18 +544,8 @@ def astype(self, dtype, copy=True):
             return np.asarray(self, dtype=dtype)
 
     def view(self, dtype=None):
-        """
-        New view on this array with the same data.
-
-        Parameters
-        ----------
-        dtype : numpy dtype, optional
-
-        Returns
-        -------
-        ndarray
-            With the specified `dtype`.
-        """
+        if dtype is None or dtype is self.dtype:
+            return type(self)(self._data, dtype=self.dtype)
         return self._data.view(dtype=dtype)
 
     # ------------------------------------------------------------------

pandas/core/arrays/interval.py

@@ -695,18 +695,14 @@ def isna(self):
         return isna(self.left)
 
     @property
-    def nbytes(self):
+    def nbytes(self) -> int:
         return self.left.nbytes + self.right.nbytes
 
     @property
-    def size(self):
+    def size(self) -> int:
         # Avoid materializing self.values
         return self.left.size
 
-    @property
-    def shape(self):
-        return self.left.shape
-
     def take(self, indices, allow_fill=False, fill_value=None, axis=None, **kwargs):
         """
         Take elements from the IntervalArray.

pandas/core/arrays/numpy_.py

@@ -245,11 +245,11 @@ def __setitem__(self, key, value):
         else:
             self._ndarray[key] = value
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._ndarray)
 
     @property
-    def nbytes(self):
+    def nbytes(self) -> int:
         return self._ndarray.nbytes
 
     def isna(self):