Merge remote-tracking branch 'upstream/main'

Author: Dr-Irv

.pre-commit-config.yaml

@@ -50,7 +50,7 @@ repos:
         - flake8==4.0.1
         - flake8-comprehensions==3.7.0
         - flake8-bugbear==21.3.2
-        - pandas-dev-flaker==0.2.0
+        - pandas-dev-flaker==0.4.0
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:

azure-pipelines.yml

@@ -18,6 +18,8 @@ pr:
 variables:
   PYTEST_WORKERS: auto
   PYTEST_TARGET:  pandas
+  PATTERN: "not slow and not high_memory and not db and not network"
+  PANDAS_CI: 1
 
 jobs:
 # Mac and Linux use the same template

ci/azure/windows.yml

@@ -4,51 +4,22 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
+  timeoutInMinutes: 90
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py38_np18_1:
-        ENV_FILE: ci/deps/azure-windows-38.yaml
+      py38:
+        ENV_FILE: ci/deps/actions-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
 
-      py38_np18_2:
-        ENV_FILE: ci/deps/azure-windows-38.yaml
-        CONDA_PY: "38"
-        PATTERN: "not slow"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-
-      py39_1:
-        ENV_FILE: ci/deps/azure-windows-39.yaml
-        CONDA_PY: "39"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
-
-      py39_2:
-        ENV_FILE: ci/deps/azure-windows-39.yaml
+      py39:
+        ENV_FILE: ci/deps/actions-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
-
-      py310_1:
-        ENV_FILE: ci/deps/azure-windows-310.yaml
-        CONDA_PY: "310"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[a-h]*"
 
-      py310_2:
-        ENV_FILE: ci/deps/azure-windows-310.yaml
+      py310:
+        ENV_FILE: ci/deps/actions-310.yaml
         CONDA_PY: "310"
-        PATTERN: "not slow and not high_memory"
-        PYTEST_WORKERS: 2  # GH-42236
-        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - powershell: |
@@ -59,7 +30,7 @@ jobs:
       displayName: 'Update conda'
 
     - bash: |
-        conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml
+        conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml
       displayName: 'Create anaconda environment'
     - bash: |
         source activate pandas-dev

ci/deps/azure-windows-310.yaml

@@ -446,7 +446,7 @@
         "pandas-gbq": ("https://pandas-gbq.readthedocs.io/en/latest/", None),
         "py": ("https://pylib.readthedocs.io/en/latest/", None),
         "python": ("https://docs.python.org/3/", None),
-        "scipy": ("https://docs.scipy.org/doc/scipy/reference/", None),
+        "scipy": ("https://docs.scipy.org/doc/scipy-1.7.1/reference/", None),
         "statsmodels": ("https://www.statsmodels.org/devel/", None),
         "pyarrow": ("https://arrow.apache.org/docs/", None),
     }

ci/deps/azure-windows-38.yaml

@@ -53,7 +53,6 @@ Excel
 
 .. autosummary::
    :toctree: api/
-   :template: autosummary/class_without_autosummary.rst
 
    ExcelWriter
 

ci/deps/azure-windows-39.yaml

@@ -34,6 +34,7 @@ Bug fixes
 - Stopped emitting unnecessary ``FutureWarning`` in :meth:`DataFrame.sort_values` with sparse columns (:issue:`45618`)
 - Fixed window aggregations in :meth:`DataFrame.rolling` and :meth:`Series.rolling` to skip over unused elements (:issue:`45647`)
 - Bug in :func:`api.types.is_bool_dtype` was raising an ``AttributeError`` when evaluating a categorical :class:`Series` (:issue:`45615`)
+- Fixed builtin highlighters in :class:`.Styler` to be responsive to ``NA`` with nullable dtypes (:issue:`45804`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/conf.py

@@ -37,6 +37,7 @@ Other enhancements
 - :meth:`to_numeric` now preserves float64 arrays when downcasting would generate values not representable in float32 (:issue:`43693`)
 - :meth:`Series.reset_index` and :meth:`DataFrame.reset_index` now support the argument ``allow_duplicates`` (:issue:`44410`)
 - :meth:`.GroupBy.min` and :meth:`.GroupBy.max` now supports `Numba <https://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`45428`)
+- Implemented a ``bool``-dtype :class:`Index`, passing a bool-dtype arraylike to ``pd.Index`` will now retain ``bool`` dtype instead of casting to ``object`` (:issue:`45061`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -183,6 +184,43 @@ use ``series.loc[i:j]``.
 
 Slicing on a :class:`DataFrame` will not be affected.
 
+.. _whatsnew_150.deprecations.excel_writer_attributes:
+
+:class:`ExcelWriter` attributes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+All attributes of :class:`ExcelWriter` were previously documented as not
+public. However some third party Excel engines documented accessing
+``ExcelWriter.book`` or ``ExcelWriter.sheets``, and users were utilizing these
+and possibly other attributes. Previously these attributes were not safe to use;
+e.g. modifications to ``ExcelWriter.book`` would not update ``ExcelWriter.sheets``
+and conversely. In order to support this, pandas has made some attributes public
+and improved their implementations so that they may now be safely used. (:issue:`45572`)
+
+The following attributes are now public and considered safe to access.
+
+ - ``book``
+ - ``check_extension``
+ - ``close``
+ - ``date_format``
+ - ``datetime_format``
+ - ``engine``
+ - ``if_sheet_exists``
+ - ``sheets``
+ - ``supported_extensions``
+
+The following attributes have been deprecated. They now raise a ``FutureWarning``
+when accessed and will removed in a future version. Users should be aware
+that their usage is considered unsafe, and can lead to unexpected results.
+
+ - ``cur_sheet``
+ - ``handles``
+ - ``path``
+ - ``save``
+ - ``write_cells``
+
+See the documentation of :class:`ExcelWriter` for further details.
+
 .. _whatsnew_150.deprecations.other:
 
 Other Deprecations
@@ -216,6 +254,7 @@ Bug fixes
 
 Categorical
 ^^^^^^^^^^^
+- Bug in :meth:`Categorical.view` not accepting integer dtypes (:issue:`25464`)
 - Bug in :meth:`CategoricalIndex.union` when the index's categories are integer-dtype and the index contains ``NaN`` values incorrectly raising instead of casting to ``float64`` (:issue:`45362`)
 -
 
@@ -225,6 +264,7 @@ Datetimelike
 - Bug in :func:`to_datetime` with sequences of ``np.str_`` objects incorrectly raising (:issue:`32264`)
 - Bug in :class:`Timestamp` construction when passing datetime components as positional arguments and ``tzinfo`` as a keyword argument incorrectly raising (:issue:`31929`)
 - Bug in :meth:`Index.astype` when casting from object dtype to ``timedelta64[ns]`` dtype incorrectly casting ``np.datetime64("NaT")`` values to ``np.timedelta64("NaT")`` instead of raising (:issue:`45722`)
+- Bug in :meth:`SeriesGroupBy.value_counts` index when passing categorical column (:issue:`44324`)
 -
 
 Timedelta
@@ -295,10 +335,11 @@ MultiIndex
 I/O
 ^^^
 - Bug in :meth:`DataFrame.to_stata` where no error is raised if the :class:`DataFrame` contains ``-np.inf`` (:issue:`45350`)
+- Bug in :func:`read_excel` results in an infinite loop with certain ``skiprows`` callables (:issue:`45585`)
 - Bug in :meth:`DataFrame.info` where a new line at the end of the output is omitted when called on an empty :class:`DataFrame` (:issue:`45494`)
 - Bug in :func:`read_csv` not recognizing line break for ``on_bad_lines="warn"`` for ``engine="c"`` (:issue:`41710`)
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
--
+- Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
 
 Period
 ^^^^^^

doc/source/reference/io.rst

@@ -32,10 +32,12 @@ dependencies:
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb
-  - sphinx
-  - sphinx-panels
   - numpydoc < 1.2  # 2021-02-09 1.2dev breaking CI
+  - pandas-dev-flaker=0.4.0
   - pydata-sphinx-theme
+  - pytest-cython
+  - sphinx
+  - sphinx-panels
   - types-python-dateutil
   - types-PyMySQL
   - types-pytz
@@ -78,7 +80,6 @@ dependencies:
   - ipywidgets
   - nbformat
   - notebook>=6.0.3
-  - pip
 
   # optional
   - blosc
@@ -120,6 +121,3 @@ dependencies:
   - pyreadstat  # pandas.read_spss
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - natsort  # DataFrame.sort_values
-  - pip:
-    - pandas-dev-flaker==0.2.0
-    - pytest-cython

doc/source/whatsnew/v1.4.1.rst

@@ -42,6 +42,7 @@ class ObjectEngine(IndexEngine): ...
 class DatetimeEngine(Int64Engine): ...
 class TimedeltaEngine(DatetimeEngine): ...
 class PeriodEngine(Int64Engine): ...
+class BoolEngine(UInt8Engine): ...
 
 class BaseMultiIndexCodesEngine:
     levels: list[np.ndarray]

doc/source/whatsnew/v1.5.0.rst

@@ -802,6 +802,13 @@ cdef class BaseMultiIndexCodesEngine:
 include "index_class_helper.pxi"
 
 
+cdef class BoolEngine(UInt8Engine):
+    cdef _check_type(self, object val):
+        if not util.is_bool_object(val):
+            raise KeyError(val)
+        return <uint8_t>val
+
+
 @cython.internal
 @cython.freelist(32)
 cdef class SharedEngine:

environment.yml

@@ -555,7 +555,8 @@ def _create_mi_with_dt64tz_level():
     "num_uint8": tm.makeNumericIndex(100, dtype="uint8"),
     "num_float64": tm.makeNumericIndex(100, dtype="float64"),
     "num_float32": tm.makeNumericIndex(100, dtype="float32"),
-    "bool": tm.makeBoolIndex(10),
+    "bool-object": tm.makeBoolIndex(10).astype(object),
+    "bool-dtype": Index(np.random.randn(10) < 0),
     "categorical": tm.makeCategoricalIndex(100),
     "interval": tm.makeIntervalIndex(100),
     "empty": Index([]),
@@ -630,7 +631,7 @@ def index_flat_unique(request):
         key
         for key in indices_dict
         if not (
-            key in ["int", "uint", "range", "empty", "repeats"]
+            key in ["int", "uint", "range", "empty", "repeats", "bool-dtype"]
             or key.startswith("num_")
         )
         and not isinstance(indices_dict[key], MultiIndex)
@@ -1775,7 +1776,7 @@ def indexer_ial(request):
 
 
 @pytest.fixture
-def using_array_manager(request):
+def using_array_manager():
     """
     Fixture to check if the array manager is being used.
     """