Merge remote-tracking branch 'upstream/main' into arrow/docs

Author: mroeschke

.github/workflows/32-bit-linux.yml

@@ -38,7 +38,7 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython==0.29.30 numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
           python setup.py build_ext -q -j2 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           export PANDAS_CI=1 && \

.github/workflows/sdist.yml

@@ -79,9 +79,9 @@ jobs:
       run: |
         case "${{matrix.python-version}}" in
         3.8)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.9)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.10)
           pip install numpy==1.21.2 ;;
         esac

.pre-commit-config.yaml

@@ -93,7 +93,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.262
+        - pyright@1.1.264
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

asv_bench/asv.conf.json

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.30"],
+        "Cython": ["0.29.32"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

ci/deps/actions-310-numpydev.yaml

@@ -16,8 +16,7 @@ dependencies:
   - pytz
   - pip
   - pip:
-    #- cython # TODO: don't install from master after Cython 3.0.0a11 is released
-    - "git+https://github.com/cython/cython.git@master"
+    - "cython"
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-310.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.10
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-downstream_compat.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-minimum_versions.yaml

@@ -7,7 +7,7 @@ dependencies:
   - python=3.8.0
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
@@ -17,7 +17,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.1
-  - numpy=1.19.5
+  - numpy=1.20.3
   - pytz=2020.1
 
   # optional dependencies

ci/deps/actions-38.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.9
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-pypy-38.yaml

@@ -8,7 +8,7 @@ dependencies:
   - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available
 
   # tools
-  - cython>=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-asyncio

ci/deps/circle-38-arm64.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # test dependencies
-  - cython=0.29.30
+  - cython>=0.29.32
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

doc/source/getting_started/install.rst

@@ -235,7 +235,7 @@ Dependencies
 ================================================================ ==========================
 Package                                                          Minimum supported version
 ================================================================ ==========================
-`NumPy <https://numpy.org>`__                                    1.19.5
+`NumPy <https://numpy.org>`__                                    1.20.3
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.8.1
 `pytz <https://pypi.org/project/pytz/>`__                        2020.1
 ================================================================ ==========================

doc/source/reference/style.rst

@@ -41,6 +41,7 @@ Style application
    Styler.applymap_index
    Styler.format
    Styler.format_index
+   Styler.relabel_index
    Styler.hide
    Styler.concat
    Styler.set_td_classes

doc/source/reference/testing.rst

@@ -27,6 +27,7 @@ Exceptions and warnings
    errors.AbstractMethodError
    errors.AccessorRegistrationWarning
    errors.AttributeConflictWarning
+   errors.CategoricalConversionWarning
    errors.ClosedFileError
    errors.CSSWarning
    errors.DatabaseError
@@ -36,6 +37,7 @@ Exceptions and warnings
    errors.EmptyDataError
    errors.IncompatibilityWarning
    errors.IndexingError
+   errors.InvalidColumnName
    errors.InvalidIndexError
    errors.IntCastingNaNError
    errors.MergeError
@@ -49,6 +51,7 @@ Exceptions and warnings
    errors.ParserWarning
    errors.PerformanceWarning
    errors.PossibleDataLossError
+   errors.PossiblePrecisionLoss
    errors.PyperclipException
    errors.PyperclipWindowsException
    errors.SettingWithCopyError
@@ -57,6 +60,7 @@ Exceptions and warnings
    errors.UndefinedVariableError
    errors.UnsortedIndexError
    errors.UnsupportedFunctionCall
+   errors.ValueLabelTypeMismatch
 
 Bug report function
 -------------------

doc/source/user_guide/10min.rst

@@ -680,12 +680,12 @@ Converting the raw grades to a categorical data type:
     df["grade"] = df["raw_grade"].astype("category")
     df["grade"]
 
-Rename the categories to more meaningful names (assigning to
-:meth:`Series.cat.categories` is in place!):
+Rename the categories to more meaningful names:
 
 .. ipython:: python
 
-    df["grade"].cat.categories = ["very good", "good", "very bad"]
+    new_categories = ["very good", "good", "very bad"]
+    df["grade"] = df["grade"].cat.rename_categories(new_categories)
 
 Reorder the categories and simultaneously add the missing categories (methods under :meth:`Series.cat` return a new :class:`Series` by default):
 

doc/source/user_guide/categorical.rst

@@ -334,18 +334,16 @@ It's also possible to pass in the categories in a specific order:
 Renaming categories
 ~~~~~~~~~~~~~~~~~~~
 
-Renaming categories is done by assigning new values to the
-``Series.cat.categories`` property or by using the
+Renaming categories is done by using the
 :meth:`~pandas.Categorical.rename_categories` method:
 
 
 .. ipython:: python
 
     s = pd.Series(["a", "b", "c", "a"], dtype="category")
     s
-    s.cat.categories = ["Group %s" % g for g in s.cat.categories]
-    s
-    s = s.cat.rename_categories([1, 2, 3])
+    new_categories = ["Group %s" % g for g in s.cat.categories]
+    s = s.cat.rename_categories(new_categories)
     s
     # You can also pass a dict-like object to map the renaming
     s = s.cat.rename_categories({1: "x", 2: "y", 3: "z"})
@@ -365,7 +363,7 @@ Categories must be unique or a ``ValueError`` is raised:
 .. ipython:: python
 
     try:
-        s.cat.categories = [1, 1, 1]
+        s = s.cat.rename_categories([1, 1, 1])
     except ValueError as e:
         print("ValueError:", str(e))
 
@@ -374,7 +372,7 @@ Categories must also not be ``NaN`` or a ``ValueError`` is raised:
 .. ipython:: python
 
     try:
-        s.cat.categories = [1, 2, np.nan]
+        s = s.cat.rename_categories([1, 2, np.nan])
     except ValueError as e:
         print("ValueError:", str(e))
 
@@ -702,7 +700,7 @@ of length "1".
 .. ipython:: python
 
     df.iat[0, 0]
-    df["cats"].cat.categories = ["x", "y", "z"]
+    df["cats"] = df["cats"].cat.rename_categories(["x", "y", "z"])
     df.at["h", "cats"]  # returns a string
 
 .. note::
@@ -960,7 +958,7 @@ relevant columns back to ``category`` and assign the right categories and catego
 
     s = pd.Series(pd.Categorical(["a", "b", "b", "a", "a", "d"]))
     # rename the categories
-    s.cat.categories = ["very good", "good", "bad"]
+    s = s.cat.rename_categories(["very good", "good", "bad"])
     # reorder the categories and add missing categories
     s = s.cat.set_categories(["very bad", "bad", "medium", "good", "very good"])
     df = pd.DataFrame({"cats": s, "vals": [1, 2, 3, 4, 5, 6]})
@@ -1164,6 +1162,7 @@ Constructing a ``Series`` from a ``Categorical`` will not copy the input
 change the original ``Categorical``:
 
 .. ipython:: python
+    :okwarning:
 
     cat = pd.Categorical([1, 2, 3, 10], categories=[1, 2, 3, 4, 10])
     s = pd.Series(cat, name="cat")

doc/source/user_guide/groupby.rst

@@ -345,17 +345,6 @@ Index level names may be supplied as keys.
 
 More on the ``sum`` function and aggregation later.
 
-When using ``.groupby()`` on a DatFrame with a  MultiIndex, do not specify both ``by`` and ``level``.
-The argument validation should be done in ``.groupby()``, using the name of the specific index.
-
-.. ipython:: python
-
-   df = pd.DataFrame({"col1": ["a", "b", "c"]})
-   df.index = pd.MultiIndex.from_arrays([["a", "a", "b"],
-                                        [1, 2, 1]],
-                                        names=["x", "y"])
-   df.groupby(["col1", "x"])
-
 Grouping DataFrame with Index levels and columns
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 A DataFrame may be grouped by a combination of columns and index levels by

doc/source/user_guide/io.rst

@@ -558,7 +558,8 @@ This matches the behavior of :meth:`Categorical.set_categories`.
       df = pd.read_csv(StringIO(data), dtype="category")
       df.dtypes
       df["col3"]
-      df["col3"].cat.categories = pd.to_numeric(df["col3"].cat.categories)
+      new_categories = pd.to_numeric(df["col3"].cat.categories)
+      df["col3"] = df["col3"].cat.rename_categories(new_categories)
       df["col3"]
 
 
@@ -3054,15 +3055,15 @@ Read in the content of the "books.xml" as instance of ``StringIO`` or
    df = pd.read_xml(bio)
    df
 
-Even read XML from AWS S3 buckets such as Python Software Foundation's IRS 990 Form:
+Even read XML from AWS S3 buckets such as NIH NCBI PMC Article Datasets providing
+Biomedical and Life Science Jorurnals:
 
 .. ipython:: python
    :okwarning:
 
    df = pd.read_xml(
-       "s3://irs-form-990/201923199349319487_public.xml",
-       xpath=".//irs:Form990PartVIISectionAGrp",
-       namespaces={"irs": "http://www.irs.gov/efile"}
+       "s3://pmc-oa-opendata/oa_comm/xml/all/PMC1236943.xml",
+       xpath=".//journal-meta",
    )
    df
 

doc/source/user_guide/visualization.rst

@@ -6,6 +6,11 @@
 Chart visualization
 *******************
 
+
+.. note::
+
+   The examples below assume that you're using `Jupyter <https://jupyter.org/>`_.
+
 This section demonstrates visualization through charting. For information on
 visualization of tabular data please see the section on `Table Visualization <style.ipynb>`_.
 

doc/source/whatsnew/v0.19.0.rst

@@ -271,6 +271,7 @@ Individual columns can be parsed as a ``Categorical`` using a dict specification
    such as :func:`to_datetime`.
 
    .. ipython:: python
+      :okwarning:
 
       df = pd.read_csv(StringIO(data), dtype="category")
       df.dtypes

doc/source/whatsnew/v1.4.4.rst

@@ -14,7 +14,10 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Fixed regression in taking NULL :class:`objects` from a :class:`DataFrame` causing a segmentation violation. These NULL values are created by :meth:`numpy.empty_like` (:issue:`46848`)
 - Fixed regression in :func:`concat` materializing :class:`Index` during sorting even if :class:`Index` was already sorted (:issue:`47501`)
+- Fixed regression in :meth:`DataFrame.loc` not updating the cache correctly after values were set (:issue:`47867`)
+- Fixed regression in :meth:`DataFrame.loc` not aligning index in some cases when setting a :class:`DataFrame` (:issue:`47578`)
 - Fixed regression in setting ``None`` or non-string value into a ``string``-dtype Series using a mask (:issue:`47628`)
 -
 
@@ -26,14 +29,15 @@ Bug fixes
 ~~~~~~~~~
 - The :class:`errors.FutureWarning` raised when passing arguments (other than ``filepath_or_buffer``) as positional in :func:`read_csv` is now raised at the correct stacklevel (:issue:`47385`)
 - Bug in :meth:`DataFrame.to_sql` when ``method`` was a ``callable`` that did not return an ``int`` and would raise a ``TypeError`` (:issue:`46891`)
+- Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 
 .. ---------------------------------------------------------------------------
 
 .. _whatsnew_144.other:
 
 Other
 ~~~~~
--
+- The minimum version of Cython needed to compile pandas is now ``0.29.32`` (:issue:`47978`)
 -
 
 .. ---------------------------------------------------------------------------