Merge branch 'pandas-dev:main' into bug-cov-nat

Author: fbourgey

.github/CODEOWNERS

@@ -4,9 +4,6 @@
 # ci
 ci/                               @mroeschke
 
-# web
-web/                              @datapythonista
-
 # docs
 doc/cheatsheet                    @Dr-Irv
 doc/source/development            @noatamir

.github/actions/run-tests/action.yml

@@ -14,7 +14,7 @@ runs:
       if: failure()
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         flags: unittests
         name: codecov-pandas

.github/actions/setup-conda/action.yml

@@ -7,7 +7,7 @@ runs:
   using: composite
   steps:
     - name: Install ${{ inputs.environment-file }}
-      uses: mamba-org/setup-micromamba@v1
+      uses: mamba-org/setup-micromamba@v2
       with:
         environment-file: ${{ inputs.environment-file }}
         environment-name: test

.github/workflows/docbuild-and-upload.yml

@@ -59,6 +59,10 @@ jobs:
     - name: Build documentation
       run: doc/make.py --warnings-are-errors
 
+    - name: Build the interactive terminal
+      working-directory: web/interactive_terminal
+      run: jupyter lite build
+
     - name: Build documentation zip
       run: doc/make.py zip_html
 

.github/workflows/wheels.yml

@@ -153,7 +153,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.0
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.4
+    rev: v0.9.9
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -70,7 +70,7 @@ repos:
     -   id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
 -   repo: https://github.com/PyCQA/isort
-    rev: 6.0.0
+    rev: 6.0.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
@@ -106,6 +106,11 @@ repos:
     hooks:
     - id: meson-fmt
       args: ['--inplace']
+-   repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.10.0.1
+    hooks:
+    -   id: shellcheck
+        args: ["--severity=warning"]
 -   repo: local
     hooks:
     -   id: pyright

Dockerfile

@@ -13,5 +13,5 @@ COPY requirements-dev.txt /tmp
 RUN python -m pip install -r /tmp/requirements-dev.txt
 RUN git config --global --add safe.directory /home/pandas
 
-ENV SHELL "/bin/bash"
+ENV SHELL="/bin/bash"
 CMD ["/bin/bash"]

ci/code_checks.sh

@@ -24,15 +24,15 @@ else
 fi
 
 [[ -z "$CHECK" || "$CHECK" == "code" || "$CHECK" == "doctests" || "$CHECK" == "docstrings" || "$CHECK" == "single-docs" || "$CHECK" == "notebooks" ]] || \
-    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
+    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 1; }
 
-BASE_DIR="$(dirname $0)/.."
+BASE_DIR="$(dirname "$0")/.."
 RET=0
 
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 
-    MSG='Check import. No warnings, and blocklist some optional dependencies' ; echo $MSG
+    MSG='Check import. No warnings, and blocklist some optional dependencies' ; echo "$MSG"
     python -W error -c "
 import sys
 import pandas
@@ -49,24 +49,24 @@ if mods:
     sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))
     "
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
+    RET=$(($RET + $?)) ; echo "$MSG" "DONE"
 
 fi
 
 ### DOCTESTS ###
 if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
-    MSG='Python and Cython Doctests' ; echo $MSG
+    MSG='Python and Cython Doctests' ; echo "$MSG"
     python -c 'import pandas as pd; pd.test(run_doctests=True)'
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
+    RET=$(($RET + $?)) ; echo "$MSG" "DONE"
 
 fi
 
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate Docstrings' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py \
+    MSG='Validate Docstrings' ; echo "$MSG"
+    "$BASE_DIR"/scripts/validate_docstrings.py \
         --format=actions \
         -i ES01 `# For now it is ok if docstrings are missing the extended summary` \
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
@@ -265,7 +265,7 @@ fi
 if [[ -z "$CHECK" || "$CHECK" == "notebooks" ]]; then
 
     MSG='Notebooks' ; echo $MSG
-    jupyter nbconvert --execute $(find doc/source -name '*.ipynb') --to notebook
+    jupyter nbconvert --execute "$(find doc/source -name '*.ipynb')" --to notebook
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/run_tests.sh

@@ -3,10 +3,8 @@
 # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set)
 # https://github.com/pytest-dev/pytest/issues/920
 # https://github.com/pytest-dev/pytest/issues/1075
-export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
-
-# May help reproduce flaky CI builds if set in subsequent runs
-echo PYTHONHASHSEED=$PYTHONHASHSEED
+PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+export PYTHONHASHSEED
 
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
@@ -16,5 +14,5 @@ if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
 fi
 
-echo $PYTEST_CMD
+echo "$PYTEST_CMD"
 sh -c "$PYTEST_CMD"

ci/upload_wheels.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Modified from numpy's https://github.com/numpy/numpy/blob/main/tools/wheels/upload_wheels.sh
 
 set_upload_vars() {
@@ -19,20 +20,20 @@ set_upload_vars() {
     fi
 }
 upload_wheels() {
-    echo ${PWD}
+    echo "${PWD}"
     if [[ ${ANACONDA_UPLOAD} == true ]]; then
-        if [ -z ${TOKEN} ]; then
+        if [ -z "${TOKEN}" ]; then
             echo no token set, not uploading
         else
             # sdists are located under dist folder when built through setup.py
             if compgen -G "./dist/*.gz"; then
                 echo "Found sdist"
-                anaconda -q -t ${TOKEN} upload --skip -u ${ANACONDA_ORG} ./dist/*.gz
+                anaconda -q -t "${TOKEN}" upload --skip -u "${ANACONDA_ORG}" ./dist/*.gz
                 echo "Uploaded sdist"
             fi
             if compgen -G "./wheelhouse/*.whl"; then
                 echo "Found wheel"
-                anaconda -q -t ${TOKEN} upload --skip -u ${ANACONDA_ORG} ./wheelhouse/*.whl
+                anaconda -q -t "${TOKEN}" upload --skip -u "${ANACONDA_ORG}" ./wheelhouse/*.whl
                 echo "Uploaded wheel"
             fi
             echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"

doc/source/development/contributing_codebase.rst

@@ -198,7 +198,7 @@ In some cases you may be tempted to use ``cast`` from the typing module when you
            obj = cast(str, obj)  # Mypy complains without this!
            return obj.upper()
 
-The limitation here is that while a human can reasonably understand that ``is_number`` would catch the ``int`` and ``float`` types mypy cannot make that same inference just yet (see `mypy #5206 <https://github.com/python/mypy/issues/5206>`_. While the above works, the use of ``cast`` is **strongly discouraged**. Where applicable a refactor of the code to appease static analysis is preferable
+The limitation here is that while a human can reasonably understand that ``is_number`` would catch the ``int`` and ``float`` types mypy cannot make that same inference just yet (see `mypy #5206 <https://github.com/python/mypy/issues/5206>`_). While the above works, the use of ``cast`` is **strongly discouraged**. Where applicable a refactor of the code to appease static analysis is preferable
 
 .. code-block:: python
 
@@ -344,7 +344,7 @@ be located.
    - tests.scalar
    - tests.tseries.offsets
 
-2. Does your test depend only on code in pd._libs?
+2. Does your test depend only on code in ``pd._libs``?
    This test likely belongs in one of:
 
    - tests.libs

doc/source/development/contributing_gitpod.rst

@@ -109,7 +109,7 @@ development experience:
 
 *  `VSCode rst extension <https://marketplace.visualstudio.com/items?itemName=lextudio.restructuredtext>`_
 *  `Markdown All in One <https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one>`_
-*  `VSCode Gitlens extension <https://marketplace.visualstudio.com/items?itemName=eamodio.gitlens>`_
+*  `VSCode GitLens extension <https://marketplace.visualstudio.com/items?itemName=eamodio.gitlens>`_
 *  `VSCode Git Graph extension <https://marketplace.visualstudio.com/items?itemName=mhutchie.git-graph>`_
 
 Development workflow with Gitpod

doc/source/development/developer.rst

@@ -99,7 +99,7 @@ Column metadata
 * Boolean: ``'bool'``
 * Integers: ``'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64'``
 * Floats: ``'float16', 'float32', 'float64'``
-* Date and Time Types: ``'datetime', 'datetimetz'``, ``'timedelta'``
+* Date and Time Types: ``'datetime', 'datetimetz', 'timedelta'``
 * String: ``'unicode', 'bytes'``
 * Categorical: ``'categorical'``
 * Other Python objects: ``'object'``

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -335,7 +335,7 @@ the name ``anonymous`` to the first 3 elements of the fourth column:
 .. ipython:: python
 
     titanic.iloc[0:3, 3] = "anonymous"
-    titanic.head()
+    titanic.iloc[:5, 3]
 
 .. raw:: html
 

doc/source/getting_started/overview.rst

@@ -174,3 +174,4 @@ License
 -------
 
 .. literalinclude:: ../../../LICENSE
+   :language: none

doc/source/reference/arrays.rst

@@ -61,7 +61,7 @@ is an :class:`ArrowDtype`.
 support as NumPy including first-class nullability support for all data types, immutability and more.
 
 The table below shows the equivalent pyarrow-backed (``pa``), pandas extension, and numpy (``np``) types that are recognized by pandas.
-Pyarrow-backed types below need to be passed into :class:`ArrowDtype` to be recognized by pandas e.g. ``pd.ArrowDtype(pa.bool_())``
+Pyarrow-backed types below need to be passed into :class:`ArrowDtype` to be recognized by pandas e.g. ``pd.ArrowDtype(pa.bool_())``.
 
 =============================================== ========================== ===================
 PyArrow type                                    pandas extension type      NumPy type
@@ -114,7 +114,7 @@ values.
 
    ArrowDtype
 
-For more information, please see the :ref:`PyArrow user guide <pyarrow>`
+For more information, please see the :ref:`PyArrow user guide <pyarrow>`.
 
 .. _api.arrays.datetime:
 
@@ -495,7 +495,7 @@ a :class:`CategoricalDtype`.
    CategoricalDtype.categories
    CategoricalDtype.ordered
 
-Categorical data can be stored in a :class:`pandas.Categorical`
+Categorical data can be stored in a :class:`pandas.Categorical`:
 
 .. autosummary::
    :toctree: api/

doc/source/reference/series.rst

@@ -25,6 +25,7 @@ Attributes
    Series.array
    Series.values
    Series.dtype
+   Series.info
    Series.shape
    Series.nbytes
    Series.ndim

doc/source/user_guide/io.rst

@@ -18,10 +18,10 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :widths: 30, 100, 60, 60
 
     text,`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__, :ref:`read_csv<io.read_csv_table>`, :ref:`to_csv<io.store_in_csv>`
-    text,Fixed-Width Text File, :ref:`read_fwf<io.fwf_reader>` , NA
+    text,Fixed-Width Text File, :ref:`read_fwf<io.fwf_reader>`, NA
     text,`JSON <https://www.json.org/>`__, :ref:`read_json<io.json_reader>`, :ref:`to_json<io.json_writer>`
     text,`HTML <https://en.wikipedia.org/wiki/HTML>`__, :ref:`read_html<io.read_html>`, :ref:`to_html<io.html>`
-    text,`LaTeX <https://en.wikipedia.org/wiki/LaTeX>`__, :ref:`Styler.to_latex<io.latex>` , NA
+    text,`LaTeX <https://en.wikipedia.org/wiki/LaTeX>`__, NA, :ref:`Styler.to_latex<io.latex>`
     text,`XML <https://www.w3.org/standards/xml/core>`__, :ref:`read_xml<io.read_xml>`, :ref:`to_xml<io.xml>`
     text, Local clipboard, :ref:`read_clipboard<io.clipboard>`, :ref:`to_clipboard<io.clipboard>`
     binary,`MS Excel <https://en.wikipedia.org/wiki/Microsoft_Excel>`__ , :ref:`read_excel<io.excel_reader>`, :ref:`to_excel<io.excel_writer>`

doc/source/user_guide/merging.rst

@@ -906,7 +906,7 @@ resetting indexes.
 Joining multiple :class:`DataFrame`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-A list or tuple of ``:class:`DataFrame``` can also be passed to :meth:`~DataFrame.join`
+A list or tuple of :class:`DataFrame` can also be passed to :meth:`~DataFrame.join`
 to join them together on their indexes.
 
 .. ipython:: python

doc/source/user_guide/text.rst

@@ -13,7 +13,7 @@ Text data types
 
 There are two ways to store text data in pandas:
 
-1. ``object`` -dtype NumPy array.
+1. ``object`` dtype NumPy array.
 2. :class:`StringDtype` extension type.
 
 We recommend using :class:`StringDtype` to store text data.
@@ -40,20 +40,20 @@ to significantly increase the performance and lower the memory overhead of
    and parts of the API may change without warning.
 
 For backwards-compatibility, ``object`` dtype remains the default type we
-infer a list of strings to
+infer a list of strings to:
 
 .. ipython:: python
 
    pd.Series(["a", "b", "c"])
 
-To explicitly request ``string`` dtype, specify the ``dtype``
+To explicitly request ``string`` dtype, specify the ``dtype``:
 
 .. ipython:: python
 
    pd.Series(["a", "b", "c"], dtype="string")
    pd.Series(["a", "b", "c"], dtype=pd.StringDtype())
 
-Or ``astype`` after the ``Series`` or ``DataFrame`` is created
+Or ``astype`` after the ``Series`` or ``DataFrame`` is created:
 
 .. ipython:: python
 
@@ -88,9 +88,9 @@ Behavior differences
 ^^^^^^^^^^^^^^^^^^^^
 
 These are places where the behavior of ``StringDtype`` objects differ from
-``object`` dtype
+``object`` dtype:
 
-l. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
+1. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
    that return **numeric** output will always return a nullable integer dtype,
    rather than either int or float dtype, depending on the presence of NA values.
    Methods returning **boolean** output will return a nullable boolean dtype.
@@ -102,7 +102,7 @@ l. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
       s.str.count("a")
       s.dropna().str.count("a")
 
-   Both outputs are ``Int64`` dtype. Compare that with object-dtype
+   Both outputs are ``Int64`` dtype. Compare that with object-dtype:
 
    .. ipython:: python
 
@@ -332,8 +332,8 @@ regular expression object will raise a ``ValueError``.
     ---------------------------------------------------------------------------
     ValueError: case and flags cannot be set when pat is a compiled regex
 
-``removeprefix`` and ``removesuffix`` have the same effect as ``str.removeprefix`` and ``str.removesuffix`` added in Python 3.9
-<https://docs.python.org/3/library/stdtypes.html#str.removeprefix>`__:
+``removeprefix`` and ``removesuffix`` have the same effect as ``str.removeprefix`` and ``str.removesuffix`` added in
+`Python 3.9 <https://docs.python.org/3/library/stdtypes.html#str.removeprefix>`__:
 
 .. versionadded:: 1.4.0
 

doc/source/user_guide/window.rst

@@ -70,7 +70,8 @@ which will first group the data by the specified keys and then perform a windowi
 
     Some windowing aggregation, ``mean``, ``sum``, ``var`` and ``std`` methods may suffer from numerical
     imprecision due to the underlying windowing algorithms accumulating sums. When values differ
-    with magnitude :math:`1/np.finfo(np.double).eps` this results in truncation. It must be
+    with magnitude ``1/np.finfo(np.double).eps`` (approximately :math:`4.5 \times 10^{15}`),
+    this results in truncation. It must be
     noted, that large values may have an impact on windows, which do not include these values. `Kahan summation
     <https://en.wikipedia.org/wiki/Kahan_summation_algorithm>`__ is used
     to compute the rolling sums to preserve accuracy as much as possible.
@@ -356,11 +357,11 @@ See :ref:`enhancing performance with Numba <enhancingperf.numba>` for general us
 
 Numba will be applied in potentially two routines:
 
-#. If ``func`` is a standard Python function, the engine will `JIT <https://numba.pydata.org/numba-doc/latest/user/overview.html>`__ the passed function. ``func`` can also be a JITed function in which case the engine will not JIT the function again.
+#. If ``func`` is a standard Python function, the engine will `JIT <https://numba.readthedocs.io/en/stable/user/overview.html>`__ the passed function. ``func`` can also be a JITed function in which case the engine will not JIT the function again.
 #. The engine will JIT the for loop where the apply function is applied to each window.
 
 The ``engine_kwargs`` argument is a dictionary of keyword arguments that will be passed into the
-`numba.jit decorator <https://numba.pydata.org/numba-doc/latest/reference/jit-compilation.html#numba.jit>`__.
+`numba.jit decorator <https://numba.readthedocs.io/en/stable/user/jit.html>`__.
 These keyword arguments will be applied to *both* the passed function (if a standard Python function)
 and the apply for loop over each window.