Merge remote-tracking branch 'upstream/main' into deps/mpl/37

Author: mroeschke

.pre-commit-config.yaml

@@ -15,15 +15,22 @@ default_stages: [
 ci:
     autofix_prs: false
 repos:
+-   repo: local
+    hooks:
+    # NOTE: we make `black` a local hook because if it's installed from
+    # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc
+    -   id: black
+        name: black
+        description: "Black: The uncompromising Python code formatter"
+        entry: black
+        language: python
+        require_serial: true
+        types_or: [python, pyi]
+        additional_dependencies: [black==23.1.0]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
     rev: v0.0.244
     hooks:
     -   id: ruff
--   repo: https://github.com/MarcoGorelli/absolufy-imports
-    rev: v0.3.1
-    hooks:
-    -   id: absolufy-imports
-        files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.7'
     hooks:
@@ -116,16 +123,6 @@ repos:
     - id: sphinx-lint
 -   repo: local
     hooks:
-    # NOTE: we make `black` a local hook because if it's installed from
-    # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc
-    -   id: black
-        name: black
-        description: "Black: The uncompromising Python code formatter"
-        entry: black
-        language: python
-        require_serial: true
-        types_or: [python, pyi]
-        additional_dependencies: [black==23.1.0]
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright

ci/code_checks.sh

@@ -91,7 +91,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Series.size \
         pandas.Series.T \
         pandas.Series.hasnans \
-        pandas.Series.to_timestamp \
         pandas.Series.to_list \
         pandas.Series.__iter__ \
         pandas.Series.keys \
@@ -218,7 +217,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Period.year \
         pandas.Period.asfreq \
         pandas.Period.now \
-        pandas.Period.to_timestamp \
         pandas.arrays.PeriodArray \
         pandas.Interval.closed \
         pandas.Interval.left \
@@ -562,7 +560,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.swapaxes \
         pandas.DataFrame.first_valid_index \
         pandas.DataFrame.last_valid_index \
-        pandas.DataFrame.to_timestamp \
         pandas.DataFrame.attrs \
         pandas.DataFrame.plot \
         pandas.DataFrame.sparse.density \
@@ -576,7 +573,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
         pandas.DataFrame.plot.line \
         pandas.Series.plot.line \
-        pandas.Timestamp.fromtimestamp \
         pandas.api.types.infer_dtype \
         pandas.api.types.is_datetime64_any_dtype \
         pandas.api.types.is_datetime64_ns_dtype \
@@ -590,15 +586,11 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.api.types.is_timedelta64_dtype \
         pandas.api.types.is_timedelta64_ns_dtype \
         pandas.api.types.is_unsigned_integer_dtype \
-        pandas.core.groupby.DataFrameGroupBy.take \
-        pandas.core.groupby.SeriesGroupBy.take \
         pandas.io.formats.style.Styler.concat \
         pandas.io.formats.style.Styler.export \
         pandas.io.formats.style.Styler.set_td_classes \
         pandas.io.formats.style.Styler.use \
         pandas.io.json.build_table_schema \
-        pandas.merge_ordered \
-        pandas.option_context \
         pandas.plotting.andrews_curves \
         pandas.plotting.autocorrelation_plot \
         pandas.plotting.lag_plot \

doc/source/development/contributing.rst

@@ -331,6 +331,26 @@ To automatically fix formatting errors on each commit you make, you can
 set up pre-commit yourself. First, create a Python :ref:`environment
 <contributing_environment>` and then set up :ref:`pre-commit <contributing.pre-commit>`.
 
+.. _contributing.update-dev:
+
+Updating the development environment
+------------------------------------
+
+After updating your branch to merge in main from upstream, you may need to update
+your development environment to reflect any changes to the various packages that
+are used during development.
+
+If using :ref:`mamba <contributing.mamba>`, do::
+
+    mamba deactivate
+    mamba env update -f environment.yml
+    mamba activate pandas-dev
+
+If using :ref:`pip <contributing.pip>` , do::
+
+    # activate the virtual environment based on your platform
+    pythom -m pip install --upgrade -r requirements-dev.txt
+
 Tips for a successful pull request
 ==================================
 

doc/source/development/contributing_codebase.rst

@@ -89,6 +89,12 @@ without needing to have done ``pre-commit install`` beforehand.
     you may run into issues if you're using conda. To solve this, you can downgrade
     ``virtualenv`` to version ``20.0.33``.
 
+.. note::
+
+    If you have recently merged in main from the upstream branch, some of the
+    dependencies used by ``pre-commit`` may have changed.  Make sure to
+    :ref:`update your development environment <contributing.update-dev>`.
+
 Optional dependencies
 ---------------------
 
@@ -266,17 +272,21 @@ This module will ultimately house types for repeatedly used concepts like "path-
 Validating type hints
 ~~~~~~~~~~~~~~~~~~~~~
 
-pandas uses `mypy <http://mypy-lang.org>`_ and `pyright <https://github.com/microsoft/pyright>`_ to statically analyze the code base and type hints. After making any change you can ensure your type hints are correct by running
+pandas uses `mypy <http://mypy-lang.org>`_ and `pyright <https://github.com/microsoft/pyright>`_ to statically analyze the code base and type hints. After making any change you can ensure your type hints are consistent by running
 
 .. code-block:: shell
 
+    pre-commit run --hook-stage manual --all-files mypy
+    pre-commit run --hook-stage manual --all-files pyright
+    pre-commit run --hook-stage manual --all-files pyright_reportGeneralTypeIssues
     # the following might fail if the installed pandas version does not correspond to your local git version
-    pre-commit run --hook-stage manual --all-files
+    pre-commit run --hook-stage manual --all-files stubtest
 
-    # if the above fails due to stubtest
-    SKIP=stubtest pre-commit run --hook-stage manual --all-files
+in your python environment.
+
+.. warning::
 
-in your activated python environment. A recent version of ``numpy`` (>=1.22.0) is required for type validation.
+    * Please be aware that the above commands will use the current python environment. If your python packages are older/newer than those installed by the pandas CI, the above commands might fail. This is often the case when the ``mypy`` or ``numpy`` versions do not match. Please see :ref:`how to setup the python environment <contributing.mamba>` or select a `recently succeeded workflow <https://github.com/pandas-dev/pandas/actions/workflows/code-checks.yml?query=branch%3Amain+is%3Asuccess>`_, select the "Docstring validation, typing, and other manual pre-commit hooks" job, then click on "Set up Conda" and "Environment info" to see which versions the pandas CI installs.
 
 .. _contributing.ci:
 

doc/source/development/contributing_environment.rst

@@ -95,6 +95,8 @@ Option 1: using mamba (recommended)
    mamba env create --file environment.yml
    mamba activate pandas-dev
 
+.. _contributing.pip:
+
 Option 2: using pip
 ~~~~~~~~~~~~~~~~~~~
 

doc/source/reference/arrays.rst

@@ -113,6 +113,8 @@ values.
 
    ArrowDtype
 
+For more information, please see the :ref:`PyArrow user guide <pyarrow>`
+
 .. _api.arrays.datetime:
 
 Datetimes

doc/source/user_guide/index.rst

@@ -64,6 +64,7 @@ Guides
     dsintro
     basics
     io
+    pyarrow
     indexing
     advanced
     merging

doc/source/user_guide/io.rst

@@ -290,6 +290,16 @@ date_parser : function, default ``None``
   values from the columns defined by parse_dates into a single array and pass
   that; and 3) call date_parser once for each row using one or more strings
   (corresponding to the columns defined by parse_dates) as arguments.
+
+  .. deprecated:: 2.0.0
+   Use ``date_format`` instead, or read in as ``object`` and then apply
+   :func:`to_datetime` as-needed.
+date_format : str or dict of column -> format, default ``None``
+   If used in conjunction with ``parse_dates``, will parse dates according to this
+   format. For anything more complex,
+   please read in as ``object`` and then apply :func:`to_datetime` as-needed.
+
+    .. versionadded:: 2.0.0
 dayfirst : boolean, default ``False``
   DD/MM format dates, international and European format.
 cache_dates : boolean, default True
@@ -800,7 +810,7 @@ Specifying date columns
 +++++++++++++++++++++++
 
 To better facilitate working with datetime data, :func:`read_csv`
-uses the keyword arguments ``parse_dates`` and ``date_parser``
+uses the keyword arguments ``parse_dates`` and ``date_format``
 to allow users to specify a variety of columns and date/time formats to turn the
 input text data into ``datetime`` objects.
 
@@ -898,33 +908,15 @@ data columns:
 Date parsing functions
 ++++++++++++++++++++++
 
-Finally, the parser allows you to specify a custom ``date_parser`` function to
-take full advantage of the flexibility of the date parsing API:
-
-.. ipython:: python
-
-   df = pd.read_csv(
-       "tmp.csv", header=None, parse_dates=date_spec, date_parser=pd.to_datetime
-   )
-   df
-
-pandas will try to call the ``date_parser`` function in three different ways. If
-an exception is raised, the next one is tried:
-
-1. ``date_parser`` is first called with one or more arrays as arguments,
-   as defined using ``parse_dates`` (e.g., ``date_parser(['2013', '2013'], ['1', '2'])``).
-
-2. If #1 fails, ``date_parser`` is called with all the columns
-   concatenated row-wise into a single array (e.g., ``date_parser(['2013 1', '2013 2'])``).
+Finally, the parser allows you to specify a custom ``date_format``.
+Performance-wise, you should try these methods of parsing dates in order:
 
-Note that performance-wise, you should try these methods of parsing dates in order:
+1. If you know the format, use ``date_format``, e.g.:
+   ``date_format="%d/%m/%Y"`` or ``date_format={column_name: "%d/%m/%Y"}``.
 
-1. If you know the format, use ``pd.to_datetime()``:
-   ``date_parser=lambda x: pd.to_datetime(x, format=...)``.
-
-2. If you have a really non-standard format, use a custom ``date_parser`` function.
-   For optimal performance, this should be vectorized, i.e., it should accept arrays
-   as arguments.
+2. If you different formats for different columns, or want to pass any extra options (such
+   as ``utc``) to ``to_datetime``, then you should read in your data as ``object`` dtype, and
+   then use ``to_datetime``.
 
 
 .. ipython:: python
@@ -952,16 +944,13 @@ an object-dtype column with strings, even with ``parse_dates``.
    df = pd.read_csv(StringIO(content), parse_dates=["a"])
    df["a"]
 
-To parse the mixed-timezone values as a datetime column, pass a partially-applied
-:func:`to_datetime` with ``utc=True`` as the ``date_parser``.
+To parse the mixed-timezone values as a datetime column, read in as ``object`` dtype and
+then call :func:`to_datetime` with ``utc=True``.
 
 .. ipython:: python
 
-   df = pd.read_csv(
-       StringIO(content),
-       parse_dates=["a"],
-       date_parser=lambda col: pd.to_datetime(col, utc=True),
-   )
+   df = pd.read_csv(StringIO(content))
+   df["a"] = pd.to_datetime(df["a"], utc=True)
    df["a"]