Merge branch 'main' into raise-on-parse-int-overflow

Author: SandroCasagrande

.github/workflows/code-checks.yml

@@ -153,6 +153,9 @@ jobs:
       - name: Build image
         run: docker build --pull --no-cache --tag pandas-dev-env .
 
+      - name: Show environment
+        run: docker run -w /home/pandas pandas-dev-env mamba run -n pandas-dev python -c "import pandas as pd; print(pd.show_versions())"
+
   requirements-dev-text-installable:
     name: Test install requirements-dev.txt
     runs-on: ubuntu-latest

.github/workflows/macos-windows.yml

@@ -28,7 +28,7 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
-    timeout-minutes: 120
+    timeout-minutes: 180
     strategy:
       matrix:
         os: [macos-latest, windows-latest]

.github/workflows/python-dev.yml

@@ -80,9 +80,10 @@ jobs:
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
+    # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
     - name: Build Pandas
       run: |
-        python setup.py build_ext -q -j2
+        python setup.py build_ext -q -j1
         python -m pip install -e . --no-build-isolation --no-use-pep517
 
     - name: Build Version

.github/workflows/ubuntu.yml

@@ -26,7 +26,7 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
-    timeout-minutes: 120
+    timeout-minutes: 180
     strategy:
       matrix:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
@@ -42,18 +42,26 @@ jobs:
           - name: "Minimum Versions"
             env_file: actions-38-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
-          - name: "Locale: it_IT.utf8"
+          - name: "Locale: it_IT"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-it"
+            # Use the utf8 version as the default, it has no bad side-effect.
             lang: "it_IT.utf8"
             lc_all: "it_IT.utf8"
-          - name: "Locale: zh_CN.utf8"
+            # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
+            # It will be temporarily activated during tests with locale.setlocale
+            extra_loc: "it_IT"
+          - name: "Locale: zh_CN"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             extra_apt: "language-pack-zh-hans"
+            # Use the utf8 version as the default, it has no bad side-effect.
             lang: "zh_CN.utf8"
             lc_all: "zh_CN.utf8"
+            # Also install zh_CN (its encoding is gb2312) but do not activate it.
+            # It will be temporarily activated during tests with locale.setlocale
+            extra_loc: "zh_CN"
           - name: "Copy-on-Write"
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -148,6 +156,12 @@ jobs:
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y xsel ${{ env.EXTRA_APT }}
 
+    - name: Generate extra locales
+      # These extra locales will be available for locale.setlocale() calls in tests
+      run: |
+        sudo locale-gen ${{ matrix.extra_loc }}
+      if: ${{ matrix.extra_loc }}
+
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
       with:

.libcst.codemod.yaml

@@ -0,0 +1,18 @@
+# String that LibCST should look for in code which indicates that the
+# module is generated code.
+generated_code_marker: '@generated'
+# Command line and arguments for invoking a code formatter. Anything
+# specified here must be capable of taking code via stdin and returning
+# formatted code via stdout.
+formatter: ['black', '-']
+# List of regex patterns which LibCST will evaluate against filenames to
+# determine if the module should be touched.
+blacklist_patterns: []
+# List of modules that contain codemods inside of them.
+modules:
+- 'libcst.codemod.commands'
+- 'autotyping'
+# Absolute or relative path of the repository root, used for providing
+# full-repo metadata. Relative paths should be specified with this file
+# location as the base.
+repo_root: '.'

.pre-commit-config.yaml

@@ -22,10 +22,14 @@ repos:
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.1.0
+    rev: v2.2.1
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
+-   repo: https://github.com/MarcoGorelli/cython-lint
+    rev: v0.1.4
+    hooks:
+    -   id: cython-lint
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
@@ -252,3 +256,14 @@ repos:
             /(__init__\.py)|(api\.py)|(_version\.py)|(testing\.py)|(conftest\.py)$
             |/tests/
             |/_testing/
+    -   id: autotyping
+        name: autotyping
+        entry: python -m libcst.tool codemod autotyping.AutotypeCommand --none-return --scalar-return --annotate-magics --annotate-imprecise-magics
+        types_or: [python, pyi]
+        files: ^pandas
+        exclude: ^(pandas/tests|pandas/io/clipboard)
+        language: python
+        additional_dependencies:
+        - autotyping==22.9.0
+        - black==22.6.0
+        - libcst==0.4.7

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/condaforge/mambaforge:4.13.0-1
+FROM quay.io/condaforge/mambaforge
 
 # if you forked pandas, you can pass in your own GitHub username to use your fork
 # i.e. gh_username=myname
@@ -10,16 +10,12 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Configure apt and install packages
 RUN apt-get update \
-    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \
+    && apt-get -y install --no-install-recommends apt-utils git tzdata dialog 2>&1 \
     #
-    # Install tzdata and configure timezone (fix for tests which try to read from "/etc/localtime")
-    && apt-get -y install tzdata \
+    # Configure timezone (fix for tests which try to read from "/etc/localtime")
     && ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime \
     && dpkg-reconfigure -f noninteractive tzdata \
     #
-    # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed
-    && apt-get -y install git iproute2 procps iproute2 lsb-release \
-    #
     # cleanup
     && apt-get autoremove -y \
     && apt-get clean -y \
@@ -35,18 +31,12 @@ RUN mkdir "$pandas_home" \
     && git remote add upstream "https://github.com/pandas-dev/pandas.git" \
     && git pull upstream main
 
-# Because it is surprisingly difficult to activate a conda environment inside a DockerFile
-# (from personal experience and per https://github.com/ContinuumIO/docker-images/issues/89),
-# we just update the base/root one from the 'environment.yml' file instead of creating a new one.
-#
 # Set up environment
-RUN mamba env update -n base -f "$pandas_home/environment.yml"
+RUN mamba env create -f "$pandas_home/environment.yml"
 
 # Build C extensions and pandas
-SHELL ["/bin/bash", "-c"]
-RUN . /opt/conda/etc/profile.d/conda.sh \
-    && conda activate base \
-    && cd "$pandas_home" \
+SHELL ["mamba", "run", "--no-capture-output", "-n", "pandas-dev", "/bin/bash", "-c"]
+RUN cd "$pandas_home" \
     && export \
     && python setup.py build_ext -j 4 \
     && python -m pip install --no-build-isolation -e .

asv_bench/benchmarks/algorithms.py

@@ -95,6 +95,29 @@ def time_duplicated(self, unique, keep, dtype):
         self.idx.duplicated(keep=keep)
 
 
+class DuplicatedMaskedArray:
+
+    params = [
+        [True, False],
+        ["first", "last", False],
+        ["Int64", "Float64"],
+    ]
+    param_names = ["unique", "keep", "dtype"]
+
+    def setup(self, unique, keep, dtype):
+        N = 10**5
+        data = pd.Series(np.arange(N), dtype=dtype)
+        data[list(range(1, N, 100))] = pd.NA
+        if not unique:
+            data = data.repeat(5)
+        self.ser = data
+        # cache is_unique
+        self.ser.is_unique
+
+    def time_duplicated(self, unique, keep, dtype):
+        self.ser.duplicated(keep=keep)
+
+
 class Hashing:
     def setup_cache(self):
         N = 10**5

asv_bench/benchmarks/array.py

@@ -32,9 +32,10 @@ def time_from_float_array(self):
 
 class IntegerArray:
     def setup(self):
-        self.values_integer = np.array([1, 0, 1, 0])
-        self.data = np.array([1, 2, 3, 4], dtype="int64")
-        self.mask = np.array([False, False, True, False])
+        N = 250_000
+        self.values_integer = np.array([1, 0, 1, 0] * N)
+        self.data = np.array([1, 2, 3, 4] * N, dtype="int64")
+        self.mask = np.array([False, False, True, False] * N)
 
     def time_constructor(self):
         pd.arrays.IntegerArray(self.data, self.mask)

asv_bench/benchmarks/multiindex_object.py

@@ -8,6 +8,7 @@
     MultiIndex,
     RangeIndex,
     Series,
+    array,
     date_range,
 )
 
@@ -176,6 +177,20 @@ def time_sortlevel_one(self):
         self.mi.sortlevel(1)
 
 
+class SortValues:
+
+    params = ["int64", "Int64"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        a = array(np.tile(np.arange(100), 1000), dtype=dtype)
+        b = array(np.tile(np.arange(1000), 100), dtype=dtype)
+        self.mi = MultiIndex.from_arrays([a, b])
+
+    def time_sort_values(self, dtype):
+        self.mi.sort_values()
+
+
 class Values:
     def setup_cache(self):
 

asv_bench/benchmarks/series_methods.py

@@ -3,6 +3,7 @@
 import numpy as np
 
 from pandas import (
+    NA,
     Index,
     NaT,
     Series,
@@ -166,6 +167,19 @@ def time_value_counts(self, N, dtype):
         self.s.value_counts()
 
 
+class ValueCountsEA:
+
+    params = [[10**3, 10**4, 10**5], [True, False]]
+    param_names = ["N", "dropna"]
+
+    def setup(self, N, dropna):
+        self.s = Series(np.random.randint(0, N, size=10 * N), dtype="Int64")
+        self.s.loc[1] = NA
+
+    def time_value_counts(self, N, dropna):
+        self.s.value_counts(dropna=dropna)
+
+
 class ValueCountsObjectDropNAFalse:
 
     params = [10**3, 10**4, 10**5]

doc/source/development/contributing_environment.rst

@@ -237,6 +237,16 @@ Run Container::
     # Run a container and bind your local repo to the container
     docker run -it -w /home/pandas --rm -v path-to-local-pandas-repo:/home/pandas pandas-yourname-env
 
+Then a ``pandas-dev`` virtual environment will be available with all the development dependencies.
+
+.. code-block:: shell
+
+    root@... :/home/pandas# conda env list
+    # conda environments:
+    #
+    base                  *  /opt/conda
+    pandas-dev               /opt/conda/envs/pandas-dev
+
 .. note::
     If you bind your local repo for the first time, you have to build the C extensions afterwards.
     Run the following command inside the container::