Merge branch 'main' into gha-win-mac

Author: jonashaag

.github/workflows/32-bit-linux.yml

@@ -0,0 +1,43 @@
+name: 32 Bit Linux
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run 32-bit manylinux2014 Docker Build / Tests
+        run: |
+          docker pull quay.io/pypa/manylinux2014_i686
+          docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
+          /bin/bash -xc "cd pandas && \
+          /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
+          . ~/virtualenvs/pandas-dev/bin/activate && \
+          python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
+          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python setup.py build_ext -q -j2 && \
+          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
+          export PANDAS_CI=1 && \
+          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
+
+      - name: Publish test results for Python 3.8-32 bit full Linux
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test results
+          path: test-data.xml
+        if: failure()

.github/workflows/code-checks.yml

@@ -74,7 +74,7 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g pyright@1.1.245
+      run: npm install -g pyright@1.1.247
 
     - name: Build Pandas
       id: build

.github/workflows/macos-windows.yml

@@ -1,4 +1,4 @@
-name: macOS and Windows
+name: Windows-MacOS
 
 on:
   push:
@@ -18,23 +18,23 @@ env:
   PYTEST_WORKERS: auto
   PATTERN: "not slow and not db and not network and not single_cpu"
 
+
 jobs:
   pytest:
-    runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash -el {0}
     timeout-minutes: 90
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-latest]
+        os: [macos-latest, windows-latest]
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
-        pyarrow_version: [6]
       fail-fast: false
+    runs-on: ${{ matrix.os }}
     name: ${{ matrix.name || format('{0} {1} pyarrow={2}', matrix.os, matrix.env_file, matrix.pyarrow_version) }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.env_file }}-${{ matrix.pyarrow_version }}-mac-win
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.os }}
       cancel-in-progress: true
 
     steps:

.github/workflows/posix.yml

@@ -155,24 +155,11 @@ jobs:
         channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
-      if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support
 
     - name: Upgrade Arrow version
       run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
       if: ${{ matrix.pyarrow_version }}
 
-    - name: Setup PyPy
-      uses: actions/setup-python@v3
-      with:
-        python-version: "pypy-3.8"
-      if: ${{ env.IS_PYPY == 'true' }}
-
-    - name: Setup PyPy dependencies
-      run: |
-        # TODO: re-enable cov, its slowing the tests down though
-        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3
-      if: ${{ env.IS_PYPY == 'true' }}
-
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 

.github/workflows/python-dev.yml

@@ -2,7 +2,7 @@
 # Unfreeze(by commentingthe if: false() condition) once the
 # next Python Dev version has released beta 1 and both Cython and numpy support it
 # After that Python has released, migrate the workflows to the
-# posix GHA workflows/Azure pipelines and "freeze" this file by
+# posix GHA workflows and "freeze" this file by
 # uncommenting the if: false() condition
 # Feel free to modify this comment as necessary.
 

.github/workflows/windows.yml

@@ -89,7 +89,7 @@ repos:
         types: [python]
         stages: [manual]
         # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['pyright@1.1.245']
+        additional_dependencies: ['pyright@1.1.247']
 -   repo: local
     hooks:
     -   id: flake8-rst

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 https://github.com/client9/stringencoders
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 http://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
  * Copyright (c) 1988-1993 The Regents of the University of California.
  * Copyright (c) 1994 Sun Microsystems, Inc.

LICENSES/ULTRAJSON_LICENSE

@@ -10,7 +10,6 @@
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3509134.svg)](https://doi.org/10.5281/zenodo.3509134)
 [![Package Status](https://img.shields.io/pypi/status/pandas.svg)](https://pypi.org/project/pandas/)
 [![License](https://img.shields.io/pypi/l/pandas.svg)](https://github.com/pandas-dev/pandas/blob/main/LICENSE)
-[![Azure Build Status](https://dev.azure.com/pandas-dev/pandas/_apis/build/status/pandas-dev.pandas?branch=main)](https://dev.azure.com/pandas-dev/pandas/_build/latest?definitionId=1&branch=main)
 [![Coverage](https://codecov.io/github/pandas-dev/pandas/coverage.svg?branch=main)](https://codecov.io/gh/pandas-dev/pandas)
 [![Downloads](https://static.pepy.tech/personalized-badge/pandas?period=month&units=international_system&left_color=black&right_color=orange&left_text=PyPI%20downloads%20per%20month)](https://pepy.tech/project/pandas)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pydata/pandas)

README.md

@@ -1,3 +1,6 @@
+from functools import wraps
+import threading
+
 import numpy as np
 
 from pandas import (
@@ -30,21 +33,57 @@
     from pandas._libs import algos
 except ImportError:
     from pandas import algos
-try:
-    from pandas._testing import test_parallel  # noqa: PDF014
 
-    have_real_test_parallel = True
-except ImportError:
-    have_real_test_parallel = False
 
-    def test_parallel(num_threads=1):
-        def wrapper(fname):
-            return fname
+from .pandas_vb_common import BaseIO  # isort:skip
 
-        return wrapper
 
+def test_parallel(num_threads=2, kwargs_list=None):
+    """
+    Decorator to run the same function multiple times in parallel.
 
-from .pandas_vb_common import BaseIO  # isort:skip
+    Parameters
+    ----------
+    num_threads : int, optional
+        The number of times the function is run in parallel.
+    kwargs_list : list of dicts, optional
+        The list of kwargs to update original
+        function kwargs on different threads.
+
+    Notes
+    -----
+    This decorator does not pass the return value of the decorated function.
+
+    Original from scikit-image:
+
+    https://github.com/scikit-image/scikit-image/pull/1519
+
+    """
+    assert num_threads > 0
+    has_kwargs_list = kwargs_list is not None
+    if has_kwargs_list:
+        assert len(kwargs_list) == num_threads
+
+    def wrapper(func):
+        @wraps(func)
+        def inner(*args, **kwargs):
+            if has_kwargs_list:
+                update_kwargs = lambda i: dict(kwargs, **kwargs_list[i])
+            else:
+                update_kwargs = lambda i: kwargs
+            threads = []
+            for i in range(num_threads):
+                updated_kwargs = update_kwargs(i)
+                thread = threading.Thread(target=func, args=args, kwargs=updated_kwargs)
+                threads.append(thread)
+            for thread in threads:
+                thread.start()
+            for thread in threads:
+                thread.join()
+
+        return inner
+
+    return wrapper
 
 
 class ParallelGroupbyMethods:
@@ -53,8 +92,7 @@ class ParallelGroupbyMethods:
     param_names = ["threads", "method"]
 
     def setup(self, threads, method):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         N = 10**6
         ngroups = 10**3
         df = DataFrame(
@@ -86,8 +124,7 @@ class ParallelGroups:
     param_names = ["threads"]
 
     def setup(self, threads):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         size = 2**22
         ngroups = 10**3
         data = Series(np.random.randint(0, ngroups, size=size))
@@ -108,8 +145,7 @@ class ParallelTake1D:
     param_names = ["dtype"]
 
     def setup(self, dtype):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         N = 10**6
         df = DataFrame({"col": np.arange(N, dtype=dtype)})
         indexer = np.arange(100, len(df) - 100)
@@ -131,8 +167,7 @@ class ParallelKth:
     repeat = 5
 
     def setup(self):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         N = 10**7
         k = 5 * 10**5
         kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}]
@@ -149,8 +184,7 @@ def time_kth_smallest(self):
 
 class ParallelDatetimeFields:
     def setup(self):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         N = 10**6
         self.dti = date_range("1900-01-01", periods=N, freq="T")
         self.period = self.dti.to_period("D")
@@ -204,8 +238,7 @@ class ParallelRolling:
     param_names = ["method"]
 
     def setup(self, method):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         win = 100
         arr = np.random.rand(100000)
         if hasattr(DataFrame, "rolling"):
@@ -248,8 +281,7 @@ class ParallelReadCSV(BaseIO):
     param_names = ["dtype"]
 
     def setup(self, dtype):
-        if not have_real_test_parallel:
-            raise NotImplementedError
+
         rows = 10000
         cols = 50
         data = {
@@ -284,8 +316,6 @@ class ParallelFactorize:
     param_names = ["threads"]
 
     def setup(self, threads):
-        if not have_real_test_parallel:
-            raise NotImplementedError
 
         strings = tm.makeStringIndex(100000)
 

asv_bench/benchmarks/gil.py

@@ -146,10 +146,10 @@ def setup(self, fill_value):
 
     def make_block_array(self, length, num_blocks, block_size, fill_value):
         arr = np.full(length, fill_value)
-        indicies = np.random.choice(
+        indices = np.random.choice(
             np.arange(0, length, block_size), num_blocks, replace=False
         )
-        for ind in indicies:
+        for ind in indices:
             arr[ind : ind + block_size] = np.random.randint(0, 100, block_size)
         return SparseArray(arr, fill_value=fill_value)
 

asv_bench/benchmarks/sparse.py

@@ -268,7 +268,7 @@ def time_get_dummies(self, dtype):
 
 class Encode:
     def setup(self):
-        self.ser = Series(tm.makeUnicodeIndex())
+        self.ser = Series(tm.makeStringIndex())
 
     def time_encode_decode(self):
         self.ser.str.encode("utf-8").str.decode("utf-8")