Merge branch 'pandas-dev:main' into raise-on-parse-int-overflow

Author: SandroCasagrande

.github/workflows/wheels.yml

@@ -54,7 +54,6 @@ jobs:
         # TODO: support PyPy?
         python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11-dev"]]# "pp38", "pp39"]
     env:
-      IS_32_BIT: ${{ matrix.buildplat[1] == 'win32' }}
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
     steps:
@@ -72,17 +71,8 @@ jobs:
         uses: pypa/cibuildwheel@v2.9.0
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
-          CIBW_ENVIRONMENT: IS_32_BIT='${{ env.IS_32_BIT }}'
-          # We can't test directly with cibuildwheel, since we need to have to wheel location
-          # to mount into the docker image
-          CIBW_TEST_COMMAND_LINUX: "python {project}/ci/test_wheels.py"
-          CIBW_TEST_COMMAND_MACOS: "python {project}/ci/test_wheels.py"
-          CIBW_TEST_REQUIRES: hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "python ci/fix_wheels.py {wheel} {dest_dir}"
-          CIBW_ARCHS_MACOS: x86_64 universal2
-          CIBW_BUILD_VERBOSITY: 3
 
-      # Used to push the built wheels
+      # Used to test the built wheels
       - uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python[1] }}
@@ -98,9 +88,27 @@ jobs:
           name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
 
+      # Used to push the built wheels
+      # TODO: once Python 3.11 is available on conda, de-dup with
+      # setup python above
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          # Really doesn't matter what version we upload with
+          # just the version we test with
+          python-version: '3.8'
+          channels: conda-forge
+          channel-priority: true
+          mamba-version: "*"
+
+      - name: Install anaconda client
+        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+        run: conda install -q -y anaconda-client
+
+
       - name: Upload wheels
         if: success()
-        shell: bash
+        shell: bash -el {0}
         env:
           PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
           PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}
@@ -136,16 +144,24 @@ jobs:
           # A shallow clone can work when the following issue is resolved:
           # https://github.com/actions/checkout/issues/338
           fetch-depth: 0
-      # Used to push the built wheels
-      - uses: actions/setup-python@v3
+
+      # Used to push the built sdist
+      - uses: conda-incubator/setup-miniconda@v2
         with:
-          # Build sdist on lowest supported Python
+          auto-update-conda: true
+          # Really doesn't matter what version we upload with
+          # just the version we test with
           python-version: '3.8'
+          channels: conda-forge
+          channel-priority: true
+          mamba-version: "*"
+
       - name: Build sdist
         run: |
           pip install build
           python -m build --sdist
       - name: Test the sdist
+        shell: bash -el {0}
         run: |
           # TODO: Don't run test suite, and instead build wheels from sdist
           # by splitting the wheel builders into a two stage job
@@ -162,9 +178,14 @@ jobs:
           name: sdist
           path: ./dist/*
 
+      - name: Install anaconda client
+        if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }}
+        run: |
+          conda install -q -y anaconda-client
+
       - name: Upload sdist
         if: success()
-        shell: bash
+        shell: bash -el {0}
         env:
           PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
           PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}

.pre-commit-config.yaml

@@ -11,14 +11,14 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.5'
+    rev: 'v2.6'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/python/black
-    rev: 22.6.0
+    rev: 22.8.0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -27,7 +27,7 @@ repos:
     -   id: codespell
         types_or: [python, rst, markdown]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.1.4
+    rev: v0.1.8
     hooks:
     -   id: cython-lint
 -   repo: https://github.com/pre-commit/pre-commit-hooks
@@ -40,7 +40,7 @@ repos:
     -   id: trailing-whitespace
         stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
 -   repo: https://github.com/cpplint/cpplint
-    rev: 1.6.0
+    rev: 1.6.1
     hooks:
     -   id: cpplint
         # We don't lint all C files because we don't want to lint any that are built
@@ -59,12 +59,16 @@ repos:
         - flake8==5.0.4
         - flake8-bugbear==22.7.1
         - pandas-dev-flaker==0.5.0
+-   repo: https://github.com/pycqa/pylint
+    rev: v2.15.3
+    hooks:
+    -   id: pylint
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.37.3
+    rev: v2.38.2
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -83,7 +87,7 @@ repos:
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
-    rev: v1.3.0
+    rev: v1.4.0
     hooks:
     -   id: yesqa
         additional_dependencies: *flake8_dependencies
@@ -222,6 +226,13 @@ repos:
         entry: python scripts/no_bool_in_generic.py
         language: python
         files: ^pandas/core/generic\.py$
+    -   id: no-return-exception
+        name: Use raise instead of return for exceptions
+        language: pygrep
+        entry: 'return [A-Za-z]+(Error|Exit|Interrupt|Exception|Iteration)'
+        files: ^pandas/
+        types: [python]
+        exclude: ^pandas/tests/
     -   id: pandas-errors-documented
         name: Ensure pandas errors are documented in doc/source/reference/testing.rst
         entry: python scripts/pandas_errors_documented.py
@@ -268,7 +279,7 @@ repos:
             |/_testing/
     -   id: autotyping
         name: autotyping
-        entry: python -m libcst.tool codemod autotyping.AutotypeCommand --none-return --scalar-return --annotate-magics --annotate-imprecise-magics --bool-param --bytes-param --str-param --float-param
+        entry: python -m scripts.run_autotyping
         types_or: [python, pyi]
         files: ^pandas
         exclude: ^(pandas/tests|pandas/_version.py|pandas/io/clipboard)

Makefile

@@ -65,6 +65,15 @@ def time_datetime_difference_disjoint(self):
         self.datetime_left.difference(self.datetime_right)
 
 
+class UnionWithDuplicates:
+    def setup(self):
+        self.left = Index(np.repeat(np.arange(1000), 100))
+        self.right = Index(np.tile(np.arange(500, 1500), 50))
+
+    def time_union_with_duplicates(self):
+        self.left.union(self.right)
+
+
 class Range:
     def setup(self):
         self.idx_inc = RangeIndex(start=0, stop=10**6, step=3)

asv_bench/benchmarks/index_object.py

@@ -1,30 +1,23 @@
-import os
+from pathlib import Path
 
 from pandas import read_sas
 
+ROOT = Path(__file__).parents[3] / "pandas" / "tests" / "io" / "sas" / "data"
+
 
 class SAS:
+    def time_read_sas7bdat(self):
+        read_sas(ROOT / "test1.sas7bdat")
 
-    params = ["sas7bdat", "xport"]
-    param_names = ["format"]
+    def time_read_xpt(self):
+        read_sas(ROOT / "paxraw_d_short.xpt")
 
-    def setup(self, format):
-        # Read files that are located in 'pandas/tests/io/sas/data'
-        files = {"sas7bdat": "test1.sas7bdat", "xport": "paxraw_d_short.xpt"}
-        file = files[format]
-        paths = [
-            os.path.dirname(__file__),
-            "..",
-            "..",
-            "..",
-            "pandas",
-            "tests",
-            "io",
-            "sas",
-            "data",
-            file,
-        ]
-        self.f = os.path.join(*paths)
+    def time_read_sas7bdat_2(self):
+        next(read_sas(ROOT / "0x00controlbyte.sas7bdat.bz2", chunksize=11000))
 
-    def time_read_sas(self, format):
-        read_sas(self.f, format=format)
+    def time_read_sas7bdat_2_chunked(self):
+        for i, _ in enumerate(
+            read_sas(ROOT / "0x00controlbyte.sas7bdat.bz2", chunksize=1000)
+        ):
+            if i == 10:
+                break

asv_bench/benchmarks/io/sas.py

@@ -276,6 +276,45 @@ def time_operation(self, index_structure, dtype, method):
         getattr(self.left, method)(self.right)
 
 
+class Difference:
+
+    params = [
+        ("datetime", "int", "string", "ea_int"),
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N = 10**4 * 2
+        level1 = range(1000)
+
+        level2 = date_range(start="1/1/2000", periods=N // 1000)
+        dates_left = MultiIndex.from_product([level1, level2])
+
+        level2 = range(N // 1000)
+        int_left = MultiIndex.from_product([level1, level2])
+
+        level2 = Series(range(N // 1000), dtype="Int64")
+        level2[0] = NA
+        ea_int_left = MultiIndex.from_product([level1, level2])
+
+        level2 = tm.makeStringIndex(N // 1000).values
+        str_left = MultiIndex.from_product([level1, level2])
+
+        data = {
+            "datetime": dates_left,
+            "int": int_left,
+            "ea_int": ea_int_left,
+            "string": str_left,
+        }
+
+        data = {k: {"left": mi, "right": mi[:5]} for k, mi in data.items()}
+        self.left = data[dtype]["left"]
+        self.right = data[dtype]["right"]
+
+    def time_difference(self, dtype):
+        self.left.difference(self.right)
+
+
 class Unique:
     params = [
         (("Int64", NA), ("int64", 0)),

asv_bench/benchmarks/multiindex_object.py

@@ -24,7 +24,6 @@ upload_wheels() {
         if [ -z ${TOKEN} ]; then
             echo no token set, not uploading
         else
-            conda install -q -y anaconda-client
             # sdists are located under dist folder when built through setup.py
             if compgen -G "./dist/*.gz"; then
                 echo "Found sdist"