Merge branch 'main' into tguard

Author: topper-123

.github/workflows/32-bit-linux.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -52,3 +54,7 @@ jobs:
           name: Test results
           path: test-data.xml
         if: failure()
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit
+      cancel-in-progress: true

.github/workflows/code-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/docbuild-and-upload.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/macos-windows.yml

@@ -4,20 +4,21 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
   PATTERN: "not slow and not db and not network and not single_cpu"
-  ERROR_ON_WARNINGS: "1"
-
 
 permissions:
   contents: read

.github/workflows/package-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [ labeled, opened, synchronize, reopened ]
 

.github/workflows/python-dev.yml

@@ -33,6 +33,7 @@ on:
       - None
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 env:
   PYTEST_WORKERS: "auto"

.github/workflows/sdist.yml

@@ -4,14 +4,17 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 permissions:
   contents: read

.github/workflows/ubuntu.yml

@@ -4,13 +4,16 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
+      - "web/**"
 
 env:
   PANDAS_CI: 1
@@ -38,7 +41,6 @@ jobs:
           - name: "Minimum Versions"
             env_file: actions-38-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
-            error_on_warnings: "0"
           - name: "Locale: it_IT"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -63,12 +65,10 @@ jobs:
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
-            error_on_warnings: "0"
           - name: "Data Manager"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
-            error_on_warnings: "0"
           - name: "Pypy"
             env_file: actions-pypy-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -77,7 +77,6 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
-            error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -97,7 +96,6 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
-      ERROR_ON_WARNINGS: ${{ matrix.error_on_warnings || '1' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}

.github/workflows/wheels.yml

@@ -173,8 +173,8 @@ jobs:
           pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
           cd .. # Not a good idea to test within the src tree
           python -c "import pandas; print(pandas.__version__);
-          pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']);
-          pandas.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-network', '--skip-db'])"
+          pandas.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']);
+          pandas.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db'])"
       - uses: actions/upload-artifact@v3
         with:
           name: sdist

.pre-commit-config.yaml

@@ -313,7 +313,7 @@ repos:
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
-        additional_dependencies: [pyyaml, toml]
+        additional_dependencies: [tomli, pyyaml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -391,10 +391,11 @@ repos:
         types: [yaml]
     -   id: validate-min-versions-in-sync
         name: Check minimum version of dependencies are aligned
-        entry: python scripts/validate_min_versions_in_sync.py
+        entry: python -m scripts.validate_min_versions_in_sync
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
-        additional_dependencies: [tomli]
+        additional_dependencies: [tomli, pyyaml]
+        pass_filenames: false
     -   id: validate-errors-locations
         name: Validate errors locations
         description: Validate errors are in appropriate locations.

asv_bench/benchmarks/frame_methods.py

@@ -17,6 +17,21 @@
 from .pandas_vb_common import tm
 
 
+class Clip:
+    params = [
+        ["float64", "Float64", "float64[pyarrow]"],
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(100_000, 10)
+        df = DataFrame(data, dtype=dtype)
+        self.df = df
+
+    def time_clip(self, dtype):
+        self.df.clip(-1.0, 1.0)
+
+
 class GetNumericData:
     def setup(self):
         self.df = DataFrame(np.random.randn(10000, 25))
@@ -429,6 +444,22 @@ def time_dropna_axis_mixed_dtypes(self, how, axis):
         self.df_mixed.dropna(how=how, axis=axis)
 
 
+class Isna:
+    params = ["float64", "Float64", "float64[pyarrow]"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(10000, 1000)
+        # all-na columns
+        data[:, 600:800] = np.nan
+        # partial-na columns
+        data[800:1000, 4000:5000] = np.nan
+        self.df = DataFrame(data, dtype=dtype)
+
+    def time_isna(self, dtype):
+        self.df.isna()
+
+
 class Count:
     params = [0, 1]
     param_names = ["axis"]
@@ -739,4 +770,44 @@ def time_memory_usage_object_dtype(self):
         self.df2.memory_usage(deep=True)
 
 
+class Where:
+    params = (
+        [True, False],
+        ["float64", "Float64", "float64[pyarrow]"],
+    )
+    param_names = ["dtype"]
+
+    def setup(self, inplace, dtype):
+        self.df = DataFrame(np.random.randn(100_000, 10), dtype=dtype)
+        self.mask = self.df < 0
+
+    def time_where(self, inplace, dtype):
+        self.df.where(self.mask, other=0.0, inplace=inplace)
+
+
+class FindValidIndex:
+    param_names = ["dtype"]
+    params = [
+        ["float", "Float64", "float64[pyarrow]"],
+    ]
+
+    def setup(self, dtype):
+        df = DataFrame(
+            np.random.randn(100000, 2),
+            columns=list("AB"),
+            dtype=dtype,
+        )
+        df.iloc[:100, 0] = None
+        df.iloc[:200, 1] = None
+        df.iloc[-100:, 0] = None
+        df.iloc[-200:, 1] = None
+        self.df = df
+
+    def time_first_valid_index(self, dtype):
+        self.df.first_valid_index()
+
+    def time_last_valid_index(self, dtype):
+        self.df.last_valid_index()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -86,23 +86,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     MSG='Partially validate docstrings (EX01)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01 --ignore_functions \
         pandas.Series.index \
-        pandas.Series.nbytes \
-        pandas.Series.ndim \
-        pandas.Series.size \
-        pandas.Series.T \
         pandas.Series.hasnans \
         pandas.Series.to_list \
         pandas.Series.__iter__ \
         pandas.Series.keys \
         pandas.Series.item \
         pandas.Series.pipe \
-        pandas.Series.kurt \
-        pandas.Series.mean \
-        pandas.Series.median \
         pandas.Series.mode \
         pandas.Series.sem \
         pandas.Series.skew \
-        pandas.Series.kurtosis \
         pandas.Series.is_unique \
         pandas.Series.is_monotonic_increasing \
         pandas.Series.is_monotonic_decreasing \
@@ -549,10 +541,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.keys \
         pandas.DataFrame.iterrows \
         pandas.DataFrame.pipe \
-        pandas.DataFrame.kurt \
-        pandas.DataFrame.kurtosis \
-        pandas.DataFrame.mean \
-        pandas.DataFrame.median \
         pandas.DataFrame.sem \
         pandas.DataFrame.skew \
         pandas.DataFrame.backfill \
@@ -573,24 +561,9 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
         pandas.DataFrame.plot.line \
         pandas.Series.plot.line \
-        pandas.api.types.infer_dtype \
         pandas.api.types.is_datetime64_any_dtype \
         pandas.api.types.is_datetime64_ns_dtype \
         pandas.api.types.is_datetime64tz_dtype \
-        pandas.api.types.is_integer_dtype \
-        pandas.api.types.is_interval_dtype \
-        pandas.api.types.is_period_dtype \
-        pandas.api.types.is_signed_integer_dtype \
-        pandas.api.types.is_sparse \
-        pandas.api.types.is_string_dtype \
-        pandas.api.types.is_timedelta64_dtype \
-        pandas.api.types.is_timedelta64_ns_dtype \
-        pandas.api.types.is_unsigned_integer_dtype \
-        pandas.io.formats.style.Styler.concat \
-        pandas.io.formats.style.Styler.export \
-        pandas.io.formats.style.Styler.set_td_classes \
-        pandas.io.formats.style.Styler.use \
-        pandas.io.json.build_table_schema \
         pandas.plotting.andrews_curves \
         pandas.plotting.autocorrelation_plot \
         pandas.plotting.lag_plot \

ci/deps/actions-310-numpydev.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.34.2
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil