Merge branch 'main' into raise-on-parse-int-overflow

Author: SandroCasagrande

.github/workflows/scorecards.yml

@@ -29,7 +29,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@v2.0.3
+        uses: ossf/scorecard-action@v2.0.6
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/wheels.yml

@@ -52,7 +52,7 @@ jobs:
         - [windows-2019, win_amd64]
         - [windows-2019, win32]
         # TODO: support PyPy?
-        python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11-dev"]]# "pp38", "pp39"]
+        python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11"]]# "pp38", "pp39"]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -73,7 +73,7 @@ jobs:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       # Used to test the built wheels
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python[1] }}
 

.pre-commit-config.yaml

@@ -102,7 +102,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.264
+        - pyright@1.1.276
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

asv_bench/asv.conf.json

@@ -54,7 +54,6 @@
         "openpyxl": [],
         "xlsxwriter": [],
         "xlrd": [],
-        "xlwt": [],
         "odfpy": [],
         "jinja2": [],
     },

asv_bench/benchmarks/array.py

@@ -44,6 +44,24 @@ def time_from_integer_array(self):
         pd.array(self.values_integer, dtype="Int64")
 
 
+class StringArray:
+    def setup(self):
+        N = 100_000
+        values = tm.rands_array(3, N)
+        self.values_obj = np.array(values, dtype="object")
+        self.values_str = np.array(values, dtype="U")
+        self.values_list = values.tolist()
+
+    def time_from_np_object_array(self):
+        pd.array(self.values_obj, dtype="string")
+
+    def time_from_np_str_array(self):
+        pd.array(self.values_str, dtype="string")
+
+    def time_from_list(self):
+        pd.array(self.values_list, dtype="string")
+
+
 class ArrowStringArray:
 
     params = [False, True]

asv_bench/benchmarks/groupby.py

@@ -14,6 +14,7 @@
     Timestamp,
     date_range,
     period_range,
+    to_timedelta,
 )
 
 from .pandas_vb_common import tm
@@ -35,7 +36,6 @@
         "pct_change",
         "min",
         "var",
-        "mad",
         "describe",
         "std",
         "quantile",
@@ -52,7 +52,6 @@
         "cummax",
         "pct_change",
         "var",
-        "mad",
         "describe",
         "std",
     },
@@ -311,7 +310,7 @@ def time_different_python_functions_multicol(self, df):
         df.groupby(["key1", "key2"]).agg([sum, min, max])
 
     def time_different_python_functions_singlecol(self, df):
-        df.groupby("key1").agg([sum, min, max])
+        df.groupby("key1")[["value1", "value2", "value3"]].agg([sum, min, max])
 
 
 class GroupStrings:
@@ -437,7 +436,6 @@ class GroupByMethods:
             "first",
             "head",
             "last",
-            "mad",
             "max",
             "min",
             "median",
@@ -483,7 +481,7 @@ def setup(self, dtype, method, application, ncols):
 
         if method == "describe":
             ngroups = 20
-        elif method in ["mad", "skew"]:
+        elif method == "skew":
             ngroups = 100
         else:
             ngroups = 1000
@@ -685,7 +683,7 @@ class String:
     def setup(self, dtype, method):
         cols = list("abcdefghjkl")
         self.df = DataFrame(
-            np.random.randint(0, 100, size=(1_000_000, len(cols))),
+            np.random.randint(0, 100, size=(10_000, len(cols))),
             columns=cols,
             dtype=dtype,
         )
@@ -990,4 +988,31 @@ def time_sample_weights(self):
         self.df.groupby(self.groups).sample(n=1, weights=self.weights)
 
 
+class Resample:
+    # GH 28635
+    def setup(self):
+        num_timedeltas = 20_000
+        num_groups = 3
+
+        index = MultiIndex.from_product(
+            [
+                np.arange(num_groups),
+                to_timedelta(np.arange(num_timedeltas), unit="s"),
+            ],
+            names=["groups", "timedeltas"],
+        )
+        data = np.random.randint(0, 1000, size=(len(index)))
+
+        self.df = DataFrame(data, index=index).reset_index("timedeltas")
+        self.df_multiindex = DataFrame(data, index=index)
+
+    def time_resample(self):
+        self.df.groupby(level="groups").resample("10s", on="timedeltas").mean()
+
+    def time_resample_multiindex(self):
+        self.df_multiindex.groupby(level="groups").resample(
+            "10s", level="timedeltas"
+        ).mean()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/excel.py

@@ -33,7 +33,7 @@ def _generate_dataframe():
 
 class WriteExcel:
 
-    params = ["openpyxl", "xlsxwriter", "xlwt"]
+    params = ["openpyxl", "xlsxwriter"]
     param_names = ["engine"]
 
     def setup(self, engine):
@@ -68,10 +68,9 @@ def time_write_excel_style(self, engine):
 
 class ReadExcel:
 
-    params = ["xlrd", "openpyxl", "odf"]
+    params = ["openpyxl", "odf"]
     param_names = ["engine"]
     fname_excel = "spreadsheet.xlsx"
-    fname_excel_xls = "spreadsheet.xls"
     fname_odf = "spreadsheet.ods"
 
     def _create_odf(self):
@@ -92,13 +91,10 @@ def setup_cache(self):
         self.df = _generate_dataframe()
 
         self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
-        self.df.to_excel(self.fname_excel_xls, sheet_name="Sheet1")
         self._create_odf()
 
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel
@@ -107,9 +103,7 @@ def time_read_excel(self, engine):
 
 class ReadExcelNRows(ReadExcel):
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel

asv_bench/benchmarks/io/sql.py

@@ -38,7 +38,7 @@ def setup(self, connection):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -88,7 +88,7 @@ def setup(self, connection, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -117,7 +117,7 @@ def setup(self):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
@@ -164,7 +164,7 @@ def setup(self, dtype):
             },
             index=tm.makeStringIndex(N),
         )
-        self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df.iloc[1000:3000, 1] = np.nan
         self.df["date"] = self.df["datetime"].dt.date
         self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)

asv_bench/benchmarks/io/stata.py

@@ -38,13 +38,13 @@ def setup(self, convert_dates):
         )
         self.df["float32_"] = np.array(np.random.randn(N), dtype=np.float32)
         self.convert_dates = {"index": convert_dates}
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
     def time_read_stata(self, convert_dates):
         read_stata(self.fname)
 
     def time_write_stata(self, convert_dates):
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 class StataMissing(Stata):
@@ -54,7 +54,7 @@ def setup(self, convert_dates):
             missing_data = np.random.randn(self.N)
             missing_data[missing_data < 0] = np.nan
             self.df[f"missing_{i}"] = missing_data
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/style.py

@@ -83,11 +83,11 @@ def _style_format(self):
     def _style_apply_format_hide(self):
         self.st = self.df.style.applymap(lambda v: "color: red;")
         self.st.format("{:.3f}")
-        self.st.hide_index(self.st.index[1:])
-        self.st.hide_columns(self.st.columns[1:])
+        self.st.hide(self.st.index[1:], axis=0)
+        self.st.hide(self.st.columns[1:], axis=1)
 
     def _style_tooltips(self):
         ttips = DataFrame("abc", index=self.df.index[::2], columns=self.df.columns[::2])
         self.st = self.df.style.set_tooltips(ttips)
-        self.st.hide_index(self.st.index[12:])
-        self.st.hide_columns(self.st.columns[12:])
+        self.st.hide(self.st.index[12:], axis=0)
+        self.st.hide(self.st.columns[12:], axis=1)

asv_bench/benchmarks/join_merge.py

@@ -97,13 +97,13 @@ class ConcatIndexDtype:
 
     params = (
         ["datetime64[ns]", "int64", "Int64", "string[python]", "string[pyarrow]"],
+        ["monotonic", "non_monotonic", "has_na"],
         [0, 1],
         [True, False],
-        [True, False],
     )
-    param_names = ["dtype", "axis", "sort", "is_monotonic"]
+    param_names = ["dtype", "structure", "axis", "sort"]
 
-    def setup(self, dtype, axis, sort, is_monotonic):
+    def setup(self, dtype, structure, axis, sort):
         N = 10_000
         if dtype == "datetime64[ns]":
             vals = date_range("1970-01-01", periods=N)
@@ -115,14 +115,21 @@ def setup(self, dtype, axis, sort, is_monotonic):
             raise NotImplementedError
 
         idx = Index(vals, dtype=dtype)
-        if is_monotonic:
+
+        if structure == "monotonic":
             idx = idx.sort_values()
-        else:
+        elif structure == "non_monotonic":
             idx = idx[::-1]
+        elif structure == "has_na":
+            if not idx._can_hold_na:
+                raise NotImplementedError
+            idx = Index([None], dtype=dtype).append(idx)
+        else:
+            raise NotImplementedError
 
-        self.series = [Series(i, idx[i:]) for i in range(5)]
+        self.series = [Series(i, idx[:-i]) for i in range(1, 6)]
 
-    def time_concat_series(self, dtype, axis, sort, is_monotonic):
+    def time_concat_series(self, dtype, structure, axis, sort):
         concat(self.series, axis=axis, sort=sort)
 
 

asv_bench/benchmarks/reshape.py

@@ -36,7 +36,7 @@ def setup(self):
         self.df = DataFrame(data)
 
     def time_reshape_pivot_time_series(self):
-        self.df.pivot("date", "variable", "value")
+        self.df.pivot(index="date", columns="variable", values="value")
 
 
 class SimpleReshape:

asv_bench/benchmarks/stat_ops.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 
-ops = ["mean", "sum", "median", "std", "skew", "kurt", "mad", "prod", "sem", "var"]
+ops = ["mean", "sum", "median", "std", "skew", "kurt", "prod", "sem", "var"]
 
 
 class FrameOps:
@@ -11,9 +11,6 @@ class FrameOps:
     param_names = ["op", "dtype", "axis"]
 
     def setup(self, op, dtype, axis):
-        if op == "mad" and dtype == "Int64":
-            # GH-33036, GH#33600
-            raise NotImplementedError
         values = np.random.randn(100000, 4)
         if dtype == "Int64":
             values = values.astype(int)

asv_bench/benchmarks/tslibs/offsets.py

@@ -71,11 +71,8 @@ def setup(self, offset):
         self.date = datetime(2011, 1, 1)
         self.dt64 = np.datetime64("2011-01-01 09:00Z")
 
-    def time_apply(self, offset):
-        offset.apply(self.date)
-
-    def time_apply_np_dt64(self, offset):
-        offset.apply(self.dt64)
+    def time_add_np_dt64(self, offset):
+        offset + self.dt64
 
     def time_add(self, offset):
         self.date + offset

ci/code_checks.sh

@@ -47,7 +47,7 @@ import pandas
 
 blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'matplotlib', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
-             'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
+             'tables', 'urllib.request', 'xlrd', 'xlsxwriter'}
 
 # GH#28227 for some of these check for top-level modules, while others are
 #  more specific (e.g. urllib.request)

ci/deps/actions-310.yaml

@@ -51,5 +51,4 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard

ci/deps/actions-38-downstream_compat.yaml

@@ -51,7 +51,6 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard
 
   # downstream packages