Merge branch 'main' into implementation-pdep-4

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -102,7 +102,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.264
+        - pyright@1.1.276
     -   id: pyright_reportGeneralTypeIssues
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

asv_bench/asv.conf.json

@@ -54,7 +54,6 @@
         "openpyxl": [],
         "xlsxwriter": [],
         "xlrd": [],
-        "xlwt": [],
         "odfpy": [],
         "jinja2": [],
     },

asv_bench/benchmarks/groupby.py

@@ -14,6 +14,7 @@
     Timestamp,
     date_range,
     period_range,
+    to_timedelta,
 )
 
 from .pandas_vb_common import tm
@@ -35,7 +36,6 @@
         "pct_change",
         "min",
         "var",
-        "mad",
         "describe",
         "std",
         "quantile",
@@ -52,7 +52,6 @@
         "cummax",
         "pct_change",
         "var",
-        "mad",
         "describe",
         "std",
     },
@@ -437,7 +436,6 @@ class GroupByMethods:
             "first",
             "head",
             "last",
-            "mad",
             "max",
             "min",
             "median",
@@ -483,7 +481,7 @@ def setup(self, dtype, method, application, ncols):
 
         if method == "describe":
             ngroups = 20
-        elif method in ["mad", "skew"]:
+        elif method == "skew":
             ngroups = 100
         else:
             ngroups = 1000
@@ -990,4 +988,31 @@ def time_sample_weights(self):
         self.df.groupby(self.groups).sample(n=1, weights=self.weights)
 
 
+class Resample:
+    # GH 28635
+    def setup(self):
+        num_timedeltas = 20_000
+        num_groups = 3
+
+        index = MultiIndex.from_product(
+            [
+                np.arange(num_groups),
+                to_timedelta(np.arange(num_timedeltas), unit="s"),
+            ],
+            names=["groups", "timedeltas"],
+        )
+        data = np.random.randint(0, 1000, size=(len(index)))
+
+        self.df = DataFrame(data, index=index).reset_index("timedeltas")
+        self.df_multiindex = DataFrame(data, index=index)
+
+    def time_resample(self):
+        self.df.groupby(level="groups").resample("10s", on="timedeltas").mean()
+
+    def time_resample_multiindex(self):
+        self.df_multiindex.groupby(level="groups").resample(
+            "10s", level="timedeltas"
+        ).mean()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/excel.py

@@ -33,7 +33,7 @@ def _generate_dataframe():
 
 class WriteExcel:
 
-    params = ["openpyxl", "xlsxwriter", "xlwt"]
+    params = ["openpyxl", "xlsxwriter"]
     param_names = ["engine"]
 
     def setup(self, engine):
@@ -68,10 +68,9 @@ def time_write_excel_style(self, engine):
 
 class ReadExcel:
 
-    params = ["xlrd", "openpyxl", "odf"]
+    params = ["openpyxl", "odf"]
     param_names = ["engine"]
     fname_excel = "spreadsheet.xlsx"
-    fname_excel_xls = "spreadsheet.xls"
     fname_odf = "spreadsheet.ods"
 
     def _create_odf(self):
@@ -92,13 +91,10 @@ def setup_cache(self):
         self.df = _generate_dataframe()
 
         self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
-        self.df.to_excel(self.fname_excel_xls, sheet_name="Sheet1")
         self._create_odf()
 
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel
@@ -107,9 +103,7 @@ def time_read_excel(self, engine):
 
 class ReadExcelNRows(ReadExcel):
     def time_read_excel(self, engine):
-        if engine == "xlrd":
-            fname = self.fname_excel_xls
-        elif engine == "odf":
+        if engine == "odf":
             fname = self.fname_odf
         else:
             fname = self.fname_excel

asv_bench/benchmarks/io/stata.py

@@ -38,13 +38,13 @@ def setup(self, convert_dates):
         )
         self.df["float32_"] = np.array(np.random.randn(N), dtype=np.float32)
         self.convert_dates = {"index": convert_dates}
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
     def time_read_stata(self, convert_dates):
         read_stata(self.fname)
 
     def time_write_stata(self, convert_dates):
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 class StataMissing(Stata):
@@ -54,7 +54,7 @@ def setup(self, convert_dates):
             missing_data = np.random.randn(self.N)
             missing_data[missing_data < 0] = np.nan
             self.df[f"missing_{i}"] = missing_data
-        self.df.to_stata(self.fname, self.convert_dates)
+        self.df.to_stata(self.fname, convert_dates=self.convert_dates)
 
 
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/join_merge.py

@@ -97,13 +97,13 @@ class ConcatIndexDtype:
 
     params = (
         ["datetime64[ns]", "int64", "Int64", "string[python]", "string[pyarrow]"],
+        ["monotonic", "non_monotonic", "has_na"],
         [0, 1],
         [True, False],
-        [True, False],
     )
-    param_names = ["dtype", "axis", "sort", "is_monotonic"]
+    param_names = ["dtype", "structure", "axis", "sort"]
 
-    def setup(self, dtype, axis, sort, is_monotonic):
+    def setup(self, dtype, structure, axis, sort):
         N = 10_000
         if dtype == "datetime64[ns]":
             vals = date_range("1970-01-01", periods=N)
@@ -115,14 +115,21 @@ def setup(self, dtype, axis, sort, is_monotonic):
             raise NotImplementedError
 
         idx = Index(vals, dtype=dtype)
-        if is_monotonic:
+
+        if structure == "monotonic":
             idx = idx.sort_values()
-        else:
+        elif structure == "non_monotonic":
             idx = idx[::-1]
+        elif structure == "has_na":
+            if not idx._can_hold_na:
+                raise NotImplementedError
+            idx = Index([None], dtype=dtype).append(idx)
+        else:
+            raise NotImplementedError
 
-        self.series = [Series(i, idx[i:]) for i in range(5)]
+        self.series = [Series(i, idx[:-i]) for i in range(1, 6)]
 
-    def time_concat_series(self, dtype, axis, sort, is_monotonic):
+    def time_concat_series(self, dtype, structure, axis, sort):
         concat(self.series, axis=axis, sort=sort)
 
 

asv_bench/benchmarks/stat_ops.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 
-ops = ["mean", "sum", "median", "std", "skew", "kurt", "mad", "prod", "sem", "var"]
+ops = ["mean", "sum", "median", "std", "skew", "kurt", "prod", "sem", "var"]
 
 
 class FrameOps:
@@ -11,9 +11,6 @@ class FrameOps:
     param_names = ["op", "dtype", "axis"]
 
     def setup(self, op, dtype, axis):
-        if op == "mad" and dtype == "Int64":
-            # GH-33036, GH#33600
-            raise NotImplementedError
         values = np.random.randn(100000, 4)
         if dtype == "Int64":
             values = values.astype(int)

asv_bench/benchmarks/tslibs/offsets.py

@@ -71,11 +71,8 @@ def setup(self, offset):
         self.date = datetime(2011, 1, 1)
         self.dt64 = np.datetime64("2011-01-01 09:00Z")
 
-    def time_apply(self, offset):
-        offset.apply(self.date)
-
-    def time_apply_np_dt64(self, offset):
-        offset.apply(self.dt64)
+    def time_add_np_dt64(self, offset):
+        offset + self.dt64
 
     def time_add(self, offset):
         self.date + offset

ci/code_checks.sh

@@ -47,7 +47,7 @@ import pandas
 
 blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'matplotlib', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
-             'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
+             'tables', 'urllib.request', 'xlrd', 'xlsxwriter'}
 
 # GH#28227 for some of these check for top-level modules, while others are
 #  more specific (e.g. urllib.request)

ci/deps/actions-310.yaml

@@ -51,5 +51,4 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard

ci/deps/actions-38-downstream_compat.yaml

@@ -51,7 +51,6 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard
 
   # downstream packages

ci/deps/actions-38-minimum_versions.yaml

@@ -53,5 +53,4 @@ dependencies:
   - xarray=0.19.0
   - xlrd=2.0.1
   - xlsxwriter=1.4.3
-  - xlwt=1.3.0
   - zstandard=0.15.2

ci/deps/actions-38.yaml

@@ -50,5 +50,4 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard

ci/deps/actions-39.yaml

@@ -51,5 +51,4 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard

ci/deps/circle-38-arm64.yaml

@@ -51,5 +51,4 @@ dependencies:
   - xarray
   - xlrd
   - xlsxwriter
-  - xlwt
   - zstandard