From 1160e8981bfe9e43f8db240280af01df25b8e787 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 19 Aug 2024 19:06:47 -0400 Subject: [PATCH 1/4] Revert "remove pyright overrides not needed with pyright 1.1.374 (#967)" This reverts commit 458ecb4bdfaa1b0bcc114f93f0c9d6ef1fa2b5f3. --- pandas-stubs/core/indexes/datetimelike.pyi | 2 +- pandas-stubs/core/series.pyi | 26 +++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pandas-stubs/core/indexes/datetimelike.pyi b/pandas-stubs/core/indexes/datetimelike.pyi index caa10d36d..fb27fae21 100644 --- a/pandas-stubs/core/indexes/datetimelike.pyi +++ b/pandas-stubs/core/indexes/datetimelike.pyi @@ -19,7 +19,7 @@ class DatetimeIndexOpsMixin(ExtensionIndex[S1]): def argmin(self, axis=..., skipna: bool = ..., *args, **kwargs): ... def max(self, axis=..., skipna: bool = ..., *args, **kwargs): ... def argmax(self, axis=..., skipna: bool = ..., *args, **kwargs): ... - def __rsub__( # type: ignore[override] + def __rsub__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, other: DatetimeIndexOpsMixin ) -> TimedeltaIndex: ... diff --git a/pandas-stubs/core/series.pyi b/pandas-stubs/core/series.pyi index edfafe6c1..209237746 100644 --- a/pandas-stubs/core/series.pyi +++ b/pandas-stubs/core/series.pyi @@ -1506,21 +1506,23 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __and__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... + def __and__( # pyright: ignore[reportIncompatibleMethodOverride] + self, other: int | np_ndarray_anyint | Series[int] + ) -> Series[int]: ... # def __array__(self, dtype: Optional[_bool] = ...) -> _np_ndarray def __div__(self, other: num | _ListLike | Series[S1]) -> Series[S1]: ... def __eq__(self, other: object) -> Series[_bool]: ... # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] def __floordiv__(self, other: num | _ListLike | Series[S1]) -> Series[int]: ... - def __ge__( # type: ignore[override] + def __ge__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, other: S1 | _ListLike | Series[S1] | datetime | timedelta | date ) -> Series[_bool]: ... - def __gt__( # type: ignore[override] + def __gt__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, other: S1 | _ListLike | Series[S1] | datetime | timedelta | date ) -> Series[_bool]: ... - def __le__( # type: ignore[override] + def __le__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, other: S1 | _ListLike | Series[S1] | datetime | timedelta | date ) -> Series[_bool]: ... - def __lt__( # type: ignore[override] + def __lt__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] self, other: S1 | _ListLike | Series[S1] | datetime | timedelta | date ) -> Series[_bool]: ... @overload @@ -1538,7 +1540,9 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __or__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... + def __or__( # pyright: ignore[reportIncompatibleMethodOverride] + self, other: int | np_ndarray_anyint | Series[int] + ) -> Series[int]: ... @overload def __radd__(self, other: S1 | Series[S1]) -> Self: ... @overload @@ -1549,7 +1553,7 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __rand__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] + def __rand__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] # pyright: ignore[reportIncompatibleMethodOverride] def __rdiv__(self, other: num | _ListLike | Series[S1]) -> Series[S1]: ... def __rdivmod__(self, other: num | _ListLike | Series[S1]) -> Series[S1]: ... # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] def __rfloordiv__(self, other: num | _ListLike | Series[S1]) -> Series[S1]: ... @@ -1568,7 +1572,7 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __ror__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] + def __ror__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] # pyright: ignore[reportIncompatibleMethodOverride] def __rsub__(self, other: num | _ListLike | Series[S1]) -> Series: ... def __rtruediv__(self, other: num | _ListLike | Series[S1]) -> Series: ... # ignore needed for mypy as we want different results based on the arguments @@ -1577,7 +1581,7 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __rxor__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] + def __rxor__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... # type: ignore[misc] # pyright: ignore[reportIncompatibleMethodOverride] @overload def __sub__( self: Series[Timestamp], @@ -1601,7 +1605,9 @@ class Series(IndexOpsMixin[S1], NDFrame): self, other: bool | list[bool] | list[int] | np_ndarray_bool | Series[bool] ) -> Series[bool]: ... @overload - def __xor__(self, other: int | np_ndarray_anyint | Series[int]) -> Series[int]: ... + def __xor__( # pyright: ignore[reportIncompatibleMethodOverride] + self, other: int | np_ndarray_anyint | Series[int] + ) -> Series[int]: ... def __invert__(self) -> Series[bool]: ... # properties # @property From cf0e1459be200f639d50acc6a25ded47381ed664 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 19 Aug 2024 19:28:07 -0400 Subject: [PATCH 2/4] fix matplotlib tests for Windows 11 --- conftest.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_plotting.py | 5 +++++ 2 files changed, 41 insertions(+) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..9dc46f43b --- /dev/null +++ b/conftest.py @@ -0,0 +1,36 @@ +import gc + +import pytest + + +@pytest.fixture +def mpl_cleanup(): + """ + Ensure Matplotlib is cleaned up around a test. + + Before a test is run: + + 1) Set the backend to "template" to avoid requiring a GUI. + + After a test is run: + + 1) Reset units registry + 2) Reset rc_context + 3) Close all figures + + See matplotlib/testing/decorators.py#L24. + """ + mpl = pytest.importorskip("matplotlib") + mpl_units = pytest.importorskip("matplotlib.units") + plt = pytest.importorskip("matplotlib.pyplot") + orig_units_registry = mpl_units.registry.copy() + try: + with mpl.rc_context(): + mpl.use("template") + yield + finally: + mpl_units.registry.clear() + mpl_units.registry.update(orig_units_registry) + plt.close("all") + # https://matplotlib.org/stable/users/prev_whats_new/whats_new_3.6.0.html#garbage-collection-is-no-longer-run-on-figure-close + gc.collect(1) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index e6ebcec20..5d744350a 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -24,6 +24,11 @@ ) +@pytest.fixture(autouse=True) +def autouse_mpl_cleanup(mpl_cleanup): + pass + + @pytest.fixture def close_figures(): plt.close("all") From 7d97b4f187aaddcf7e3662ee92423038c2aebc15 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Mon, 19 Aug 2024 20:17:52 -0400 Subject: [PATCH 3/4] lock numpy up to 2.0.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d0158c90d..54da448ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ packages = [{ "include" = "pandas-stubs" }] [tool.poetry.dependencies] python = ">=3.9" types-pytz = ">= 2022.1.1" -numpy = ">= 1.23.5" +numpy = ">= 1.23.5, <= 2.0.1" [tool.poetry.group.dev.dependencies] mypy = "1.10.1" From c153773dc9ff87f91e6c29dd3d13860e63e1b261 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Tue, 20 Aug 2024 11:23:47 -0400 Subject: [PATCH 4/4] Support numpy 2.1. Remove python 3.9 support. Update docs --- .github/setup/action.yaml | 10 +++------- .github/workflows/test.yml | 9 ++------- docs/release_procedure.md | 5 ++--- docs/setup.md | 4 ++-- pandas-stubs/core/dtypes/base.pyi | 2 +- pyproject.toml | 4 ++-- tests/test_scalars.py | 16 ++++++++-------- tests/test_series.py | 24 +++++++++++++++++++++--- 8 files changed, 41 insertions(+), 33 deletions(-) diff --git a/.github/setup/action.yaml b/.github/setup/action.yaml index 703efdba6..2162cee70 100644 --- a/.github/setup/action.yaml +++ b/.github/setup/action.yaml @@ -1,17 +1,17 @@ name: Install project dependencies +description: Install project dependencies inputs: python-version: + description: The python version to use required: true os: - required: true - numpy-version: + description: The OS to run on required: true runs: using: composite steps: - - name: Set up Python uses: actions/setup-python@v4 with: @@ -35,7 +35,3 @@ runs: - name: Install project dependencies shell: bash run: poetry install -vvv --no-root - - - name: Set numpy version - shell: bash - run: pip install numpy"${{ inputs. numpy-version }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e09b6df22..b04329fc4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,13 +19,9 @@ jobs: matrix: # Don't use macos-latest because it is arm64 os: [ubuntu-latest, windows-latest, macos-13] - python-version: ["3.9", "3.10", "3.11", "3.12"] - include: - - numpy-version: "<2.0" - - numpy-version: ">= 2.0" - os: ubuntu-latest + python-version: ["3.10", "3.11", "3.12"] - name: OS ${{ matrix.os }} - Python ${{ matrix.python-version }} - Numpy ${{ matrix.numpy-version }} + name: OS ${{ matrix.os }} - Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 @@ -35,7 +31,6 @@ jobs: with: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} - numpy-version: ${{ matrix.numpy-version }} - name: Run mypy on 'tests' (using the local stubs) and on the local stubs run: poetry run poe mypy diff --git a/docs/release_procedure.md b/docs/release_procedure.md index 3ba543955..60aa0fd37 100644 --- a/docs/release_procedure.md +++ b/docs/release_procedure.md @@ -7,7 +7,8 @@ ```shell rm dist/* -poetry publish --build # you will get prompted for your pypi username/password +poetry build +twine upload dist/* # Requires having the pypi API token allowing uploads git commit -a -m "Version a.b.c.yymmdd" git push upstream main git tag va.b.c.yymmdd @@ -15,5 +16,3 @@ git push upstream --tags ``` The conda bots will recognize that a new version has been uploaded to pypi, and generate a pull request sent to the maintainers to approve it. - -Note - Changes will need to be made to use pypi API tokens in the near future. diff --git a/docs/setup.md b/docs/setup.md index 747886368..402a18ecb 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,7 +1,7 @@ ## Set Up Environment -- Make sure you have `python >= 3.9` installed. -- Install poetry: `pip install 'poetry>=1.2'` +- Make sure you have `python >= 3.10` installed. +- Install poetry: `pip install 'poetry>=1.8'` - Install the project dependencies: `poetry update` - Enter the virtual environment: `poetry shell` - Run all tests: `poe test_all` diff --git a/pandas-stubs/core/dtypes/base.pyi b/pandas-stubs/core/dtypes/base.pyi index 4ce9ac70d..41c866f92 100644 --- a/pandas-stubs/core/dtypes/base.pyi +++ b/pandas-stubs/core/dtypes/base.pyi @@ -16,7 +16,7 @@ class ExtensionDtype: @property def kind( self, - ) -> Literal["b", "i", "u", "f", "c", "m", "M", "O", "S", "U", "V"]: ... + ) -> Literal["b", "i", "u", "f", "c", "m", "M", "O", "S", "U", "V", "T"]: ... @property def names(self) -> list[str] | None: ... def empty(self, size: int | tuple[int, ...]) -> type_t[ExtensionArray]: ... diff --git a/pyproject.toml b/pyproject.toml index 54da448ee..ad3d97183 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,9 +28,9 @@ packages = [{ "include" = "pandas-stubs" }] "Documentation" = "https://pandas.pydata.org/pandas-docs/stable" [tool.poetry.dependencies] -python = ">=3.9" +python = ">=3.10" types-pytz = ">= 2022.1.1" -numpy = ">= 1.23.5, <= 2.0.1" +numpy = ">= 2.1" [tool.poetry.group.dev.dependencies] mypy = "1.10.1" diff --git a/tests/test_scalars.py b/tests/test_scalars.py index 70479da79..eb9b17e13 100644 --- a/tests/test_scalars.py +++ b/tests/test_scalars.py @@ -875,8 +875,8 @@ def test_timedelta_cmp() -> None: le = check(assert_type(c_dt_timedelta <= td, bool), bool) assert gt != le - gt_b = check(assert_type(c_timedelta64 > td, Any), bool) - le_b = check(assert_type(c_timedelta64 <= td, Any), bool) + gt_b = check(assert_type(c_timedelta64 > td, np.bool), bool) + le_b = check(assert_type(c_timedelta64 <= td, np.bool), bool) assert gt_b != le_b gt_a = check( @@ -943,8 +943,8 @@ def test_timedelta_cmp() -> None: ge = check(assert_type(c_dt_timedelta >= td, bool), bool) assert lt != ge - lt_b = check(assert_type(c_timedelta64 < td, Any), bool) - ge_b = check(assert_type(c_timedelta64 >= td, Any), bool) + lt_b = check(assert_type(c_timedelta64 < td, np.bool), bool) + ge_b = check(assert_type(c_timedelta64 >= td, np.bool), bool) assert lt_b != ge_b lt_a = check( @@ -1267,8 +1267,8 @@ def test_timestamp_cmp() -> None: check(assert_type(ts > c_series_dt64, "pd.Series[bool]"), pd.Series, np.bool_) check(assert_type(ts <= c_series_dt64, "pd.Series[bool]"), pd.Series, np.bool_) - check(assert_type(c_np_dt64 > ts, Any), bool) - check(assert_type(c_np_dt64 <= ts, Any), bool) + check(assert_type(c_np_dt64 > ts, np.bool), bool) + check(assert_type(c_np_dt64 <= ts, np.bool), bool) gt = check(assert_type(c_dt_datetime > ts, bool), bool) lte = check(assert_type(c_dt_datetime <= ts, bool), bool) @@ -1311,8 +1311,8 @@ def test_timestamp_cmp() -> None: lt = check(assert_type(c_dt_datetime < ts, bool), bool) assert gte != lt - check(assert_type(c_np_dt64 >= ts, Any), bool) - check(assert_type(c_np_dt64 < ts, Any), bool) + check(assert_type(c_np_dt64 >= ts, np.bool), bool) + check(assert_type(c_np_dt64 < ts, np.bool), bool) check(assert_type(c_datetimeindex >= ts, np_ndarray_bool), np.ndarray, np.bool_) check(assert_type(c_datetimeindex < ts, np_ndarray_bool), np.ndarray, np.bool_) diff --git a/tests/test_series.py b/tests/test_series.py index 21f368475..e68028cc4 100644 --- a/tests/test_series.py +++ b/tests/test_series.py @@ -15,6 +15,7 @@ TYPE_CHECKING, Any, Generic, + Literal, TypeVar, Union, cast, @@ -81,6 +82,23 @@ ) from pandas._typing import np_ndarray_int # noqa: F401 +# Tests will use numpy 2.1 in python 3.10 or later +# From Numpy 2.1 __init__.pyi +_DTypeKind: TypeAlias = Literal[ + "b", # boolean + "i", # signed integer + "u", # unsigned integer + "f", # floating-point + "c", # complex floating-point + "m", # timedelta64 + "M", # datetime64 + "O", # python object + "S", # byte-string (fixed-width) + "U", # unicode-string (fixed-width) + "V", # void + "T", # unicode-string (variable-width) +] + def test_types_init() -> None: pd.Series(1) @@ -1675,15 +1693,15 @@ def test_dtype_type() -> None: # GH 216 s1 = pd.Series(["foo"], dtype="string") check(assert_type(s1.dtype, DtypeObj), ExtensionDtype) - check(assert_type(s1.dtype.kind, str), str) + check(assert_type(s1.dtype.kind, _DTypeKind), str) s2 = pd.Series([1], dtype="Int64") check(assert_type(s2.dtype, DtypeObj), ExtensionDtype) - check(assert_type(s2.dtype.kind, str), str) + check(assert_type(s2.dtype.kind, _DTypeKind), str) s3 = pd.Series([1, 2, 3]) check(assert_type(s3.dtype, DtypeObj), np.dtype) - check(assert_type(s3.dtype.kind, str), str) + check(assert_type(s3.dtype.kind, _DTypeKind), str) def test_types_to_numpy() -> None: