TYP: Enable mypy stricter typing for defs/calls (#54271)

* TYP: Enable disallowing untyped defs

* finish rest of core

* add typings

* Install google auth for typing

* type another credentials, list[str]

Author: mroeschke

environment.yml

@@ -83,6 +83,7 @@ dependencies:
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb
+  - google-auth
   - natsort  # DataFrame.sort_values doctest
   - numpydoc
   - pydata-sphinx-theme

pandas/compat/__init__.py

@@ -36,7 +36,7 @@
     from pandas._typing import F
 
 
-def set_function_name(f: F, name: str, cls) -> F:
+def set_function_name(f: F, name: str, cls: type) -> F:
     """
     Bind the name/qualname attributes of the function.
     """

pandas/core/array_algos/transforms.py

@@ -9,10 +9,15 @@
 import numpy as np
 
 if TYPE_CHECKING:
-    from pandas._typing import AxisInt
+    from pandas._typing import (
+        AxisInt,
+        Scalar,
+    )
 
 
-def shift(values: np.ndarray, periods: int, axis: AxisInt, fill_value) -> np.ndarray:
+def shift(
+    values: np.ndarray, periods: int, axis: AxisInt, fill_value: Scalar
+) -> np.ndarray:
     new_values = values
 
     if periods == 0 or values.size == 0:

pandas/core/dtypes/base.py

@@ -398,7 +398,7 @@ class StorageExtensionDtype(ExtensionDtype):
     name: str
     _metadata = ("storage",)
 
-    def __init__(self, storage=None) -> None:
+    def __init__(self, storage: str | None = None) -> None:
         self.storage = storage
 
     def __repr__(self) -> str:

pandas/core/interchange/buffer.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Any
+
 import numpy as np
 
 from pandas.core.interchange.dataframe_protocol import (
@@ -49,7 +51,7 @@ def ptr(self) -> int:
         """
         return self._x.__array_interface__["data"][0]
 
-    def __dlpack__(self):
+    def __dlpack__(self) -> Any:
         """
         Represent this structure as DLPack interface.
         """

pandas/core/interchange/dataframe.py

@@ -7,6 +7,11 @@
 from pandas.core.interchange.dataframe_protocol import DataFrame as DataFrameXchg
 
 if TYPE_CHECKING:
+    from collections.abc import (
+        Iterable,
+        Sequence,
+    )
+
     from pandas import (
         DataFrame,
         Index,
@@ -72,7 +77,7 @@ def get_columns(self) -> list[PandasColumn]:
             for name in self._df.columns
         ]
 
-    def select_columns(self, indices) -> PandasDataFrameXchg:
+    def select_columns(self, indices: Sequence[int]) -> PandasDataFrameXchg:
         if not isinstance(indices, abc.Sequence):
             raise ValueError("`indices` is not a sequence")
         if not isinstance(indices, list):
@@ -82,7 +87,7 @@ def select_columns(self, indices) -> PandasDataFrameXchg:
             self._df.iloc[:, indices], self._nan_as_null, self._allow_copy
         )
 
-    def select_columns_by_name(self, names) -> PandasDataFrameXchg:
+    def select_columns_by_name(self, names: list[str]) -> PandasDataFrameXchg:  # type: ignore[override]  # noqa: E501
         if not isinstance(names, abc.Sequence):
             raise ValueError("`names` is not a sequence")
         if not isinstance(names, list):
@@ -92,7 +97,7 @@ def select_columns_by_name(self, names) -> PandasDataFrameXchg:
             self._df.loc[:, names], self._nan_as_null, self._allow_copy
         )
 
-    def get_chunks(self, n_chunks: int | None = None):
+    def get_chunks(self, n_chunks: int | None = None) -> Iterable[PandasDataFrameXchg]:
         """
         Return an iterator yielding the chunks.
         """

pandas/io/feather_format.py

@@ -1,7 +1,10 @@
 """ feather-format compat """
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Any,
+)
 
 from pandas._libs import lib
 from pandas.compat._optional import import_optional_dependency
@@ -34,7 +37,7 @@ def to_feather(
     df: DataFrame,
     path: FilePath | WriteBuffer[bytes],
     storage_options: StorageOptions | None = None,
-    **kwargs,
+    **kwargs: Any,
 ) -> None:
     """
     Write a DataFrame to the binary Feather format.
@@ -70,7 +73,7 @@ def read_feather(
     use_threads: bool = True,
     storage_options: StorageOptions | None = None,
     dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
-):
+) -> DataFrame:
     """
     Load a feather-format object from the file path.
 

pandas/io/gbq.py

@@ -9,6 +9,8 @@
 from pandas.compat._optional import import_optional_dependency
 
 if TYPE_CHECKING:
+    import google.auth
+
     from pandas import DataFrame
 
 
@@ -33,7 +35,7 @@ def read_gbq(
     dialect: str | None = None,
     location: str | None = None,
     configuration: dict[str, Any] | None = None,
-    credentials=None,
+    credentials: google.auth.credentials.Credentials | None = None,
     use_bqstorage_api: bool | None = None,
     max_results: int | None = None,
     progress_bar_type: str | None = None,
@@ -215,7 +217,7 @@ def to_gbq(
     table_schema: list[dict[str, str]] | None = None,
     location: str | None = None,
     progress_bar: bool = True,
-    credentials=None,
+    credentials: google.auth.credentials.Credentials | None = None,
 ) -> None:
     pandas_gbq = _try_import()
     pandas_gbq.to_gbq(

pandas/io/orc.py

@@ -30,6 +30,9 @@
 )
 
 if TYPE_CHECKING:
+    import fsspec
+    import pyarrow.fs
+
     from pandas._typing import (
         DtypeBackend,
         FilePath,
@@ -44,8 +47,8 @@ def read_orc(
     path: FilePath | ReadBuffer[bytes],
     columns: list[str] | None = None,
     dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
-    filesystem=None,
-    **kwargs,
+    filesystem: pyarrow.fs.FileSystem | fsspec.spec.AbstractFileSystem | None = None,
+    **kwargs: Any,
 ) -> DataFrame:
     """
     Load an ORC object from the file path, returning a DataFrame.

pandas/io/pickle.py

@@ -24,6 +24,11 @@
         WriteBuffer,
     )
 
+    from pandas import (
+        DataFrame,
+        Series,
+    )
+
 
 @doc(
     storage_options=_shared_docs["storage_options"],
@@ -116,7 +121,7 @@ def read_pickle(
     filepath_or_buffer: FilePath | ReadPickleBuffer,
     compression: CompressionOptions = "infer",
     storage_options: StorageOptions | None = None,
-):
+) -> DataFrame | Series:
     """
     Load pickled pandas object (or any object) from file.
 

pyproject.toml

@@ -538,9 +538,9 @@ disallow_any_explicit = false # TODO
 disallow_any_generics = false # TODO
 disallow_subclassing_any = false # TODO
 # Untyped definitions and calls
-disallow_untyped_calls = false # TODO
-disallow_untyped_defs = false # TODO
-disallow_incomplete_defs = false # TODO
+disallow_untyped_calls = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
 check_untyped_defs = true
 disallow_untyped_decorators = true
 # None and Optional handling
@@ -566,6 +566,129 @@ show_error_context = false
 show_column_numbers = false
 show_error_codes = true
 
+[[tool.mypy.overrides]]
+module = [
+  "pandas._config.config", # TODO
+  "pandas._libs.*",
+  "pandas._testing.*", # TODO
+  "pandas.arrays", # TODO
+  "pandas.compat.numpy.function", # TODO
+  "pandas.compat._optional", # TODO
+  "pandas.compat.compressors", # TODO
+  "pandas.compat.pickle_compat", # TODO
+  "pandas.core._numba.executor", # TODO
+  "pandas.core.array_algos.datetimelike_accumulations", # TODO
+  "pandas.core.array_algos.masked_accumulations", # TODO
+  "pandas.core.array_algos.masked_reductions", # TODO
+  "pandas.core.array_algos.putmask", # TODO
+  "pandas.core.array_algos.quantile", # TODO
+  "pandas.core.array_algos.replace", # TODO
+  "pandas.core.array_algos.take", # TODO
+  "pandas.core.arrays.*", # TODO
+  "pandas.core.computation.*", # TODO
+  "pandas.core.dtypes.astype", # TODO
+  "pandas.core.dtypes.cast", # TODO
+  "pandas.core.dtypes.common", # TODO
+  "pandas.core.dtypes.concat", # TODO
+  "pandas.core.dtypes.dtypes", # TODO
+  "pandas.core.dtypes.generic", # TODO
+  "pandas.core.dtypes.inference", # TODO
+  "pandas.core.dtypes.missing", # TODO
+  "pandas.core.groupby.categorical", # TODO
+  "pandas.core.groupby.generic", # TODO
+  "pandas.core.groupby.grouper", # TODO
+  "pandas.core.groupby.groupby", # TODO
+  "pandas.core.groupby.ops", # TODO
+  "pandas.core.indexers.*", # TODO
+  "pandas.core.indexes.*", # TODO
+  "pandas.core.interchange.column", # TODO
+  "pandas.core.interchange.dataframe_protocol", # TODO
+  "pandas.core.interchange.from_dataframe", # TODO
+  "pandas.core.internals.*", # TODO
+  "pandas.core.methods.*", # TODO
+  "pandas.core.ops.array_ops", # TODO
+  "pandas.core.ops.common", # TODO
+  "pandas.core.ops.invalid", # TODO
+  "pandas.core.ops.mask_ops", # TODO
+  "pandas.core.ops.missing", # TODO
+  "pandas.core.reshape.*", # TODO
+  "pandas.core.strings.*", # TODO
+  "pandas.core.tools.*", # TODO
+  "pandas.core.window.common", # TODO
+  "pandas.core.window.ewm", # TODO
+  "pandas.core.window.expanding", # TODO
+  "pandas.core.window.numba_", # TODO
+  "pandas.core.window.online", # TODO
+  "pandas.core.window.rolling", # TODO
+  "pandas.core.accessor", # TODO
+  "pandas.core.algorithms", # TODO
+  "pandas.core.apply", # TODO
+  "pandas.core.arraylike", # TODO
+  "pandas.core.base", # TODO
+  "pandas.core.common", # TODO
+  "pandas.core.config_init", # TODO
+  "pandas.core.construction", # TODO
+  "pandas.core.flags", # TODO
+  "pandas.core.frame", # TODO
+  "pandas.core.generic", # TODO
+  "pandas.core.indexing", # TODO
+  "pandas.core.missing", # TODO
+  "pandas.core.nanops", # TODO
+  "pandas.core.resample", # TODO
+  "pandas.core.roperator", # TODO
+  "pandas.core.sample", # TODO
+  "pandas.core.series", # TODO
+  "pandas.core.sorting", # TODO
+  "pandas.errors", # TODO
+  "pandas.io.clipboard", # TODO
+  "pandas.io.excel._base", # TODO
+  "pandas.io.excel._odfreader", # TODO
+  "pandas.io.excel._odswriter", # TODO
+  "pandas.io.excel._openpyxl", # TODO
+  "pandas.io.excel._pyxlsb", # TODO
+  "pandas.io.excel._xlrd", # TODO
+  "pandas.io.excel._xlsxwriter", # TODO
+  "pandas.io.formats.console", # TODO
+  "pandas.io.formats.css", # TODO
+  "pandas.io.formats.excel", # TODO
+  "pandas.io.formats.format", # TODO
+  "pandas.io.formats.info", # TODO
+  "pandas.io.formats.printing", # TODO
+  "pandas.io.formats.style", # TODO
+  "pandas.io.formats.style_render", # TODO
+  "pandas.io.formats.xml", # TODO
+  "pandas.io.json.*", # TODO
+  "pandas.io.parsers.*", # TODO
+  "pandas.io.sas.sas_xport", # TODO
+  "pandas.io.sas.sas7bdat", # TODO
+  "pandas.io.clipboards", # TODO
+  "pandas.io.common", # TODO
+  "pandas.io.gbq", # TODO
+  "pandas.io.html", # TODO
+  "pandas.io.gbq", # TODO
+  "pandas.io.parquet", # TODO
+  "pandas.io.pytables", # TODO
+  "pandas.io.sql", # TODO
+  "pandas.io.stata", # TODO
+  "pandas.io.xml", # TODO
+  "pandas.plotting.*", # TODO
+  "pandas.tests.*",
+  "pandas.tseries.frequencies", # TODO
+  "pandas.tseries.holiday", # TODO
+  "pandas.util._decorators", # TODO
+  "pandas.util._doctools", # TODO
+  "pandas.util._print_versions", # TODO
+  "pandas.util._test_decorators", # TODO
+  "pandas.util._validators", # TODO
+  "pandas.util", # TODO
+  "pandas._version",
+  "pandas.conftest",
+  "pandas"
+]
+disallow_untyped_calls = false
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+
 [[tool.mypy.overrides]]
 module = [
   "pandas.tests.*",

requirements-dev.txt

@@ -58,6 +58,7 @@ tokenize-rt
 pre-commit>=2.15.0
 gitpython
 gitdb
+google-auth
 natsort
 numpydoc
 pydata-sphinx-theme