Support mypy 0.910 to 0.930 including CI tests (#3594)

* cleanup bumping mypy to 0.930, #3573

* add tests for old mypy

* tweak test-old-mypy job

* alter mypy plugin to work with older versions

* mypy.py compatibility with multiple versions

* fix mypy tests to allow for varied output

* toml parsing, fix #3579

* formatting :-(

* ignore missing types for toml package

* remove unused ignore_missing_imports

* undo removal of ignore_missing_imports for dotenv

* tweak coverage ignore

* fully uninstall mypy and toml/tomli

Author: samuelcolvin

.github/workflows/ci.yml

@@ -138,7 +138,7 @@ jobs:
       COMPILED: no
       DEPS: yes
 
-    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    runs-on: ${{ matrix.os }}-latest
     steps:
     - uses: actions/checkout@v2
 
@@ -164,8 +164,49 @@ jobs:
         name: coverage
         path: coverage
 
+  test-old-mypy:
+    name: test mypy v${{ matrix.mypy-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        mypy-version: ['0.910', '0.920', '0.921']
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: set up python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+
+    - name: install
+      run: |
+        make install-testing
+        pip freeze
+
+    - name: uninstall deps
+      run: pip uninstall -y mypy tomli toml
+
+    - name: install specific mypy version
+      run: pip install mypy==${{ matrix.mypy-version }}
+
+    - run: mkdir coverage
+
+    - name: run tests
+      run: pytest --cov=pydantic tests/mypy
+      env:
+        COVERAGE_FILE: coverage/.coverage.linux-py3.10-mypy${{ matrix.mypy-version }}
+        CONTEXT: linux-py3.10-mypy${{ matrix.mypy-version }}
+
+    - name: store coverage files
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: coverage
+
   coverage-combine:
-    needs: [test-linux, test-windows-mac]
+    needs: [test-linux, test-windows-mac, test-old-mypy]
     runs-on: ubuntu-latest
 
     steps:
@@ -236,7 +277,7 @@ jobs:
 
   build:
     name: build py3.${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }}
-    needs: [lint, test-linux, test-windows-mac, test-fastapi, benchmark]
+    needs: [lint, test-linux, test-windows-mac, test-old-mypy, test-fastapi, benchmark]
     if: "success() && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master')"
     strategy:
       fail-fast: false
@@ -249,7 +290,7 @@ jobs:
         - os: windows
           ls: dir
 
-    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    runs-on: ${{ matrix.os }}-latest
     steps:
     - uses: actions/checkout@v2
 

docs/mypy_plugin.md

@@ -86,6 +86,8 @@ To get started, all you need to do is create a `mypy.ini` file with following co
 plugins = pydantic.mypy
 ```
 
+The plugin is compatible with mypy versions 0.910, 0.920, 0.921 and 0.930.
+
 See the [mypy usage](usage/mypy.md) and [plugin configuration](#configuring-the-plugin) docs for more details.
 
 ### Plugin Settings

pydantic/class_validators.py

@@ -9,6 +9,9 @@
 from .typing import AnyCallable
 from .utils import ROOT_KEY, in_ipython
 
+if TYPE_CHECKING:
+    from .typing import AnyClassMethod
+
 
 class Validator:
     __slots__ = 'func', 'pre', 'each_item', 'always', 'check_fields', 'skip_on_failure'
@@ -54,7 +57,7 @@ def validator(
     check_fields: bool = True,
     whole: bool = None,
     allow_reuse: bool = False,
-) -> Callable[[AnyCallable], classmethod]:  # type: ignore[type-arg]
+) -> Callable[[AnyCallable], 'AnyClassMethod']:
     """
     Decorate methods on the class indicating that they should be used to validate fields
     :param fields: which field(s) the method should be called on
@@ -81,7 +84,7 @@ def validator(
         assert each_item is False, '"each_item" and "whole" conflict, remove "whole"'
         each_item = not whole
 
-    def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+    def dec(f: AnyCallable) -> 'AnyClassMethod':
         f_cls = _prepare_validator(f, allow_reuse)
         setattr(
             f_cls,
@@ -97,20 +100,20 @@ def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
 
 
 @overload
-def root_validator(_func: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+def root_validator(_func: AnyCallable) -> 'AnyClassMethod':
     ...
 
 
 @overload
 def root_validator(
     *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False
-) -> Callable[[AnyCallable], classmethod]:  # type: ignore[type-arg]
+) -> Callable[[AnyCallable], 'AnyClassMethod']:
     ...
 
 
 def root_validator(
     _func: Optional[AnyCallable] = None, *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False
-) -> Union[classmethod, Callable[[AnyCallable], classmethod]]:  # type: ignore[type-arg]
+) -> Union['AnyClassMethod', Callable[[AnyCallable], 'AnyClassMethod']]:
     """
     Decorate methods on a model indicating that they should be used to validate (and perhaps modify) data either
     before or after standard model parsing/validation is performed.
@@ -122,7 +125,7 @@ def root_validator(
         )
         return f_cls
 
-    def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
+    def dec(f: AnyCallable) -> 'AnyClassMethod':
         f_cls = _prepare_validator(f, allow_reuse)
         setattr(
             f_cls, ROOT_VALIDATOR_CONFIG_KEY, Validator(func=f_cls.__func__, pre=pre, skip_on_failure=skip_on_failure)
@@ -132,7 +135,7 @@ def dec(f: AnyCallable) -> classmethod:  # type: ignore[type-arg]
     return dec
 
 
-def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> classmethod:  # type: ignore[type-arg]
+def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> 'AnyClassMethod':
     """
     Avoid validators with duplicated names since without this, validators can be overwritten silently
     which generally isn't the intended behaviour, don't run in ipython (see #312) or if allow_reuse is False.
@@ -325,7 +328,7 @@ def _generic_validator_basic(validator: AnyCallable, sig: 'Signature', args: Set
         return lambda cls, v, values, field, config: validator(v, values=values, field=field, config=config)
 
 
-def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, classmethod]:  # type: ignore[type-arg]
+def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, 'AnyClassMethod']:
     all_attributes = ChainMap(*[cls.__dict__ for cls in type_.__mro__])
     return {
         k: v

pydantic/main.py

@@ -66,6 +66,7 @@
     from .types import ModelOrDc
     from .typing import (
         AbstractSetIntStr,
+        AnyClassMethod,
         CallableGenerator,
         DictAny,
         DictStrAny,
@@ -890,7 +891,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: None = None,
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['BaseModel']:
     ...
@@ -903,7 +904,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: Union[Type['Model'], Tuple[Type['Model'], ...]],
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['Model']:
     ...
@@ -915,7 +916,7 @@ def create_model(
     __config__: Optional[Type[BaseConfig]] = None,
     __base__: Union[None, Type['Model'], Tuple[Type['Model'], ...]] = None,
     __module__: str = __name__,
-    __validators__: Dict[str, classmethod] = None,  # type: ignore[type-arg]
+    __validators__: Dict[str, 'AnyClassMethod'] = None,
     **field_definitions: Any,
 ) -> Type['Model']:
     """

pydantic/mypy.py

@@ -1,20 +1,6 @@
 from configparser import ConfigParser
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union
 
-from pydantic.utils import is_valid_field
-
-try:
-    import toml
-except ImportError:  # pragma: no cover
-    # future-proofing for upcoming `mypy` releases which will switch dependencies
-    try:
-        import tomli as toml  # type: ignore
-    except ImportError:
-        import warnings
-
-        warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
-        toml = None  # type: ignore
-
 from mypy.errorcodes import ErrorCode
 from mypy.nodes import (
     ARG_NAMED,
@@ -60,20 +46,29 @@
     Type,
     TypeOfAny,
     TypeType,
-    TypeVarLikeType,
     TypeVarType,
     UnionType,
     get_proper_type,
 )
 from mypy.typevars import fill_typevars
 from mypy.util import get_unique_redefinition_name
+from mypy.version import __version__ as mypy_version
+
+from pydantic.utils import is_valid_field
+
+try:
+    from mypy.types import TypeVarDef  # type: ignore[attr-defined]
+except ImportError:  # pragma: no cover
+    # Backward-compatible with TypeVarDef from Mypy 0.910.
+    from mypy.types import TypeVarType as TypeVarDef
 
 CONFIGFILE_KEY = 'pydantic-mypy'
 METADATA_KEY = 'pydantic-mypy-metadata'
 BASEMODEL_FULLNAME = 'pydantic.main.BaseModel'
 BASESETTINGS_FULLNAME = 'pydantic.env_settings.BaseSettings'
 FIELD_FULLNAME = 'pydantic.fields.Field'
 DATACLASS_FULLNAME = 'pydantic.dataclasses.dataclass'
+BUILTINS_NAME = 'builtins' if float(mypy_version) >= 0.930 else '__builtins__'
 
 
 def plugin(version: str) -> 'TypingType[Plugin]':
@@ -125,9 +120,9 @@ def __init__(self, options: Options) -> None:
         if options.config_file is None:  # pragma: no cover
             return
 
-        if toml and options.config_file.endswith('.toml'):
-            with open(options.config_file, 'r') as rf:
-                config = toml.load(rf).get('tool', {}).get('pydantic-mypy', {})
+        toml_config = parse_toml(options.config_file)
+        if toml_config is not None:
+            config = toml_config.get('tool', {}).get('pydantic-mypy', {})
             for key in self.__slots__:
                 setting = config.get(key, False)
                 if not isinstance(setting, bool):
@@ -348,26 +343,32 @@ def add_construct_method(self, fields: List['PydanticModelField']) -> None:
         and does not treat settings fields as optional.
         """
         ctx = self._ctx
-        set_str = ctx.api.named_type('builtins.set', [ctx.api.named_type('builtins.str')])
+        set_str = ctx.api.named_type(f'{BUILTINS_NAME}.set', [ctx.api.named_type(f'{BUILTINS_NAME}.str')])
         optional_set_str = UnionType([set_str, NoneType()])
         fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT)
         construct_arguments = self.get_field_arguments(fields, typed=True, force_all_optional=False, use_alias=False)
         construct_arguments = [fields_set_argument] + construct_arguments
 
-        obj_type = ctx.api.named_type('builtins.object')
+        obj_type = ctx.api.named_type(f'{BUILTINS_NAME}.object')
         self_tvar_name = '_PydanticBaseModel'  # Make sure it does not conflict with other names in the class
         tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name
-        self_type = TypeVarType(self_tvar_name, tvar_fullname, -1, [], obj_type)
+        tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type)
         self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type)
         ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr)
 
+        # Backward-compatible with TypeVarDef from Mypy 0.910.
+        if isinstance(tvd, TypeVarType):
+            self_type = tvd
+        else:
+            self_type = TypeVarType(tvd)  # type: ignore[call-arg]
+
         add_method(
             ctx,
             'construct',
             construct_arguments,
             return_type=self_type,
             self_type=self_type,
-            tvar_like_type=self_type,
+            tvar_def=tvd,
             is_classmethod=True,
         )
 
@@ -619,7 +620,7 @@ def add_method(
     args: List[Argument],
     return_type: Type,
     self_type: Optional[Type] = None,
-    tvar_like_type: Optional[TypeVarLikeType] = None,
+    tvar_def: Optional[TypeVarDef] = None,
     is_classmethod: bool = False,
     is_new: bool = False,
     # is_staticmethod: bool = False,
@@ -654,10 +655,10 @@ def add_method(
         arg_names.append(get_name(arg.variable))
         arg_kinds.append(arg.kind)
 
-    function_type = ctx.api.named_type('builtins.function')
+    function_type = ctx.api.named_type(f'{BUILTINS_NAME}.function')
     signature = CallableType(arg_types, arg_kinds, arg_names, return_type, function_type)
-    if tvar_like_type:
-        signature.variables = [tvar_like_type]
+    if tvar_def:
+        signature.variables = [tvar_def]
 
     func = FuncDef(name, args, Block([PassStmt()]))
     func.info = info
@@ -714,3 +715,25 @@ def get_name(x: Union[FuncBase, SymbolNode]) -> str:
     if callable(fn):  # pragma: no cover
         return fn()
     return fn
+
+
+def parse_toml(config_file: str) -> Optional[Dict[str, Any]]:
+    if not config_file.endswith('.toml'):
+        return None
+
+    read_mode = 'rb'
+    try:
+        import tomli as toml_
+    except ImportError:
+        # older versions of mypy have toml as a dependency, not tomli
+        read_mode = 'r'
+        try:
+            import toml as toml_  # type: ignore[no-redef]
+        except ImportError:  # pragma: no cover
+            import warnings
+
+            warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.')
+            return None
+
+    with open(config_file, read_mode) as rf:
+        return toml_.load(rf)  # type: ignore[arg-type]

pydantic/typing.py

@@ -228,6 +228,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool:
     MappingIntStrAny = Mapping[IntStr, Any]
     CallableGenerator = Generator[AnyCallable, None, None]
     ReprArgs = Sequence[Tuple[Optional[str], Any]]
+    AnyClassMethod = classmethod[Any]
 
 __all__ = (
     'ForwardRef',
@@ -258,6 +259,7 @@ def is_union(tp: Optional[Type[Any]]) -> bool:
     'DictIntStrAny',
     'CallableGenerator',
     'ReprArgs',
+    'AnyClassMethod',
     'CallableGenerator',
     'WithArgsTypes',
     'get_args',

pydantic/utils.py

@@ -574,7 +574,8 @@ def _coerce_items(items: Union['AbstractSetIntStr', 'MappingIntStrAny']) -> 'Map
         elif isinstance(items, AbstractSet):
             items = dict.fromkeys(items, ...)
         else:
-            raise TypeError(f'Unexpected type of exclude value {items.__class__}')  # type: ignore[attr-defined]
+            class_name = getattr(items, '__class__', '???')
+            raise TypeError(f'Unexpected type of exclude value {class_name}')
         return items
 
     @classmethod

setup.cfg

@@ -76,3 +76,6 @@ ignore_missing_imports = true
 
 [mypy-dotenv]
 ignore_missing_imports = true
+
+[mypy-toml]
+ignore_missing_imports = true