Sync mypy with recent runtime updates in typing (#7013)

This introduces the following updates:
* Allow using `typing.TypedDict` (still support all the old forms)
* Add a test for `typing.Literal` (it was already supported, but there were no tests)
* Rename `@runtime` to `@runtime_checkable`, while keeping the alias in `typing_extensions` for backwards compatibility.

(Note that `typing.Final` and `typing.Protocol` were already supported and there are tests.)
See also python/typeshed#3070

Author: ilevkivskyi

mypy/message_registry.py

@@ -141,5 +141,5 @@
 
 # Protocol
 RUNTIME_PROTOCOL_EXPECTED = \
-    'Only @runtime protocols can be used with instance and class checks'  # type: Final
+    'Only @runtime_checkable protocols can be used with instance and class checks'  # type: Final
 CANNOT_INSTANTIATE_PROTOCOL = 'Cannot instantiate protocol class "{}"'  # type: Final

mypy/newsemanal/semanal.py

@@ -73,7 +73,7 @@
     PlaceholderNode, COVARIANT, CONTRAVARIANT, INVARIANT,
     nongen_builtins, get_member_expr_fullname, REVEAL_TYPE,
     REVEAL_LOCALS, is_final_node, TypedDictExpr, type_aliases_target_versions,
-    EnumCallExpr
+    EnumCallExpr, RUNTIME_PROTOCOL_DECOS
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.typevars import fill_typevars
@@ -1137,11 +1137,12 @@ def leave_class(self) -> None:
     def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None:
         decorator.accept(self)
         if isinstance(decorator, RefExpr):
-            if decorator.fullname in ('typing.runtime', 'typing_extensions.runtime'):
+            if decorator.fullname in RUNTIME_PROTOCOL_DECOS:
                 if defn.info.is_protocol:
                     defn.info.runtime_protocol = True
                 else:
-                    self.fail('@runtime can only be used with protocol classes', defn)
+                    self.fail('@runtime_checkable can only be used with protocol classes',
+                              defn)
             elif decorator.fullname in ('typing.final',
                                         'typing_extensions.final'):
                 defn.info.is_final = True

mypy/newsemanal/semanal_typeddict.py

@@ -303,8 +303,9 @@ def fail_typeddict_arg(self, message: str,
     def build_typeddict_typeinfo(self, name: str, items: List[str],
                                  types: List[Type],
                                  required_keys: Set[str]) -> TypeInfo:
-        # Prefer typing_extensions if available.
-        fallback = (self.api.named_type_or_none('typing_extensions._TypedDict', []) or
+        # Prefer typing then typing_extensions if available.
+        fallback = (self.api.named_type_or_none('typing._TypedDict', []) or
+                    self.api.named_type_or_none('typing_extensions._TypedDict', []) or
                     self.api.named_type_or_none('mypy_extensions._TypedDict', []))
         assert fallback is not None
         info = self.api.basic_new_typeinfo(name, fallback)

mypy/nodes.py

@@ -143,6 +143,10 @@ def get_column(self) -> int:
                    'builtins.enumerate': ''}  # type: Final
 nongen_builtins.update((name, alias) for alias, name in type_aliases.items())
 
+RUNTIME_PROTOCOL_DECOS = ('typing.runtime_checkable',
+                          'typing_extensions.runtime',
+                          'typing_extensions.runtime_checkable')  # type: Final
+
 
 class Node(Context):
     """Common base class for all non-type parse tree nodes."""

mypy/semanal.py

@@ -56,7 +56,8 @@
     YieldExpr, ExecStmt, BackquoteExpr, ImportBase, AwaitExpr,
     IntExpr, FloatExpr, UnicodeExpr, TempNode, ImportedName, OverloadPart,
     COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, nongen_builtins,
-    get_member_expr_fullname, REVEAL_TYPE, REVEAL_LOCALS, is_final_node
+    get_member_expr_fullname, REVEAL_TYPE, REVEAL_LOCALS, is_final_node,
+    RUNTIME_PROTOCOL_DECOS,
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.typevars import fill_typevars
@@ -945,11 +946,12 @@ def leave_class(self) -> None:
     def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None:
         decorator.accept(self)
         if isinstance(decorator, RefExpr):
-            if decorator.fullname in ('typing.runtime', 'typing_extensions.runtime'):
+            if decorator.fullname in RUNTIME_PROTOCOL_DECOS:
                 if defn.info.is_protocol:
                     defn.info.runtime_protocol = True
                 else:
-                    self.fail('@runtime can only be used with protocol classes', defn)
+                    self.fail('@runtime_checkable can only be used with protocol classes',
+                              defn)
             elif decorator.fullname in ('typing.final',
                                         'typing_extensions.final'):
                 defn.info.is_final = True

mypy/semanal_typeddict.py

@@ -271,8 +271,9 @@ def fail_typeddict_arg(self, message: str,
     def build_typeddict_typeinfo(self, name: str, items: List[str],
                                  types: List[Type],
                                  required_keys: Set[str]) -> TypeInfo:
-        # Prefer typing_extensions if available.
-        fallback = (self.api.named_type_or_none('typing_extensions._TypedDict', []) or
+        # Prefer typing then typing_extensions if available.
+        fallback = (self.api.named_type_or_none('typing._TypedDict', []) or
+                    self.api.named_type_or_none('typing_extensions._TypedDict', []) or
                     self.api.named_type_or_none('mypy_extensions._TypedDict', []))
         assert fallback is not None
         info = self.api.basic_new_typeinfo(name, fallback)

mypy/types.py

@@ -70,10 +70,14 @@
     )
 
 # Supported names of TypedDict type constructors.
-TPDICT_NAMES = ('mypy_extensions.TypedDict', 'typing_extensions.TypedDict')  # type: Final
+TPDICT_NAMES = ('typing.TypedDict',
+                'typing_extensions.TypedDict',
+                'mypy_extensions.TypedDict')  # type: Final
 
 # Supported fallback instance type names for TypedDict types.
-TPDICT_FB_NAMES = ('mypy_extensions._TypedDict', 'typing_extensions._TypedDict')  # type: Final
+TPDICT_FB_NAMES = ('typing._TypedDict',
+                   'typing_extensions._TypedDict',
+                   'mypy_extensions._TypedDict')  # type: Final
 
 # A placeholder used for Bogus[...] parameters
 _dummy = object()  # type: Final[Any]

mypy/typeshed

@@ -36,6 +36,16 @@ reveal_type(f)  # N: Revealed type is 'def (x: Any)'
 reveal_type(g)  # N: Revealed type is 'def (x: Literal['A['])'
 [out]
 
+[case testLiteralFromTypingWorks]
+from typing import Literal
+
+x: Literal[42]
+x = 43  # E: Incompatible types in assignment (expression has type "Literal[43]", variable has type "Literal[42]")
+
+y: Literal[43]
+y = 43
+[typing fixtures/typing-full.pyi]
+
 [case testLiteralParsingPython2]
 # flags: --python-version 2.7
 from typing import Optional

test-data/unit/check-literal.test

@@ -359,9 +359,9 @@ class B2(P2):
 x2: P2 = B2()  # OK
 
 [case testProtocolAndRuntimeAreDefinedAlsoInTypingExtensions]
-from typing_extensions import Protocol, runtime
+from typing_extensions import Protocol, runtime_checkable
 
-@runtime
+@runtime_checkable
 class P(Protocol):
     def meth(self) -> int:
         pass
@@ -1296,9 +1296,9 @@ reveal_type(last(L[int]())) # N: Revealed type is '__main__.Box*[builtins.int*]'
 reveal_type(last(L[str]()).content) # N: Revealed type is 'builtins.str*'
 
 [case testOverloadOnProtocol]
-from typing import overload, Protocol, runtime
+from typing import overload, Protocol, runtime_checkable
 
-@runtime
+@runtime_checkable
 class P1(Protocol):
     attr1: int
 class P2(Protocol):
@@ -1317,7 +1317,7 @@ def f(x: P2) -> str: ...
 def f(x):
     if isinstance(x, P1):
         return P1.attr1
-    if isinstance(x, P2): # E: Only @runtime protocols can be used with instance and class checks
+    if isinstance(x, P2): # E: Only @runtime_checkable protocols can be used with instance and class checks
         return P1.attr2
 
 reveal_type(f(C1())) # N: Revealed type is 'builtins.int'
@@ -1480,28 +1480,28 @@ class C(Protocol):
         Logger.log(cls)  #OK for classmethods
 [builtins fixtures/classmethod.pyi]
 
--- isinstance() with @runtime protocols
--- ------------------------------------
+-- isinstance() with @runtime_checkable protocols
+-- ----------------------------------------------
 
 [case testSimpleRuntimeProtocolCheck]
-from typing import Protocol, runtime
+from typing import Protocol, runtime_checkable
 
-@runtime
-class C:  # E: @runtime can only be used with protocol classes
+@runtime_checkable
+class C:  # E: @runtime_checkable can only be used with protocol classes
     pass
 
 class P(Protocol):
     def meth(self) -> None:
         pass
 
-@runtime
+@runtime_checkable
 class R(Protocol):
     def meth(self) -> int:
         pass
 
 x: object
 
-if isinstance(x, P):  # E: Only @runtime protocols can be used with instance and class checks
+if isinstance(x, P):  # E: Only @runtime_checkable protocols can be used with instance and class checks
     reveal_type(x)  # N: Revealed type is '__main__.P'
 
 if isinstance(x, R):
@@ -1521,19 +1521,19 @@ if isinstance(x, Iterable):
 [typing fixtures/typing-full.pyi]
 
 [case testConcreteClassesInProtocolsIsInstance]
-from typing import Protocol, runtime, TypeVar, Generic
+from typing import Protocol, runtime_checkable, TypeVar, Generic
 
 T = TypeVar('T')
 
-@runtime
+@runtime_checkable
 class P1(Protocol):
     def meth1(self) -> int:
         pass
-@runtime
+@runtime_checkable
 class P2(Protocol):
     def meth2(self) -> int:
         pass
-@runtime
+@runtime_checkable
 class P(P1, P2, Protocol):
     pass
 
@@ -1581,15 +1581,15 @@ else:
 [typing fixtures/typing-full.pyi]
 
 [case testConcreteClassesUnionInProtocolsIsInstance]
-from typing import Protocol, runtime, TypeVar, Generic, Union
+from typing import Protocol, runtime_checkable, TypeVar, Generic, Union
 
 T = TypeVar('T')
 
-@runtime
+@runtime_checkable
 class P1(Protocol):
     def meth1(self) -> int:
         pass
-@runtime
+@runtime_checkable
 class P2(Protocol):
     def meth2(self) -> int:
         pass
@@ -2193,12 +2193,12 @@ y: PBad = None  # E: Incompatible types in assignment (expression has type "None
 [out]
 
 [case testOnlyMethodProtocolUsableWithIsSubclass]
-from typing import Protocol, runtime, Union, Type
-@runtime
+from typing import Protocol, runtime_checkable, Union, Type
+@runtime_checkable
 class P(Protocol):
     def meth(self) -> int:
         pass
-@runtime
+@runtime_checkable
 class PBad(Protocol):
     x: str
 

test-data/unit/check-protocols.test

@@ -1600,3 +1600,16 @@ class Point(TypedDict):
 p = Point(x=42, y=1337)
 reveal_type(p)  # N: Revealed type is 'TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})'
 [builtins fixtures/dict.pyi]
+
+[case testCanCreateTypedDictWithTypingProper]
+# flags: --python-version 3.8
+from typing import TypedDict
+
+class Point(TypedDict):
+    x: int
+    y: int
+
+p = Point(x=42, y=1337)
+reveal_type(p)  # N: Revealed type is 'TypedDict('__main__.Point', {'x': builtins.int, 'y': builtins.int})'
+[builtins fixtures/dict.pyi]
+[typing fixtures/typing-full.pyi]

test-data/unit/check-typeddict.test

@@ -25,6 +25,9 @@ NamedTuple = 0
 Type = 0
 no_type_check = 0
 ClassVar = 0
+Final = 0
+Literal = 0
+TypedDict = 0
 NoReturn = 0
 NewType = 0
 
@@ -38,23 +41,23 @@ S = TypeVar('S')
 # Note: definitions below are different from typeshed, variances are declared
 # to silence the protocol variance checks. Maybe it is better to use type: ignore?
 
-@runtime
+@runtime_checkable
 class Container(Protocol[T_co]):
     @abstractmethod
     # Use int because bool isn't in the default test builtins
     def __contains__(self, arg: object) -> int: pass
 
-@runtime
+@runtime_checkable
 class Sized(Protocol):
     @abstractmethod
     def __len__(self) -> int: pass
 
-@runtime
+@runtime_checkable
 class Iterable(Protocol[T_co]):
     @abstractmethod
     def __iter__(self) -> 'Iterator[T_co]': pass
 
-@runtime
+@runtime_checkable
 class Iterator(Iterable[T_co], Protocol):
     @abstractmethod
     def __next__(self) -> T_co: pass
@@ -88,7 +91,7 @@ class AsyncGenerator(AsyncIterator[T], Generic[T, U]):
     @abstractmethod
     def __aiter__(self) -> 'AsyncGenerator[T, U]': pass
 
-@runtime
+@runtime_checkable
 class Awaitable(Protocol[T]):
     @abstractmethod
     def __await__(self) -> Generator[Any, Any, T]: pass
@@ -106,12 +109,12 @@ class Coroutine(Awaitable[V], Generic[T, U, V]):
     @abstractmethod
     def close(self) -> None: pass
 
-@runtime
+@runtime_checkable
 class AsyncIterable(Protocol[T]):
     @abstractmethod
     def __aiter__(self) -> 'AsyncIterator[T]': pass
 
-@runtime
+@runtime_checkable
 class AsyncIterator(AsyncIterable[T], Protocol):
     def __aiter__(self) -> 'AsyncIterator[T]': return self
     @abstractmethod
@@ -137,7 +140,7 @@ class MutableMapping(Mapping[T, U], metaclass=ABCMeta):
 class SupportsInt(Protocol):
     def __int__(self) -> int: pass
 
-def runtime(cls: T) -> T:
+def runtime_checkable(cls: T) -> T:
     return cls
 
 class ContextManager(Generic[T]):
@@ -146,3 +149,16 @@ class ContextManager(Generic[T]):
     def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Any: pass
 
 TYPE_CHECKING = 1
+
+# Fallback type for all typed dicts (does not exist at runtime).
+class _TypedDict(Mapping[str, object]):
+    # Needed to make this class non-abstract. It is explicitly declared abstract in
+    # typeshed, but we don't want to import abc here, as it would slow down the tests.
+    def __iter__(self) -> Iterator[str]: ...
+    def copy(self: T) -> T: ...
+    # Using NoReturn so that only calls using the plugin hook can go through.
+    def setdefault(self, k: NoReturn, default: object) -> object: ...
+    # Mypy expects that 'default' has a type variable type.
+    def pop(self, k: NoReturn, default: T = ...) -> object: ...
+    def update(self: T, __m: T) -> None: ...
+    def __delitem__(self, k: NoReturn) -> None: ...

test-data/unit/fixtures/typing-full.pyi

@@ -8,7 +8,8 @@ class _SpecialForm:
         pass
 
 Protocol: _SpecialForm = ...
-def runtime(x: _T) -> _T: pass
+def runtime_checkable(x: _T) -> _T: pass
+runtime = runtime_checkable
 
 Final: _SpecialForm = ...
 def final(x: _T) -> _T: pass