Make Options work under mypyc (#5518)

This requires reworking a bunch of operations to not directly depend
on the __dict__

Author: msullivan

mypy/main.py

@@ -19,7 +19,7 @@
 from mypy.find_sources import create_source_list, InvalidSourceList
 from mypy.fscache import FileSystemCache
 from mypy.errors import CompileError
-from mypy.options import Options, BuildType
+from mypy.options import Options, BuildType, PER_MODULE_OPTIONS
 from mypy.report import reporter_classes
 
 from mypy.version import __version__
@@ -1014,11 +1014,11 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:
                 print("%s: Per-module sections should not specify reports (%s)" %
                       (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
                       file=sys.stderr)
-            if set(updates) - Options.PER_MODULE_OPTIONS:
+            if set(updates) - PER_MODULE_OPTIONS:
                 print("%s: Per-module sections should only specify per-module flags (%s)" %
-                      (prefix, ', '.join(sorted(set(updates) - Options.PER_MODULE_OPTIONS))),
+                      (prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))),
                       file=sys.stderr)
-                updates = {k: v for k, v in updates.items() if k in Options.PER_MODULE_OPTIONS}
+                updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS}
             globs = name[5:]
             for glob in globs.split(','):
                 # For backwards compatibility, replace (back)slashes with dots.

mypy/options.py

@@ -4,54 +4,59 @@
 import sys
 
 from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
+MYPY = False
+if MYPY:
+    from typing import ClassVar
 
 from mypy import defaults
+from mypy.util import get_class_descriptors, replace_object_state
 
 
 class BuildType:
-    STANDARD = 0
-    MODULE = 1
-    PROGRAM_TEXT = 2
+    STANDARD = 0  # type: ClassVar[int]
+    MODULE = 1  # type: ClassVar[int]
+    PROGRAM_TEXT = 2  # type: ClassVar[int]
+
+
+PER_MODULE_OPTIONS = {
+    # Please keep this list sorted
+    "always_false",
+    "always_true",
+    "check_untyped_defs",
+    "debug_cache",
+    "disallow_any_decorated",
+    "disallow_any_explicit",
+    "disallow_any_expr",
+    "disallow_any_generics",
+    "disallow_any_unimported",
+    "disallow_incomplete_defs",
+    "disallow_subclassing_any",
+    "disallow_untyped_calls",
+    "disallow_untyped_decorators",
+    "disallow_untyped_defs",
+    "follow_imports",
+    "follow_imports_for_stubs",
+    "ignore_errors",
+    "ignore_missing_imports",
+    "local_partial_types",
+    "no_implicit_optional",
+    "show_none_errors",
+    "strict_boolean",
+    "strict_optional",
+    "strict_optional_whitelist",
+    "warn_no_return",
+    "warn_return_any",
+    "warn_unused_ignores",
+}
+
+OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS |
+                            {"quick_and_dirty", "platform", "bazel"})
+                           - {"debug_cache"})
 
 
 class Options:
     """Options collected from flags."""
 
-    PER_MODULE_OPTIONS = {
-        # Please keep this list sorted
-        "always_false",
-        "always_true",
-        "check_untyped_defs",
-        "debug_cache",
-        "disallow_any_decorated",
-        "disallow_any_explicit",
-        "disallow_any_expr",
-        "disallow_any_generics",
-        "disallow_any_unimported",
-        "disallow_incomplete_defs",
-        "disallow_subclassing_any",
-        "disallow_untyped_calls",
-        "disallow_untyped_decorators",
-        "disallow_untyped_defs",
-        "follow_imports",
-        "follow_imports_for_stubs",
-        "ignore_errors",
-        "ignore_missing_imports",
-        "local_partial_types",
-        "no_implicit_optional",
-        "show_none_errors",
-        "strict_boolean",
-        "strict_optional",
-        "strict_optional_whitelist",
-        "warn_no_return",
-        "warn_return_any",
-        "warn_unused_ignores",
-    }
-
-    OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS |
-                                {"quick_and_dirty", "platform", "bazel"})
-                               - {"debug_cache"})
-
     def __init__(self) -> None:
         # Cache for clone_for_module()
         self.per_module_cache = None  # type: Optional[Dict[str, Options]]
@@ -209,23 +214,23 @@ def __init__(self) -> None:
 
     def snapshot(self) -> object:
         """Produce a comparable snapshot of this Option"""
-        d = dict(self.__dict__)
+        # Under mypyc, we don't have a __dict__, so we need to do worse things.
+        d = dict(getattr(self, '__dict__', ()))
+        for k in get_class_descriptors(Options):
+            if hasattr(self, k):
+                d[k] = getattr(self, k)
         del d['per_module_cache']
         return d
 
-    def __eq__(self, other: object) -> bool:
-        return self.__class__ == other.__class__ and self.__dict__ == other.__dict__
-
-    def __ne__(self, other: object) -> bool:
-        return not self == other
-
     def __repr__(self) -> str:
         return 'Options({})'.format(pprint.pformat(self.snapshot()))
 
     def apply_changes(self, changes: Dict[str, object]) -> 'Options':
         new_options = Options()
-        new_options.__dict__.update(self.__dict__)
-        new_options.__dict__.update(changes)
+        # Under mypyc, we don't have a __dict__, so we need to do worse things.
+        replace_object_state(new_options, self, copy_dict=True)
+        for key, value in changes.items():
+            setattr(new_options, key, value)
         return new_options
 
     def build_per_module_cache(self) -> None:
@@ -281,7 +286,7 @@ def clone_for_module(self, module: str) -> 'Options':
         """
         if self.per_module_cache is None:
             self.build_per_module_cache()
-            assert self.per_module_cache is not None
+        assert self.per_module_cache is not None
 
         # If the module just directly has a config entry, use it.
         if module in self.per_module_cache:
@@ -327,4 +332,4 @@ def compile_glob(self, s: str) -> Pattern[str]:
         return re.compile(expr + '\\Z')
 
     def select_options_affecting_cache(self) -> Mapping[str, object]:
-        return {opt: getattr(self, opt) for opt in self.OPTIONS_AFFECTING_CACHE}
+        return {opt: getattr(self, opt) for opt in OPTIONS_AFFECTING_CACHE}

mypy/test/testargs.py

@@ -21,7 +21,7 @@ def test_coherence(self) -> None:
         _, parsed_options = process_options([], require_targets=False)
         # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg'
         options.config_file = parsed_options.config_file
-        assert_equal(options, parsed_options)
+        assert_equal(options.snapshot(), parsed_options.snapshot())
 
     def test_executable_inference(self) -> None:
         """Test the --python-executable flag with --python-version"""

mypy/types.py

@@ -1749,11 +1749,7 @@ def copy_type(t: Type) -> Type:
     # and copying everything in with replace_object_state.
     typ = type(t)
     nt = typ.__new__(typ)
-    replace_object_state(nt, t)
-    # replace_object_state leaves the new object with the same
-    # __dict__ as the old, so make a copy.
-    if hasattr(nt, '__dict__'):
-        nt.__dict__ = nt.__dict__.copy()
+    replace_object_state(nt, t, copy_dict=True)
     return nt
 
 

mypy/util.py

@@ -196,7 +196,18 @@ def correct_relative_import(cur_mod_id: str,
 fields_cache = {}  # type: Dict[Type[object], List[str]]
 
 
-def replace_object_state(new: object, old: object) -> None:
+def get_class_descriptors(cls: 'Type[object]') -> Sequence[str]:
+    # Maintain a cache of type -> attributes defined by descriptors in the class
+    # (that is, attributes from __slots__ and C extension classes)
+    if cls not in fields_cache:
+        members = inspect.getmembers(
+            cls,
+            lambda o: inspect.isgetsetdescriptor(o) or inspect.ismemberdescriptor(o))
+        fields_cache[cls] = [x for x, y in members if x != '__weakref__']
+    return fields_cache[cls]
+
+
+def replace_object_state(new: object, old: object, copy_dict: bool=False) -> None:
     """Copy state of old node to the new node.
 
     This handles cases where there is __dict__ and/or attribute descriptors
@@ -205,17 +216,12 @@ def replace_object_state(new: object, old: object) -> None:
     Assume that both objects have the same __class__.
     """
     if hasattr(old, '__dict__'):
-        new.__dict__ = old.__dict__
+        if copy_dict:
+            new.__dict__ = dict(old.__dict__)
+        else:
+            new.__dict__ = old.__dict__
 
-    cls = old.__class__
-    # Maintain a cache of type -> attributes defined by descriptors in the class
-    # (that is, attributes from __slots__ and C extension classes)
-    if cls not in fields_cache:
-        members = inspect.getmembers(
-            cls,
-            lambda o: inspect.isgetsetdescriptor(o) or inspect.ismemberdescriptor(o))
-        fields_cache[cls] = [x for x, y in members if x != '__weakref__']
-    for attr in fields_cache[cls]:
+    for attr in get_class_descriptors(old.__class__):
         try:
             if hasattr(old, attr):
                 setattr(new, attr, getattr(old, attr))