From 2fdf6fd371a88d8cc06411b02167a3d01eb1348d Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 21 Aug 2018 16:05:51 -0700 Subject: [PATCH 1/6] Make Options work under mypyc This requires reworking a bunch of operations to not directly depend on the __dict__ --- mypy/options.py | 35 ++++++++++++++++++++++------------- mypy/types.py | 6 +----- mypy/util.py | 28 +++++++++++++++++----------- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index 9171dd865cdd..eea7b8369f39 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -4,14 +4,17 @@ import sys from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple +MYPY = False +if MYPY: + from typing import ClassVar from mypy import defaults - +from mypy.util import get_class_descriptors, replace_object_state class BuildType: - STANDARD = 0 - MODULE = 1 - PROGRAM_TEXT = 2 + STANDARD = 0 # type: ClassVar[int] + MODULE = 1 # type: ClassVar[int] + PROGRAM_TEXT = 2 # type: ClassVar[int] class Options: @@ -46,11 +49,11 @@ class Options: "warn_no_return", "warn_return_any", "warn_unused_ignores", - } + } # type: ClassVar[Set[str]] - OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS | + OPTIONS_AFFECTING_CACHE = ((Options.PER_MODULE_OPTIONS | {"quick_and_dirty", "platform", "bazel"}) - - {"debug_cache"}) + - {"debug_cache"}) # type: ClassVar[Set[str]] def __init__(self) -> None: # Cache for clone_for_module() @@ -209,12 +212,16 @@ def __init__(self) -> None: def snapshot(self) -> object: """Produce a comparable snapshot of this Option""" - d = dict(self.__dict__) + # Under mypyc, we don't have a __dict__, so we need to do worse things. + d = dict(getattr(self, '__dict__', ())) + for k in get_class_descriptors(Options): + if hasattr(self, k): + d[k] = getattr(self, k) del d['per_module_cache'] return d def __eq__(self, other: object) -> bool: - return self.__class__ == other.__class__ and self.__dict__ == other.__dict__ + return isinstance(other, Options) and self.snapshot() == other.snapshot() def __ne__(self, other: object) -> bool: return not self == other @@ -224,8 +231,10 @@ def __repr__(self) -> str: def apply_changes(self, changes: Dict[str, object]) -> 'Options': new_options = Options() - new_options.__dict__.update(self.__dict__) - new_options.__dict__.update(changes) + # Under mypyc, we don't have a __dict__, so we need to do worse things. + replace_object_state(new_options, self, copy_dict=True) + for key, value in changes.items(): + setattr(new_options, key, value) return new_options def build_per_module_cache(self) -> None: @@ -281,7 +290,7 @@ def clone_for_module(self, module: str) -> 'Options': """ if self.per_module_cache is None: self.build_per_module_cache() - assert self.per_module_cache is not None + assert self.per_module_cache is not None # If the module just directly has a config entry, use it. if module in self.per_module_cache: @@ -327,4 +336,4 @@ def compile_glob(self, s: str) -> Pattern[str]: return re.compile(expr + '\\Z') def select_options_affecting_cache(self) -> Mapping[str, object]: - return {opt: getattr(self, opt) for opt in self.OPTIONS_AFFECTING_CACHE} + return {opt: getattr(self, opt) for opt in Options.OPTIONS_AFFECTING_CACHE} diff --git a/mypy/types.py b/mypy/types.py index 19a8d9274d0f..2fbe0d66a239 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1743,11 +1743,7 @@ def copy_type(t: Type) -> Type: # and copying everything in with replace_object_state. typ = type(t) nt = typ.__new__(typ) - replace_object_state(nt, t) - # replace_object_state leaves the new object with the same - # __dict__ as the old, so make a copy. - if hasattr(nt, '__dict__'): - nt.__dict__ = nt.__dict__.copy() + replace_object_state(nt, t, copy_dict=True) return nt diff --git a/mypy/util.py b/mypy/util.py index d20cb40e17e3..d2bc5db676bc 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -196,7 +196,18 @@ def correct_relative_import(cur_mod_id: str, fields_cache = {} # type: Dict[Type[object], List[str]] -def replace_object_state(new: object, old: object) -> None: +def get_class_descriptors(cls: 'Type[object]') -> Sequence[str]: + # Maintain a cache of type -> attributes defined by descriptors in the class + # (that is, attributes from __slots__ and C extension classes) + if cls not in fields_cache: + members = inspect.getmembers( + cls, + lambda o: inspect.isgetsetdescriptor(o) or inspect.ismemberdescriptor(o)) + fields_cache[cls] = [x for x, y in members if x != '__weakref__'] + return fields_cache[cls] + + +def replace_object_state(new: object, old: object, copy_dict: bool=False) -> None: """Copy state of old node to the new node. This handles cases where there is __dict__ and/or attribute descriptors @@ -205,17 +216,12 @@ def replace_object_state(new: object, old: object) -> None: Assume that both objects have the same __class__. """ if hasattr(old, '__dict__'): - new.__dict__ = old.__dict__ + if copy_dict: + new.__dict__ = dict(old.__dict__) + else: + new.__dict__ = old.__dict__ - cls = old.__class__ - # Maintain a cache of type -> attributes defined by descriptors in the class - # (that is, attributes from __slots__ and C extension classes) - if cls not in fields_cache: - members = inspect.getmembers( - cls, - lambda o: inspect.isgetsetdescriptor(o) or inspect.ismemberdescriptor(o)) - fields_cache[cls] = [x for x, y in members if x != '__weakref__'] - for attr in fields_cache[cls]: + for attr in get_class_descriptors(old.__class__): try: if hasattr(old, attr): setattr(new, attr, getattr(old, attr)) From 069649a30bb7414d0698da01cacdfea687194539 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 21 Aug 2018 16:48:35 -0700 Subject: [PATCH 2/6] fix it to *also* still work under cpython... --- mypy/main.py | 8 +++--- mypy/options.py | 72 ++++++++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index b39bbd2f680c..5dce58a3e64f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -19,7 +19,7 @@ from mypy.find_sources import create_source_list, InvalidSourceList from mypy.fscache import FileSystemCache from mypy.errors import CompileError -from mypy.options import Options, BuildType +from mypy.options import Options, BuildType, PER_MODULE_OPTIONS from mypy.report import reporter_classes from mypy.version import __version__ @@ -1014,11 +1014,11 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None: print("%s: Per-module sections should not specify reports (%s)" % (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))), file=sys.stderr) - if set(updates) - Options.PER_MODULE_OPTIONS: + if set(updates) - PER_MODULE_OPTIONS: print("%s: Per-module sections should only specify per-module flags (%s)" % - (prefix, ', '.join(sorted(set(updates) - Options.PER_MODULE_OPTIONS))), + (prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))), file=sys.stderr) - updates = {k: v for k, v in updates.items() if k in Options.PER_MODULE_OPTIONS} + updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS} globs = name[5:] for glob in globs.split(','): # For backwards compatibility, replace (back)slashes with dots. diff --git a/mypy/options.py b/mypy/options.py index eea7b8369f39..d739815be439 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -17,44 +17,44 @@ class BuildType: PROGRAM_TEXT = 2 # type: ClassVar[int] +PER_MODULE_OPTIONS = { + # Please keep this list sorted + "always_false", + "always_true", + "check_untyped_defs", + "debug_cache", + "disallow_any_decorated", + "disallow_any_explicit", + "disallow_any_expr", + "disallow_any_generics", + "disallow_any_unimported", + "disallow_incomplete_defs", + "disallow_subclassing_any", + "disallow_untyped_calls", + "disallow_untyped_decorators", + "disallow_untyped_defs", + "follow_imports", + "follow_imports_for_stubs", + "ignore_errors", + "ignore_missing_imports", + "local_partial_types", + "no_implicit_optional", + "show_none_errors", + "strict_boolean", + "strict_optional", + "strict_optional_whitelist", + "warn_no_return", + "warn_return_any", + "warn_unused_ignores", +} + +OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS | + {"quick_and_dirty", "platform", "bazel"}) + - {"debug_cache"}) + class Options: """Options collected from flags.""" - PER_MODULE_OPTIONS = { - # Please keep this list sorted - "always_false", - "always_true", - "check_untyped_defs", - "debug_cache", - "disallow_any_decorated", - "disallow_any_explicit", - "disallow_any_expr", - "disallow_any_generics", - "disallow_any_unimported", - "disallow_incomplete_defs", - "disallow_subclassing_any", - "disallow_untyped_calls", - "disallow_untyped_decorators", - "disallow_untyped_defs", - "follow_imports", - "follow_imports_for_stubs", - "ignore_errors", - "ignore_missing_imports", - "local_partial_types", - "no_implicit_optional", - "show_none_errors", - "strict_boolean", - "strict_optional", - "strict_optional_whitelist", - "warn_no_return", - "warn_return_any", - "warn_unused_ignores", - } # type: ClassVar[Set[str]] - - OPTIONS_AFFECTING_CACHE = ((Options.PER_MODULE_OPTIONS | - {"quick_and_dirty", "platform", "bazel"}) - - {"debug_cache"}) # type: ClassVar[Set[str]] - def __init__(self) -> None: # Cache for clone_for_module() self.per_module_cache = None # type: Optional[Dict[str, Options]] @@ -336,4 +336,4 @@ def compile_glob(self, s: str) -> Pattern[str]: return re.compile(expr + '\\Z') def select_options_affecting_cache(self) -> Mapping[str, object]: - return {opt: getattr(self, opt) for opt in Options.OPTIONS_AFFECTING_CACHE} + return {opt: getattr(self, opt) for opt in OPTIONS_AFFECTING_CACHE} From 3f14f681591734e608ce24fedcb2ba150c3bf9bc Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Tue, 21 Aug 2018 18:42:46 -0700 Subject: [PATCH 3/6] fix lint --- mypy/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index d739815be439..a217826187fd 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -11,6 +11,7 @@ from mypy import defaults from mypy.util import get_class_descriptors, replace_object_state + class BuildType: STANDARD = 0 # type: ClassVar[int] MODULE = 1 # type: ClassVar[int] @@ -52,6 +53,7 @@ class BuildType: {"quick_and_dirty", "platform", "bazel"}) - {"debug_cache"}) + class Options: """Options collected from flags.""" From 3b9618baca183654b1ca4194a46e377cedd78f7f Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 28 Aug 2018 09:27:08 -0500 Subject: [PATCH 4/6] Drop __eq__ on Options --- mypy/options.py | 6 ------ mypy/test/testargs.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index a217826187fd..991c1a77fdae 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -222,12 +222,6 @@ def snapshot(self) -> object: del d['per_module_cache'] return d - def __eq__(self, other: object) -> bool: - return isinstance(other, Options) and self.snapshot() == other.snapshot() - - def __ne__(self, other: object) -> bool: - return not self == other - def __repr__(self) -> str: return 'Options({})'.format(pprint.pformat(self.snapshot())) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index ecfe4143d7b4..bccbafaf4c8b 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -21,7 +21,7 @@ def test_coherence(self) -> None: _, parsed_options = process_options([], require_targets=False) # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file - assert_equal(options, parsed_options) + assert_equal(options.snapshot(), parsed_options.snapshot()) def test_executable_inference(self) -> None: """Test the --python-executable flag with --python-version""" From b336dc851a9f21eb810d444b7053f22b4b15aaf6 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 28 Aug 2018 10:17:58 -0500 Subject: [PATCH 5/6] add a comment --- mypy/options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/options.py b/mypy/options.py index 991c1a77fdae..846c87e5cff8 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -18,6 +18,8 @@ class BuildType: PROGRAM_TEXT = 2 # type: ClassVar[int] +# These aren't classvars in Options because mypyc doesn't properly +# implement class namespaces yet. PER_MODULE_OPTIONS = { # Please keep this list sorted "always_false", From b73125dec36b3a9d3ab8b131f557f1ca545e1479 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 28 Aug 2018 11:24:55 -0500 Subject: [PATCH 6/6] Revert "add a comment" This reverts commit b336dc851a9f21eb810d444b7053f22b4b15aaf6. --- mypy/options.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy/options.py b/mypy/options.py index 846c87e5cff8..991c1a77fdae 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -18,8 +18,6 @@ class BuildType: PROGRAM_TEXT = 2 # type: ClassVar[int] -# These aren't classvars in Options because mypyc doesn't properly -# implement class namespaces yet. PER_MODULE_OPTIONS = { # Please keep this list sorted "always_false",