[3.6] bpo-32303 - Consistency fixes for namespace loaders (GH-5481) (#5504)

* Make sure ``__spec__.loader`` matches ``__loader__`` for namespace packages.
* Make sure ``__spec__.origin` matches ``__file__`` for namespace packages.

https://bugs.python.org/issue32303
https://bugs.python.org/issue32305.
(cherry picked from commit bbbcf86)

Co-authored-by: Barry Warsaw <barry@python.org>

Author: warsaw

Doc/library/importlib.rst

@@ -1077,7 +1077,7 @@ find and load modules.
    Name of the place from which the module is loaded, e.g. "builtin" for
    built-in modules and the filename for modules loaded from source.
    Normally "origin" should be set, but it may be ``None`` (the default)
-   which indicates it is unspecified.
+   which indicates it is unspecified (e.g. for namespace packages).
 
    .. attribute:: submodule_search_locations
 

Lib/importlib/_bootstrap.py

@@ -522,6 +522,18 @@ def _init_module_attrs(spec, module, *, override=False):
 
                 loader = _NamespaceLoader.__new__(_NamespaceLoader)
                 loader._path = spec.submodule_search_locations
+                spec.loader = loader
+                # While the docs say that module.__file__ is not set for
+                # built-in modules, and the code below will avoid setting it if
+                # spec.has_location is false, this is incorrect for namespace
+                # packages.  Namespace packages have no location, but their
+                # __spec__.origin is None, and thus their module.__file__
+                # should also be None for consistency.  While a bit of a hack,
+                # this is the best place to ensure this consistency.
+                #
+                # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module
+                # and bpo-32305
+                module.__file__ = None
         try:
             module.__loader__ = loader
         except AttributeError:

Lib/importlib/_bootstrap_external.py

@@ -1160,9 +1160,9 @@ def find_spec(cls, fullname, path=None, target=None):
         elif spec.loader is None:
             namespace_path = spec.submodule_search_locations
             if namespace_path:
-                # We found at least one namespace path.  Return a
-                #  spec which can create the namespace package.
-                spec.origin = 'namespace'
+                # We found at least one namespace path.  Return a spec which
+                # can create the namespace package.
+                spec.origin = None
                 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
                 return spec
             else:

Lib/test/test_importlib/test_api.py

@@ -307,6 +307,7 @@ def test_reload_namespace_changed(self):
                     expected = {'__name__': name,
                                 '__package__': name,
                                 '__doc__': None,
+                                '__file__': None,
                                 }
                     os.mkdir(name)
                     with open(bad_path, 'w') as init_file:
@@ -318,8 +319,9 @@ def test_reload_namespace_changed(self):
                     spec = ns.pop('__spec__')
                     ns.pop('__builtins__', None)  # An implementation detail.
                     self.assertEqual(spec.name, name)
-                    self.assertIs(spec.loader, None)
-                    self.assertIsNot(loader, None)
+                    self.assertIsNotNone(spec.loader)
+                    self.assertIsNotNone(loader)
+                    self.assertEqual(spec.loader, loader)
                     self.assertEqual(set(path),
                                      set([os.path.dirname(bad_path)]))
                     with self.assertRaises(AttributeError):

Lib/test/test_importlib/test_namespace_pkgs.py

@@ -317,5 +317,21 @@ def test_dynamic_path(self):
         self.assertEqual(foo.two.attr, 'portion2 foo two')
 
 
+class LoaderTests(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_namespace_loader_consistency(self):
+        # bpo-32303
+        import foo
+        self.assertEqual(foo.__loader__, foo.__spec__.loader)
+        self.assertIsNotNone(foo.__loader__)
+
+    def test_namespace_origin_consistency(self):
+        # bpo-32305
+        import foo
+        self.assertIsNone(foo.__spec__.origin)
+        self.assertIsNone(foo.__file__)
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2018-02-01-10-16-28.bpo-32303.VsvhSl.rst

@@ -0,0 +1 @@
+Make sure ``__spec__.loader`` matches ``__loader__`` for namespace packages.

Misc/NEWS.d/next/Core and Builtins/2018-02-01-10-56-41.bpo-32305.dkU9Qa.rst

@@ -0,0 +1,2 @@
+For namespace packages, ensure that both ``__file__`` and
+``__spec__.origin`` are set to None.