Rework support for --allow-hosts / --index-url

CHANGES.rst

@@ -1,3 +1,12 @@
+2.12
+~~~~
+
+* #30: Rework support for ``--allow-hosts`` and
+  ``--index-url``, removing dependence on
+  ``setuptools.Distribution``'s private member.
+  Additionally corrects logic in marker evaluation
+  along with unit tests!
+
 2.11.1
 ~~~~~~
 

docs/conf.py

@@ -49,6 +49,10 @@
 				pattern=r"PEP[- ](?P<pep_number>\d+)",
 				url='https://www.python.org/dev/peps/pep-{pep_number:0>4}/',
 			),
+			dict(
+				pattern=r"Setuptools #(?P<setuptools_issue>\d+)",
+				url='https://github.com/pypa/setuptools/issues/{setuptools_issue}/',
+			),
 		],
 	),
 }

ptr.py

@@ -17,6 +17,7 @@
 
 import pkg_resources
 import setuptools.command.test as orig
+from setuptools import Distribution
 
 
 @_contextlib.contextmanager
@@ -35,6 +36,46 @@ def null():
 	yield
 
 
+class CustomizedDist(Distribution):
+
+	allow_hosts = None
+	index_url = None
+
+	def fetch_build_egg(self, req):
+		""" Specialized version of Distribution.fetch_build_egg
+		that respects respects allow_hosts and index_url. """
+		from setuptools.command.easy_install import easy_install
+		dist = Distribution({'script_args': ['easy_install']})
+		dist.parse_config_files()
+		opts = dist.get_option_dict('easy_install')
+		keep = (
+			'find_links', 'site_dirs', 'index_url', 'optimize',
+			'site_dirs', 'allow_hosts'
+		)
+		for key in list(opts):
+			if key not in keep:
+				del opts[key]  # don't use any other settings
+		if self.dependency_links:
+			links = self.dependency_links[:]
+			if 'find_links' in opts:
+				links = opts['find_links'][1].split() + links
+			opts['find_links'] = ('setup', links)
+		if self.allow_hosts:
+			opts['allow_hosts'] = ('test', self.allow_hosts)
+		if self.index_url:
+			opts['index_url'] = ('test', self.index_url)
+		install_dir_func = getattr(self, 'get_egg_cache_dir', _os.getcwd)
+		install_dir = install_dir_func()
+		cmd = easy_install(
+			dist, args=["x"], install_dir=install_dir,
+			exclude_scripts=True,
+			always_copy=False, build_directory=None, editable=False,
+			upgrade=False, multi_version=True, no_report=True, user=False
+		)
+		cmd.ensure_finalized()
+		return cmd.easy_install(req)
+
+
 class PyTest(orig.test):
 	"""
 	>>> import setuptools
@@ -69,8 +110,8 @@ def marker_passes(marker):
 		and matches this environment.
 		"""
 		return (
-			marker
-			and not pkg_resources.invalid_marker(marker)
+			not marker
+			or not pkg_resources.invalid_marker(marker)
 			and pkg_resources.evaluate_marker(marker)
 		)
 
@@ -104,10 +145,10 @@ def install_extra_dists(self, dist):
 		matching_extras = (
 			reqs
 			for (name, sep, marker), reqs in spec_extras
-			# never include extras that fail to pass marker eval
-			if self.marker_passes(marker)
 			# include unnamed extras or all if self.extras indicated
-			and (not name or self.extras)
+			if (not name or self.extras)
+			# never include extras that fail to pass marker eval
+			and self.marker_passes(marker)
 		)
 		results = list(map(dist.fetch_build_eggs, matching_extras))
 		return _itertools.chain.from_iterable(results)
@@ -129,11 +170,12 @@ def paths_on_pythonpath(paths):
 			return null()
 
 	def _super_run(self):
-		if hasattr(orig.test, 'install_dists'):
-			return orig.test.run(self)
-
-		# for backward compatibility with setuptools < 27.3
-		installed_dists = self.install_dists(self.distribution)
+		dist = CustomizedDist()
+		for attr in 'allow_hosts index_url'.split():
+			setattr(dist, attr, getattr(self, attr))
+		for attr in 'install_requires tests_require extras_require'.split():
+			setattr(dist, attr, getattr(self.distribution, attr))
+		installed_dists = self.install_dists(dist)
 		if self.dry_run:
 			self.announce('skipping tests (dry run)')
 			return
@@ -146,47 +188,11 @@ def run(self):
 		Override run to ensure requirements are available in this session (but
 		don't install them anywhere).
 		"""
-		self._build_egg_fetcher()
 		self._super_run()
 		if self.result_code:
 			raise SystemExit(self.result_code)
 		return self.result_code
 
-	def _build_egg_fetcher(self):
-		"""Build an egg fetcher that respects index_url and allow_hosts"""
-		# modified from setuptools.dist:Distribution.fetch_build_egg
-		from setuptools.command.easy_install import easy_install
-		main_dist = self.distribution
-		# construct a fake distribution to store the args for easy_install
-		dist = main_dist.__class__({'script_args': ['easy_install']})
-		dist.parse_config_files()
-		opts = dist.get_option_dict('easy_install')
-		keep = (
-			'find_links', 'site_dirs', 'index_url', 'optimize',
-			'site_dirs', 'allow_hosts'
-		)
-		for key in list(opts.keys()):
-			if key not in keep:
-				del opts[key]   # don't use any other settings
-		if main_dist.dependency_links:
-			links = main_dist.dependency_links[:]
-			if 'find_links' in opts:
-				links = opts['find_links'][1].split() + links
-			opts['find_links'] = ('setup', links)
-		if self.allow_hosts:
-			opts['allow_hosts'] = ('test', self.allow_hosts)
-		if self.index_url:
-			opts['index_url'] = ('test', self.index_url)
-		install_dir_func = getattr(dist, 'get_egg_cache_dir', _os.getcwd)
-		install_dir = install_dir_func()
-		cmd = easy_install(
-			dist, args=["x"], install_dir=install_dir, exclude_scripts=True,
-			always_copy=False, build_directory=None, editable=False,
-			upgrade=False, multi_version=True, no_report = True
-		)
-		cmd.ensure_finalized()
-		main_dist._egg_fetcher = cmd
-
 	@property
 	def _argv(self):
 		return ['pytest'] + self.addopts

tests/requirements.txt

@@ -1,2 +1,4 @@
 pytest >= 2.8
+pytest-virtualenv
+importlib; python_version=="2.6"
 subprocess32; python_version=="2.6"

tests/test_ptr.py

@@ -0,0 +1,157 @@
+from __future__ import unicode_literals
+
+import contextlib
+import io
+import os
+import sys
+import tarfile
+import textwrap
+import time
+
+import pytest
+
+
+def DALS(s):
+    "dedent and left-strip"
+    return textwrap.dedent(s).lstrip()
+
+
+def _tarfile_open_ex(*args, **kwargs):
+    """
+    Extend result as a context manager.
+    """
+    return contextlib.closing(tarfile.open(*args, **kwargs))
+
+
+if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 2):
+    tarfile_open = _tarfile_open_ex
+else:
+    tarfile_open = tarfile.open
+
+
+def make_sdist(dist_path, files):
+    """
+    Create a simple sdist tarball at dist_path, containing the files
+    listed in ``files`` as ``(filename, content)`` tuples.
+    """
+
+    with tarfile_open(dist_path, 'w:gz') as dist:
+        for filename, content in files:
+            file_bytes = io.BytesIO(content.encode('utf-8'))
+            file_info = tarfile.TarInfo(name=filename)
+            file_info.size = len(file_bytes.getvalue())
+            file_info.mtime = int(time.time())
+            dist.addfile(file_info, fileobj=file_bytes)
+
+
+@pytest.fixture
+def venv(virtualenv):
+    yield virtualenv
+    # Workaround virtualenv not cleaning itself as it should...
+    virtualenv.delete = True
+    virtualenv.teardown()
+
+
+@pytest.mark.parametrize('setuptools_req, test_args', (
+    ('setuptools==27.2.0', ''),
+    ('setuptools==27.2.0', '--extras'),
+    ('setuptools==27.3.0', ''),
+    ('setuptools==27.3.0', '--extras'),
+    ('setuptools==32.3.1', ''),
+    ('setuptools==32.3.1', '--extras'),
+    ('setuptools==36.3.0', ''),
+    ('setuptools==36.3.0', '--extras'),
+    ('setuptools'        , ''),
+    ('setuptools'        , '--extras'),
+))
+def test_egg_fetcher(venv, setuptools_req, test_args):
+    test_args = test_args.split()
+    # Install pytest & pytest-runner.
+    venv.run('python setup.py develop', cwd=os.getcwd())
+    venv.run('pip install pytest')
+    # Install setuptools version.
+    venv.run('pip install -U'.split() + [setuptools_req])
+    # For debugging purposes.
+    venv.run('pip freeze --all')
+    # Prepare fake index.
+    index_dir = (venv.workspace / 'index').mkdir()
+    for n in range(5):
+        dist_name = 'barbazquux' + str(n + 1)
+        dist_version = '0.1'
+        dist_sdist = '%s-%s.tar.gz' % (dist_name, dist_version)
+        dist_dir = (index_dir / dist_name).mkdir()
+        make_sdist(dist_dir / dist_sdist, (
+            ('setup.py', textwrap.dedent(
+                '''
+                from setuptools import setup
+                setup(
+                    name={dist_name!r},
+                    version={dist_version!r},
+                    py_modules=[{dist_name!r}],
+                )
+                '''
+            ).format(dist_name=dist_name, dist_version=dist_version)),
+            (dist_name + '.py', ''),
+        ))
+        with (dist_dir / 'index.html').open('w') as fp:
+            fp.write(DALS(
+                '''
+                <!DOCTYPE html><html><body>
+                <a href="{dist_sdist}" rel="internal">{dist_sdist}</a><br/>
+                </body></html>
+                '''
+            ).format(dist_sdist=dist_sdist))
+    # Prepare fake project.
+    project_dir = (venv.workspace / 'project-0.1').mkdir()
+    with open(project_dir / 'setup.py', 'w') as fp:
+        fp.write(DALS(
+            '''
+            from setuptools import setup
+            setup(
+                name='project',
+                version='0.1',
+                setup_requires=[
+                    'pytest-runner',
+                ],
+                install_requires=[
+                    'barbazquux1',
+                ],
+                tests_require=[
+                    'pytest',
+                    'barbazquux2',
+                ],
+                extras_require={{
+                    ':"{sys_platform}" in sys_platform': 'barbazquux3',
+                    ':"barbazquux" in sys_platform': 'barbazquux4',
+                    'extra': 'barbazquux5',
+                }}
+            )
+            ''').format(sys_platform=sys.platform))
+    with open(project_dir / 'setup.cfg', 'w') as fp:
+        fp.write(DALS(
+            '''
+            [easy_install]
+            index_url = .
+            '''))
+    with open(project_dir / 'test_stuff.py', 'w') as fp:
+        fp.write(DALS(
+            '''
+            import pytest
+
+            def test_stuff():
+                import barbazquux1
+                import barbazquux2
+                import barbazquux3
+                with pytest.raises(ImportError):
+                    import barbazquux4
+                if {importable_barbazquux5}:
+                    import barbazquux5
+                else:
+                    with pytest.raises(ImportError):
+                        import barbazquux5
+            ''').format(importable_barbazquux5=('--extras' in test_args)))
+    # Run fake project tests.
+    cmd = 'python setup.py pytest'.split()
+    cmd += ['--index-url=' + index_dir.abspath()]
+    cmd += test_args
+    venv.run(cmd, cwd=project_dir)