refactor: simpler design with two outputs

Author: henryiii

.github/workflows/ci.yml

@@ -151,6 +151,7 @@ jobs:
     - name: Interface test
       run: cmake --build build2 --target test_cmake_build
 
+
   clang:
     runs-on: ubuntu-latest
     strategy:

.github/workflows/configure.yml

@@ -76,3 +76,61 @@ jobs:
       working-directory: build dir
       if: github.event_name == 'workflow_dispatch'
       run: cmake --build . --config Release --target check
+
+  test-packaging:
+    name: 🐍 2.7 • 📦 tests • windows-latest
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Setup 🐍 2.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 2.7
+
+    - name: Prepare env
+      run: python -m pip install -r tests/requirements.txt --prefer-binary
+
+    - name: Python Packaging tests
+      run: pytest tests/test_python_package/
+
+
+  packaging:
+    name: 🐍 3.8 • 📦 & 📦 tests • ubuntu-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Setup 🐍 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Prepare env
+      run: python -m pip install -r tests/requirements.txt wheel twine --prefer-binary
+
+    - name: Python Packaging tests
+      run: pytest tests/test_python_package/
+
+    - name: Build SDist
+      run: |
+        python setup.py sdist
+        PYBIND11_ALT_SDIST=1 python setup.py sdist
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: dist/*
+
+    - name: Build wheel
+      run: |
+        python -m pip wheel . -w wheels
+        PYBIND11_ALT_SDIST=1 python -m pip wheel . -w wheels
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: wheels/pybind11*.whl
+
+    - name: Check metadata
+      run: twine check dist/* wheels/*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v3.1.0
+  rev: v3.2.0
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -15,10 +15,10 @@ repos:
   - id: fix-encoding-pragma
 
 - repo: https://github.com/psf/black
-  rev: 19.10b0
+  rev: 20.8b1
   hooks:
   - id: black
-    files: ^(setup.py|pybind11)
+    files: ^(setup.py|pybind11|tests/test_python_package)
 
 - repo: https://github.com/Lucas-C/pre-commit-hooks
   rev: v1.1.9

MANIFEST.in

@@ -1,7 +1,4 @@
-recursive-include include/pybind11 *.h
-recursive-include pybind11 *.h
-recursive-include pybind11 *.cmake
+recursive-include pybind11/include/pybind11 *.h
 recursive-include pybind11 *.py
-recursive-include tools *.cmake
-recursive-include tools *.in
-include CMakeLists.txt LICENSE README.md .github/CONTRIBUTING.md
+include pybind11/share/cmake/pybind11/*.cmake
+include LICENSE README.md pyproject.toml setup.py setup.cfg

setup.cfg

@@ -1,5 +1,4 @@
 [metadata]
-name = pybind11
 long_description = file: README.md
 long_description_content_type = text/markdown
 version = attr: pybind11.__version__
@@ -33,19 +32,7 @@ keywords =
 
 [options]
 python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4
-find_package_data = True
 zip_safe = False
-packages =
-    pybind11
-    pybind11.include.pybind11
-    pybind11.include.pybind11.detail
-    pybind11.share.cmake.pybind11
-
-[options.package_data]
-pybind11.include.pybind11 = *.h
-pybind11.include.pybind11.detail = *.h
-pybind11.share.cmake.pybind11 = *.cmake
-
 
 [sbdist_wheel]
 universal=1
@@ -54,17 +41,17 @@ universal=1
 ignore =
     tests/**
     docs/**
-    tools/check-style.sh
-    tools/clang
-    tools/libsize.py
-    tools/mkdoc.py
+    tools/**
+    include/**
     .appveyor.yml
     .cmake-format.yaml
     .gitmodules
     .pre-commit-config.yaml
     .readthedocs.yml
     pybind11/include/**
     pybind11/share/**
+    CMakeLists.txt
+
 
 [flake8]
 max-line-length = 99
@@ -76,4 +63,4 @@ ignore =
     # camelcase 'cPickle' imported as lowercase 'pickle'
     N813
     # Black conflict
-    W503
+    W503, E203

setup.py

@@ -4,47 +4,41 @@
 # Setup script for PyPI; use CMakeFile.txt to build extension modules
 
 import contextlib
-import glob
 import os
-import re
 import shutil
 import subprocess
 import sys
 import tempfile
-from distutils.command.install_headers import install_headers
 
-from setuptools import setup
+# PYBIND11_ALT_SDIST will build a different sdist, with the python-headers
+# files, and the sys.prefix files (CMake and headers).
 
-# For now, there are three parts to this package. Besides the "normal" module:
-# PYBIND11_USE_HEADERS will include the python-headers files.
-# PYBIND11_USE_SYSTEM will include the sys.prefix files (CMake and headers).
-# The final version will likely only include the normal module or come in
-# different versions.
-
-use_headers = os.environ.get("PYBIND11_USE_HEADERS", False)
-use_system = os.environ.get("PYBIND11_USE_SYSTEM", False)
-
-setup_opts = dict()
+alt_sdist = os.environ.get("PYBIND11_ALT_SDIST", False)
+setup_py = "tools/setup_alt.py" if alt_sdist else "tools/setup_main.py"
+pyproject_toml = "tools/pyproject.toml"
 
 # In a PEP 518 build, this will be in its own environment, so it will not
 # create extra files in the source
 
 DIR = os.path.abspath(os.path.dirname(__file__))
 
-prexist_include = os.path.exists("pybind11/include")
-prexist_share = os.path.exists("pybind11/share")
-
 
 @contextlib.contextmanager
-def monkey_patch_file(input_file):
-    "Allow a file to be temporarily modified"
+def monkey_patch_file(input_file, replacement_file):
+    "Allow a file to be temporarily replaced"
+    inp_file = os.path.abspath(os.path.join(DIR, input_file))
+    rep_file = os.path.abspath(os.path.join(DIR, replacement_file))
 
-    with open(os.path.join(DIR, input_file), "r") as f:
+    with open(inp_file, "rb") as f:
         contents = f.read()
+    with open(rep_file, "rb") as f:
+        replacement = f.read()
     try:
-        yield contents
+        with open(inp_file, "wb") as f:
+            f.write(replacement)
+        yield
     finally:
-        with open(os.path.join(DIR, input_file), "w") as f:
+        with open(inp_file, "wb") as f:
             f.write(contents)
 
 
@@ -67,118 +61,20 @@ def remove_output(*sources):
             shutil.rmtree(src)
 
 
-def check_compare(input_set, *patterns):
-    "Just a quick way to make sure all files are present"
-    disk_files = set()
-    for pattern in patterns:
-        disk_files |= set(glob.glob(pattern, recursive=True))
-
-    assert input_set == disk_files, "{} setup.py only, {} on disk only".format(
-        input_set - disk_files, disk_files - input_set
-    )
-
-
-class InstallHeadersNested(install_headers):
-    def run(self):
-        headers = self.distribution.headers or []
-        for header in headers:
-            # Remove include/*/
-            short_header = header.split("/", 2)[-1]
-
-            dst = os.path.join(self.install_dir, os.path.dirname(short_header))
-            self.mkpath(dst)
-            (out, _) = self.copy_file(header, dst)
-            self.outfiles.append(out)
-
-
-main_headers = {
-    "include/pybind11/attr.h",
-    "include/pybind11/buffer_info.h",
-    "include/pybind11/cast.h",
-    "include/pybind11/chrono.h",
-    "include/pybind11/common.h",
-    "include/pybind11/complex.h",
-    "include/pybind11/eigen.h",
-    "include/pybind11/embed.h",
-    "include/pybind11/eval.h",
-    "include/pybind11/functional.h",
-    "include/pybind11/iostream.h",
-    "include/pybind11/numpy.h",
-    "include/pybind11/operators.h",
-    "include/pybind11/options.h",
-    "include/pybind11/pybind11.h",
-    "include/pybind11/pytypes.h",
-    "include/pybind11/stl.h",
-    "include/pybind11/stl_bind.h",
-}
-
-detail_headers = {
-    "include/pybind11/detail/class.h",
-    "include/pybind11/detail/common.h",
-    "include/pybind11/detail/descr.h",
-    "include/pybind11/detail/init.h",
-    "include/pybind11/detail/internals.h",
-    "include/pybind11/detail/typeid.h",
-}
-
-headers = main_headers | detail_headers
-check_compare(headers, "include/**/*.h")
-
-if use_headers:
-    setup_opts["headers"] = headers
-    setup_opts["cmdclass"] = {"install_headers": InstallHeadersNested}
-
-cmake_files = {
-    "pybind11/share/cmake/pybind11/FindPythonLibsNew.cmake",
-    "pybind11/share/cmake/pybind11/pybind11Common.cmake",
-    "pybind11/share/cmake/pybind11/pybind11Config.cmake",
-    "pybind11/share/cmake/pybind11/pybind11ConfigVersion.cmake",
-    "pybind11/share/cmake/pybind11/pybind11NewTools.cmake",
-    "pybind11/share/cmake/pybind11/pybind11Targets.cmake",
-    "pybind11/share/cmake/pybind11/pybind11Tools.cmake",
-}
-
-
-package_headers = set("pybind11/{}".format(h) for h in headers)
-package_files = package_headers | cmake_files
-
-# Generate the files if they are not generated (will be present in tarball)
-GENERATED = (
-    []
-    if all(os.path.exists(h) for h in package_files)
-    else ["pybind11/include", "pybind11/share"]
-)
-with remove_output(*GENERATED):
+with remove_output("pybind11/include", "pybind11/share"):
     # Generate the files if they are not present.
-    if GENERATED:
-        with TemporaryDirectory() as tmpdir:
-            cmd = ["cmake", "-S", ".", "-B", tmpdir] + [
-                "-DCMAKE_INSTALL_PREFIX=pybind11",
-                "-DBUILD_TESTING=OFF",
-                "-DPYBIND11_NOPYTHON=ON",
-            ]
-            cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr)
-            subprocess.check_call(cmd, **cmake_opts)
-            subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts)
-
-    # Make sure all files are present
-    check_compare(package_files, "pybind11/include/**/*.h", "pybind11/share/**/*.cmake")
-
-    if use_system:
-        setup_opts["data_files"] = [
-            ("share/cmake", cmake_files),
-            ("include/pybind11", main_headers),
-            ("include/pybind11/detail", detail_headers),
+    with TemporaryDirectory() as tmpdir:
+        cmd = ["cmake", "-S", ".", "-B", tmpdir] + [
+            "-DCMAKE_INSTALL_PREFIX=pybind11",
+            "-DBUILD_TESTING=OFF",
+            "-DPYBIND11_NOPYTHON=ON",
         ]
-
-    # Remove the cmake / ninja requirements as now all files are guaranteed to exist
-    if GENERATED:
-        REQUIRES = re.compile(r"requires\s*=.+?\]", re.DOTALL | re.MULTILINE)
-        with monkey_patch_file("pyproject.toml") as txt:
-            with open("pyproject.toml", "w") as f:
-                new_txt = REQUIRES.sub('requires = ["setuptools", "wheel"]', txt)
-                f.write(new_txt)
-
-            setup(**setup_opts)
-    else:
-        setup(**setup_opts)
+        cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr)
+        subprocess.check_call(cmd, **cmake_opts)
+        subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts)
+
+    with monkey_patch_file("pyproject.toml", pyproject_toml):
+        with monkey_patch_file("setup.py", setup_py):
+            with open(setup_py) as f:
+                code = compile(f.read(), setup_py, "exec")
+                exec(code, globals(), {})