Parallel Cythonization for spawned Processes

setup.py

@@ -9,6 +9,7 @@
 import argparse
 from distutils.sysconfig import get_config_vars
 from distutils.version import LooseVersion
+import multiprocessing
 import os
 from os.path import join as pjoin
 import platform
@@ -35,17 +36,6 @@ def is_platform_mac():
 min_numpy_ver = "1.13.3"
 min_cython_ver = "0.29.13"  # note: sync with pyproject.toml
 
-setuptools_kwargs = {
-    "install_requires": [
-        "python-dateutil >= 2.6.1",
-        "pytz >= 2017.2",
-        f"numpy >= {min_numpy_ver}",
-    ],
-    "setup_requires": [f"numpy >= {min_numpy_ver}"],
-    "zip_safe": False,
-}
-
-
 try:
     import Cython
 
@@ -531,11 +521,6 @@ def maybe_cythonize(extensions, *args, **kwargs):
     elif parsed.j:
         nthreads = parsed.j
 
-    # GH#30356 Cythonize doesn't support parallel on Windows
-    if is_platform_windows() and nthreads > 0:
-        print("Parallel build for cythonize ignored on Windows")
-        nthreads = 0
-
     kwargs["nthreads"] = nthreads
     build_ext.render_templates(_pxifiles)
     return cythonize(extensions, *args, **kwargs)
@@ -744,37 +729,51 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
 # ----------------------------------------------------------------------
 
 
-# The build cache system does string matching below this point.
-# if you change something, be careful.
-
-setup(
-    name=DISTNAME,
-    maintainer=AUTHOR,
-    version=versioneer.get_version(),
-    packages=find_packages(include=["pandas", "pandas.*"]),
-    package_data={"": ["templates/*", "_libs/*.dll"]},
-    ext_modules=maybe_cythonize(extensions, compiler_directives=directives),
-    maintainer_email=EMAIL,
-    description=DESCRIPTION,
-    license=LICENSE,
-    cmdclass=cmdclass,
-    url=URL,
-    download_url=DOWNLOAD_URL,
-    project_urls=PROJECT_URLS,
-    long_description=LONG_DESCRIPTION,
-    classifiers=CLASSIFIERS,
-    platforms="any",
-    python_requires=">=3.6.1",
-    extras_require={
-        "test": [
-            # sync with setup.cfg minversion & install.rst
-            "pytest>=4.0.2",
-            "pytest-xdist",
-            "hypothesis>=3.58",
-        ]
-    },
-    entry_points={
-        "pandas_plotting_backends": ["matplotlib = pandas:plotting._matplotlib"]
-    },
-    **setuptools_kwargs,
-)
+def setup_package():
+    setuptools_kwargs = {
+        "install_requires": [
+            "python-dateutil >= 2.6.1",
+            "pytz >= 2017.2",
+            f"numpy >= {min_numpy_ver}",
+        ],
+        "setup_requires": [f"numpy >= {min_numpy_ver}"],
+        "zip_safe": False,
+    }
+
+    setup(
+        name=DISTNAME,
+        maintainer=AUTHOR,
+        version=versioneer.get_version(),
+        packages=find_packages(include=["pandas", "pandas.*"]),
+        package_data={"": ["templates/*", "_libs/*.dll"]},
+        ext_modules=maybe_cythonize(extensions, compiler_directives=directives),
+        maintainer_email=EMAIL,
+        description=DESCRIPTION,
+        license=LICENSE,
+        cmdclass=cmdclass,
+        url=URL,
+        download_url=DOWNLOAD_URL,
+        project_urls=PROJECT_URLS,
+        long_description=LONG_DESCRIPTION,
+        classifiers=CLASSIFIERS,
+        platforms="any",
+        python_requires=">=3.6.1",
+        extras_require={
+            "test": [
+                # sync with setup.cfg minversion & install.rst
+                "pytest>=4.0.2",
+                "pytest-xdist",
+                "hypothesis>=3.58",
+            ]
+        },
+        entry_points={
+            "pandas_plotting_backends": ["matplotlib = pandas:plotting._matplotlib"]
+        },
+        **setuptools_kwargs,
+    )
+
+
+if __name__ == "__main__":
+    # Freeze to support parallel compilation when using spawn instead of fork
+    multiprocessing.freeze_support()
+    setup_package()