bpo-1294959: Add sys.platlibdir attribute (GH-18381)

vstinner · Jan Matějek · mcepl · web-flow · commit 8510f4307811 · 2020-03-10T09:53:09.000+01:00
Add --with-platlibdir option to the configure script: name of the
platform-specific library directory, stored in the new sys.platlitdir
attribute. It is used to build the path of platform-specific dynamic
libraries and the path of the standard library.

It is equal to "lib" on most platforms. On Fedora and SuSE, it is
equal to "lib64" on 64-bit systems.

Co-Authored-By: Jan Matějek &lt;jmatejek@suse.com&gt;
Co-Authored-By: Matěj Cepl &lt;mcepl@cepl.eu&gt;
Co-Authored-By: Charalampos Stratakis &lt;cstratak@redhat.com&gt;
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
@@ -1138,6 +1138,27 @@ always available.
       system's identity.
 
 
+.. data:: platlibdir
+
+   Name of the platform-specific library directory. It is used to build the
+   path of platform-specific dynamic libraries and the path of the standard
+   library.
+
+   It is equal to ``"lib"`` on most platforms. On Fedora and SuSE, it is equal
+   to ``"lib64"`` on 64-bit platforms which gives the following ``sys.path``
+   paths (where ``X.Y`` is the Python ``major.minor`` version):
+
+   * ``/usr/lib64/pythonX.Y/``:
+     Standard library (like ``os.py`` of the :mod:`os` module)
+   * ``/usr/lib64/pythonX.Y/lib-dynload/``:
+     C extension modules of the standard library (like the :mod:`errno` module,
+     the exact filename is platform specific)
+   * ``/usr/lib/pythonX.Y/site-packages`` (always use ``lib``, not
+     :data:`sys.platlibdir`): Third-party modules
+
+   .. versionadded:: 3.9
+
+
 .. data:: prefix
 
    A string giving the site-specific directory prefix where the platform
diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst
@@ -353,6 +353,17 @@ finalization crashed with a Python fatal error if a daemon thread was still
 running.
 (Contributed by Victor Stinner in :issue:`37266`.)
 
+sys
+---
+
+Add a new :attr:`sys.platlitdir` attribute: name of the platform-specific
+library directory. It is used to build the path of platform-specific dynamic
+libraries and the path of the standard library. It is equal to ``"lib"`` on
+most platforms.  On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit
+platforms.
+(Contributed by Jan Matějek, Matěj Cepl, Charalampos Stratakis and Victor Stinner in :issue:`1294959`.)
+
+
 typing
 ------
 
@@ -390,6 +401,11 @@ Optimizations
 Build and C API Changes
 =======================
 
+* Add ``--with-platlibdir`` option to the ``configure`` script: name of the
+  platform-specific library directory, stored in the new :attr:`sys.platlitdir`
+  attribute. See :attr:`sys.platlibdir` attribute for more information.
+  (Contributed by Jan Matějek, Matěj Cepl, Charalampos Stratakis and Victor Stinner in :issue:`1294959`.)
+
 * Add a new public :c:func:`PyObject_CallNoArgs` function to the C API, which
   calls a callable Python object without any arguments. It is the most efficient
   way to call a callable Python object without any argument.
diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py
@@ -30,14 +30,14 @@
 INSTALL_SCHEMES = {
     'unix_prefix': {
         'purelib': '$base/lib/python$py_version_short/site-packages',
-        'platlib': '$platbase/lib/python$py_version_short/site-packages',
+        'platlib': '$platbase/$platlibdir/python$py_version_short/site-packages',
         'headers': '$base/include/python$py_version_short$abiflags/$dist_name',
         'scripts': '$base/bin',
         'data'   : '$base',
         },
     'unix_home': {
         'purelib': '$base/lib/python',
-        'platlib': '$base/lib/python',
+        'platlib': '$base/$platlibdir/python',
         'headers': '$base/include/python/$dist_name',
         'scripts': '$base/bin',
         'data'   : '$base',
@@ -298,6 +298,7 @@ def finalize_options(self):
                             'sys_exec_prefix': exec_prefix,
                             'exec_prefix': exec_prefix,
                             'abiflags': abiflags,
+                            'platlibdir': sys.platlibdir,
                            }
 
         if HAS_USER_SITE:
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
@@ -146,8 +146,15 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             prefix = plat_specific and EXEC_PREFIX or PREFIX
 
     if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
+        if plat_specific or standard_lib:
+            # Platform-specific modules (any module from a non-pure-Python
+            # module distribution) or standard Python library modules.
+            libdir = sys.platlibdir
+        else:
+            # Pure Python
+            libdir = "lib"
+        libpython = os.path.join(prefix, libdir,
+                                 "python" + get_python_version())
         if standard_lib:
             return libpython
         else:
diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py
@@ -58,7 +58,8 @@ def check_path(got, expected):
 
         libdir = os.path.join(destination, "lib", "python")
         check_path(cmd.install_lib, libdir)
-        check_path(cmd.install_platlib, libdir)
+        platlibdir = os.path.join(destination, sys.platlibdir, "python")
+        check_path(cmd.install_platlib, platlibdir)
         check_path(cmd.install_purelib, libdir)
         check_path(cmd.install_headers,
                    os.path.join(destination, "include", "python", "foopkg"))
diff --git a/Lib/site.py b/Lib/site.py
@@ -334,13 +334,22 @@ def getsitepackages(prefixes=None):
             continue
         seen.add(prefix)
 
+        libdirs = [sys.platlibdir]
+        if sys.platlibdir != "lib":
+            libdirs.append("lib")
+
         if os.sep == '/':
-            sitepackages.append(os.path.join(prefix, "lib",
-                                        "python%d.%d" % sys.version_info[:2],
-                                        "site-packages"))
+            for libdir in libdirs:
+                path = os.path.join(prefix, libdir,
+                                    "python%d.%d" % sys.version_info[:2],
+                                    "site-packages")
+                sitepackages.append(path)
         else:
             sitepackages.append(prefix)
-            sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
+
+            for libdir in libdirs:
+                path = os.path.join(prefix, libdir, "site-packages")
+                sitepackages.append(path)
     return sitepackages
 
 def addsitepackages(known_paths, prefixes=None):
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
@@ -20,10 +20,10 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
+        'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}',
+        'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}',
         'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages',
         'include':
             '{installed_base}/include/python{py_version_short}{abiflags}',
         'platinclude':
@@ -62,10 +62,10 @@
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
+        'stdlib': '{userbase}/{platlibdir}/python{py_version_short}',
+        'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}',
         'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
+        'platlib': '{userbase}/{platlibdir}/python{py_version_short}/site-packages',
         'include': '{userbase}/include/python{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -539,6 +539,7 @@ def get_config_vars(*args):
         _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX
         _CONFIG_VARS['platbase'] = _EXEC_PREFIX
         _CONFIG_VARS['projectbase'] = _PROJECT_BASE
+        _CONFIG_VARS['platlibdir'] = sys.platlibdir
         try:
             _CONFIG_VARS['abiflags'] = sys.abiflags
         except AttributeError:
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -1068,11 +1068,11 @@ def module_search_paths(self, prefix=None, exec_prefix=None):
         else:
             ver = sys.version_info
             return [
-                os.path.join(prefix, 'lib',
+                os.path.join(prefix, sys.platlibdir,
                              f'python{ver.major}{ver.minor}.zip'),
-                os.path.join(prefix, 'lib',
+                os.path.join(prefix, sys.platlibdir,
                              f'python{ver.major}.{ver.minor}'),
-                os.path.join(exec_prefix, 'lib',
+                os.path.join(exec_prefix, sys.platlibdir,
                              f'python{ver.major}.{ver.minor}', 'lib-dynload'),
             ]
 
@@ -1183,7 +1183,7 @@ def test_init_pyvenv_cfg(self):
 
             if not MS_WINDOWS:
                 lib_dynload = os.path.join(pyvenv_home,
-                                           'lib',
+                                           sys.platlibdir,
                                            f'python{ver.major}.{ver.minor}',
                                            'lib-dynload')
                 os.makedirs(lib_dynload)
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
@@ -266,11 +266,18 @@ def test_getsitepackages(self):
         dirs = site.getsitepackages()
         if os.sep == '/':
             # OS X, Linux, FreeBSD, etc
-            self.assertEqual(len(dirs), 1)
+            if sys.platlibdir != "lib":
+                self.assertEqual(len(dirs), 2)
+                wanted = os.path.join('xoxo', sys.platlibdir,
+                                      'python%d.%d' % sys.version_info[:2],
+                                      'site-packages')
+                self.assertEqual(dirs[0], wanted)
+            else:
+                self.assertEqual(len(dirs), 1)
             wanted = os.path.join('xoxo', 'lib',
                                   'python%d.%d' % sys.version_info[:2],
                                   'site-packages')
-            self.assertEqual(dirs[0], wanted)
+            self.assertEqual(dirs[-1], wanted)
         else:
             # other platforms
             self.assertEqual(len(dirs), 2)
diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -143,7 +143,8 @@ LIBDIR=		@libdir@
 MANDIR=		@mandir@
 INCLUDEDIR=	@includedir@
 CONFINCLUDEDIR=	$(exec_prefix)/include
-SCRIPTDIR=	$(prefix)/lib
+PLATLIBDIR=	@PLATLIBDIR@
+SCRIPTDIR=	$(prefix)/$(PLATLIBDIR)
 ABIFLAGS=	@ABIFLAGS@
 
 # Detailed destination directories
@@ -754,6 +755,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
 		-DEXEC_PREFIX='"$(exec_prefix)"' \
 		-DVERSION='"$(VERSION)"' \
 		-DVPATH='"$(VPATH)"' \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
 Programs/python.o: $(srcdir)/Programs/python.c
@@ -785,6 +787,7 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile
 Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h
 	$(CC) -c $(PY_CORE_CFLAGS) \
 		-DABIFLAGS='"$(ABIFLAGS)"' \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
 		$(MULTIARCH_CPPFLAGS) \
 		-o $@ $(srcdir)/Python/sysmodule.c
 
diff --git a/Misc/NEWS.d/next/Build/2020-02-06-18-08-25.bpo-1294959.AZPg4R.rst b/Misc/NEWS.d/next/Build/2020-02-06-18-08-25.bpo-1294959.AZPg4R.rst
@@ -0,0 +1,6 @@
+Add ``--with-platlibdir`` option to the configure script: name of the
+platform-specific library directory, stored in the new :attr:`sys.platlitdir`
+attribute. It is used to build the path of platform-specific dynamic libraries
+and the path of the standard library. It is equal to ``"lib"`` on most
+platforms. On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit platforms.
+Patch by Jan Matějek, Matěj Cepl, Charalampos Stratakis and Victor Stinner.
diff --git a/Modules/getpath.c b/Modules/getpath.c
@@ -105,8 +105,9 @@ extern "C" {
 #endif
 
 
-#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH)
-#error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined"
+#if (!defined(PREFIX) || !defined(EXEC_PREFIX) \
+        || !defined(VERSION) || !defined(VPATH) || !defined(PLATLIBDIR))
+#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined"
 #endif
 
 #ifndef LANDMARK
@@ -128,6 +129,7 @@ typedef struct {
     wchar_t *pythonpath_macro;         /* PYTHONPATH macro */
     wchar_t *prefix_macro;             /* PREFIX macro */
     wchar_t *exec_prefix_macro;        /* EXEC_PREFIX macro */
+    wchar_t *platlibdir_macro;         /* PLATLIBDIR macro */
     wchar_t *vpath_macro;              /* VPATH macro */
 
     wchar_t *lib_python;               /* "lib/pythonX.Y" */
@@ -809,8 +811,17 @@ calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
                 "Could not find platform dependent libraries <exec_prefix>\n");
         }
 
+        /* <PLATLIBDIR> / "lib-dynload" */
+        wchar_t *lib_dynload = joinpath2(calculate->platlibdir_macro,
+                                         L"lib-dynload");
+        if (lib_dynload == NULL) {
+            return _PyStatus_NO_MEMORY();
+        }
+
         calculate->exec_prefix = joinpath2(calculate->exec_prefix_macro,
-                                           L"lib/lib-dynload");
+                                           lib_dynload);
+        PyMem_RawFree(lib_dynload);
+
         if (calculate->exec_prefix == NULL) {
             return _PyStatus_NO_MEMORY();
         }
@@ -1284,35 +1295,47 @@ calculate_read_pyenv(PyCalculatePath *calculate)
 static PyStatus
 calculate_zip_path(PyCalculatePath *calculate)
 {
-    const wchar_t *lib_python = L"lib/python00.zip";
+    PyStatus res;
+
+    /* Path: <PLATLIBDIR> / "python00.zip" */
+    wchar_t *path = joinpath2(calculate->platlibdir_macro, L"python00.zip");
+    if (path == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
 
     if (calculate->prefix_found > 0) {
         /* Use the reduced prefix returned by Py_GetPrefix()
 
-           Path: <basename(basename(prefix))> / <lib_python> */
+           Path: <basename(basename(prefix))> / <PLATLIBDIR> / "python00.zip" */
         wchar_t *parent = _PyMem_RawWcsdup(calculate->prefix);
         if (parent == NULL) {
-            return _PyStatus_NO_MEMORY();
+            res = _PyStatus_NO_MEMORY();
+            goto done;
         }
         reduce(parent);
         reduce(parent);
-        calculate->zip_path = joinpath2(parent, lib_python);
+        calculate->zip_path = joinpath2(parent, path);
         PyMem_RawFree(parent);
     }
     else {
-        calculate->zip_path = joinpath2(calculate->prefix_macro, lib_python);
+        calculate->zip_path = joinpath2(calculate->prefix_macro, path);
     }
 
     if (calculate->zip_path == NULL) {
-        return _PyStatus_NO_MEMORY();
+        res = _PyStatus_NO_MEMORY();
+        goto done;
     }
 
     /* Replace "00" with version */
     size_t len = wcslen(calculate->zip_path);
     calculate->zip_path[len - 6] = VERSION[0];
     calculate->zip_path[len - 5] = VERSION[2];
 
-    return _PyStatus_OK();
+    res = _PyStatus_OK();
+
+done:
+    PyMem_RawFree(path);
+    return res;
 }
 
 
@@ -1434,10 +1457,14 @@ calculate_init(PyCalculatePath *calculate, const PyConfig *config)
     if (!calculate->vpath_macro) {
         return DECODE_LOCALE_ERR("VPATH macro", len);
     }
+    calculate->platlibdir_macro = Py_DecodeLocale(PLATLIBDIR, &len);
+    if (!calculate->platlibdir_macro) {
+        return DECODE_LOCALE_ERR("PLATLIBDIR macro", len);
+    }
 
-    calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);
+    calculate->lib_python = Py_DecodeLocale(PLATLIBDIR "/python" VERSION, &len);
     if (!calculate->lib_python) {
-        return DECODE_LOCALE_ERR("EXEC_PREFIX macro", len);
+        return DECODE_LOCALE_ERR("VERSION macro", len);
     }
 
     calculate->warnings = config->pathconfig_warnings;
@@ -1454,6 +1481,7 @@ calculate_free(PyCalculatePath *calculate)
     PyMem_RawFree(calculate->prefix_macro);
     PyMem_RawFree(calculate->exec_prefix_macro);
     PyMem_RawFree(calculate->vpath_macro);
+    PyMem_RawFree(calculate->platlibdir_macro);
     PyMem_RawFree(calculate->lib_python);
     PyMem_RawFree(calculate->path_env);
     PyMem_RawFree(calculate->zip_path);
diff --git a/PC/pyconfig.h b/PC/pyconfig.h
@@ -683,4 +683,6 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
 /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */
 #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1
 
+#define PLATLIBDIR "lib"
+
 #endif /* !Py_CONFIG_H */
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
diff --git a/configure b/configure
diff --git a/configure.ac b/configure.ac
