From fbebdb7bf4e1265f429b1ca7f2c4267d20d37fa4 Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Tue, 16 Jan 2018 10:30:02 +0100 Subject: [PATCH 1/4] bpo-29708: allow to force hash-based pycs using the well-established SOURCE_DATE_EPOCH environment variable to allow for reproducible builds of python packages. See https://reproducible-builds.org/ for why this is good. --- Doc/library/py_compile.rst | 5 ++++- Lib/py_compile.py | 2 ++ Lib/test/test_py_compile.py | 12 ++++++++++++ .../Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index a4f06de597b14e..4e7da91e002bae 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -55,7 +55,8 @@ byte-code cache files in the directory containing the source code. *invalidation_mode* should be a member of the :class:`PycInvalidationMode` enum and controls how the generated ``.pyc`` files are invalidated at - runtime. + runtime. If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, + *invalidation_mode* will be forced to `CHECKED_HASH`. .. versionchanged:: 3.2 Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous @@ -71,6 +72,8 @@ byte-code cache files in the directory containing the source code. .. versionchanged:: 3.7 The *invalidation_mode* parameter was added as specified in :pep:`552`. + If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, this will + be forced to `CHECKED_HASH`. .. class:: PycInvalidationMode diff --git a/Lib/py_compile.py b/Lib/py_compile.py index a0f4defdce68c9..16dc0a011ffab2 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -112,6 +112,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, the resulting file would be regular and thus not the same type of file as it was previously. """ + if os.environ.get('SOURCE_DATE_EPOCH'): + invalidation_mode = PycInvalidationMode.CHECKED_HASH if cfile is None: if optimize >= 0: optimization = optimize if optimize >= 1 else '' diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index bcb686c86b5229..8fc0b3308c91de 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -98,6 +98,18 @@ def test_bad_coding(self): self.assertFalse(os.path.exists( importlib.util.cache_from_source(bad_coding))) + def test_source_date_epoch(self): + testtime = 123456789 + with support.EnvironmentVarGuard() as env: + env["SOURCE_DATE_EPOCH"] = str(testtime) + py_compile.compile(self.source_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + self.assertFalse(os.path.exists(self.cache_path)) + with open(self.pyc_path, 'rb') as fp: + flags = importlib._bootstrap_external._classify_pyc( + fp.read(), 'test', {}) + self.assertEqual(flags, 0b11) + @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O') def test_double_dot_no_clobber(self): # http://bugs.python.org/issue22966 diff --git a/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst new file mode 100644 index 00000000000000..c1f15f64fc340b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst @@ -0,0 +1,2 @@ +If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, +py_compile will always create hash-based .pyc files. From c4b165a4c40c479708e248eed0be7b8b9e57fe7c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 24 Jan 2018 12:41:48 -0800 Subject: [PATCH 2/4] Tweak "version changed" wording [no ci] --- Doc/library/py_compile.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 4e7da91e002bae..78a7260752d54b 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -72,8 +72,8 @@ byte-code cache files in the directory containing the source code. .. versionchanged:: 3.7 The *invalidation_mode* parameter was added as specified in :pep:`552`. - If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, this will - be forced to `CHECKED_HASH`. + If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, + *invalidation_mode* will be forced to `CHECKED_HASH`. .. class:: PycInvalidationMode From 27b2a702b2514189b02801063b3b90862d7e1971 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 24 Jan 2018 12:46:20 -0800 Subject: [PATCH 3/4] Add more reST markup to news entry --- Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst index c1f15f64fc340b..e52c0018e5e10d 100644 --- a/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst +++ b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst @@ -1,2 +1,2 @@ If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, -py_compile will always create hash-based .pyc files. +:mod:`py_compile` will always create hash-based ``.pyc`` files. From 071c943c972c6897af1321fb1c947d75b88dbf46 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 24 Jan 2018 13:01:01 -0800 Subject: [PATCH 4/4] Add Sphinx metadata [no ci] --- Doc/library/py_compile.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 78a7260752d54b..d720e01050577d 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -56,7 +56,8 @@ byte-code cache files in the directory containing the source code. *invalidation_mode* should be a member of the :class:`PycInvalidationMode` enum and controls how the generated ``.pyc`` files are invalidated at runtime. If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, - *invalidation_mode* will be forced to `CHECKED_HASH`. + *invalidation_mode* will be forced to + :attr:`PycInvalidationMode.CHECKED_HASH`. .. versionchanged:: 3.2 Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous @@ -73,7 +74,8 @@ byte-code cache files in the directory containing the source code. .. versionchanged:: 3.7 The *invalidation_mode* parameter was added as specified in :pep:`552`. If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, - *invalidation_mode* will be forced to `CHECKED_HASH`. + *invalidation_mode* will be forced to + :attr:`PycInvalidationMode.CHECKED_HASH`. .. class:: PycInvalidationMode