From d7865b55185fc11bdffbcf0e88261f4c80790999 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 31 Jan 2023 07:33:05 +0100 Subject: [PATCH 01/10] Increase version number --- setup.py | 2 +- src/zlib_ng/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 177298e..df5d966 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.1.0", + version="0.2.0-dev", description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index 80b634d..847a3d8 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -5,4 +5,4 @@ # This file is part of python-zlib-ng which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__ = "0.1.0" +__version__ = "0.2.0-dev" From e3df5a940b4bddf705bc99190b290a91b79871f2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 10 Feb 2023 14:01:58 +0100 Subject: [PATCH 02/10] Escape GIL for adler32 and crc32 functions --- CHANGELOG.rst | 4 ++++ src/zlib_ng/zlib_ngmodule.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd4ffee..31ab2ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 0.2.0-dev +----------------- ++ Escape GIL for adler32 and crc32 functions. + version 0.1.0 ----------------- + Build wheels for all three major operating systems. diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index 5bb6f95..4dd8521 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -1466,12 +1466,14 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t len = data.len ; uint8_t *buf = data.buf; + Py_BEGIN_ALLOW_THREADS while ((size_t)len > UINT32_MAX) { value = zng_adler32(value, buf, UINT32_MAX); buf += (size_t) UINT32_MAX; len -= (size_t) UINT32_MAX; } value = zng_adler32(value, buf, (uint32_t)len); + Py_END_ALLOW_THREADS return_value = PyLong_FromUnsignedLong(value & 0xffffffffU); PyBuffer_Release(&data); return return_value; @@ -1519,12 +1521,14 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t len = data.len ; uint8_t *buf = data.buf; + Py_BEGIN_ALLOW_THREADS while ((size_t)len > UINT32_MAX) { value = zng_crc32(value, buf, UINT32_MAX); buf += (size_t) UINT32_MAX; len -= (size_t) UINT32_MAX; } value = zng_crc32(value, buf, (uint32_t)len); + Py_END_ALLOW_THREADS return_value = PyLong_FromUnsignedLong(value & 0xffffffffU); PyBuffer_Release(&data); return return_value; From 128619b51f11b54ccf68c71111329e444b3bb587 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 10 Feb 2023 16:10:02 +0100 Subject: [PATCH 03/10] Do not release GIL on small values to prevent overhead costs --- src/zlib_ng/zlib_ngmodule.c | 46 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index 4dd8521..3e5f7ae 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -1466,14 +1466,21 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t len = data.len ; uint8_t *buf = data.buf; - Py_BEGIN_ALLOW_THREADS - while ((size_t)len > UINT32_MAX) { - value = zng_adler32(value, buf, UINT32_MAX); - buf += (size_t) UINT32_MAX; - len -= (size_t) UINT32_MAX; - } - value = zng_adler32(value, buf, (uint32_t)len); - Py_END_ALLOW_THREADS + + /* Do not drop GIL for small values as it increases overhead */ + if (len > 1024 * 5) { + Py_BEGIN_ALLOW_THREADS + while ((size_t)len > UINT32_MAX) { + value = zng_adler32(value, buf, UINT32_MAX); + buf += (size_t) UINT32_MAX; + len -= (size_t) UINT32_MAX; + } + value = zng_adler32(value, buf, (uint32_t)len); + Py_END_ALLOW_THREADS + } else { + value = zng_adler32(value, buf, (uint32_t)len); + } + return_value = PyLong_FromUnsignedLong(value & 0xffffffffU); PyBuffer_Release(&data); return return_value; @@ -1521,14 +1528,21 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t len = data.len ; uint8_t *buf = data.buf; - Py_BEGIN_ALLOW_THREADS - while ((size_t)len > UINT32_MAX) { - value = zng_crc32(value, buf, UINT32_MAX); - buf += (size_t) UINT32_MAX; - len -= (size_t) UINT32_MAX; - } - value = zng_crc32(value, buf, (uint32_t)len); - Py_END_ALLOW_THREADS + + /* Do not drop GIL for small values as it increases overhead */ + if (len > 1024 * 5) { + Py_BEGIN_ALLOW_THREADS + while ((size_t)len > UINT32_MAX) { + value = zng_crc32(value, buf, UINT32_MAX); + buf += (size_t) UINT32_MAX; + len -= (size_t) UINT32_MAX; + } + value = zng_crc32(value, buf, (uint32_t)len); + Py_END_ALLOW_THREADS + } else { + value = zng_crc32(value, buf, (uint32_t)len); + } + return_value = PyLong_FromUnsignedLong(value & 0xffffffffU); PyBuffer_Release(&data); return return_value; From edd8fcda16014e3e3ba49d2b9103eed192292138 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 13 Feb 2023 08:53:14 +0100 Subject: [PATCH 04/10] Update embedded zlib-ng version to 2.0.7 --- src/zlib_ng/zlib-ng | 2 +- tests/test_compat.py | 28 +++------------------------- tests/test_zlib_compliance.py | 4 ---- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/src/zlib_ng/zlib-ng b/src/zlib_ng/zlib-ng index b56a2fd..a583e21 160000 --- a/src/zlib_ng/zlib-ng +++ b/src/zlib_ng/zlib-ng @@ -1 +1 @@ -Subproject commit b56a2fd0b126cfe5f13e68ab9090cd4f6a773286 +Subproject commit a583e215afa2356e23b418efa871a1cc4348702a diff --git a/tests/test_compat.py b/tests/test_compat.py index 7595cbe..12dda97 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -82,14 +82,7 @@ def test_adler32(data_size, value): def test_compress(data_size, level, wbits): data = DATA[:data_size] compressed = zlib_ng.compress(data, level=level, wbits=wbits) - try: - decompressed = zlib.decompress(compressed, wbits) - except zlib.error: - # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1. - if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and - level == 1 and - wbits & 0b1111 < 13): - pytest.xfail() + decompressed = zlib.decompress(compressed, wbits) assert decompressed == data @@ -118,15 +111,7 @@ def test_decompress_wbits(data_size, level, wbits, memLevel, strategy): def test_decompress_zlib_ng(data_size, level, wbits): data = DATA[:data_size] compressed = zlib_ng.compress(data, level=level, wbits=wbits) - try: - decompressed = zlib_ng.decompress(compressed, wbits=wbits) - except zlib_ng.error: - # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1. - if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and - level == 1 and - wbits & 0b1111 < 13): - pytest.xfail() - assert decompressed == data + decompressed = zlib_ng.decompress(compressed, wbits=wbits) assert decompressed == data @@ -139,14 +124,7 @@ def test_compress_compressobj(data_size, level, wbits, memLevel, strategy): memLevel=memLevel, strategy=strategy) compressed = compressobj.compress(data) + compressobj.flush() - try: - decompressed = zlib.decompress(compressed, wbits=wbits) - except zlib.error: - # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1. - if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and - level == 1 and - wbits & 0b1111 < 13): - pytest.xfail() + decompressed = zlib.decompress(compressed, wbits=wbits) assert data == decompressed diff --git a/tests/test_zlib_compliance.py b/tests/test_zlib_compliance.py index 3060128..1f8ff53 100644 --- a/tests/test_zlib_compliance.py +++ b/tests/test_zlib_compliance.py @@ -835,10 +835,6 @@ def test_large_unconsumed_tail(self, size): finally: comp = uncomp = data = None - # TODO: zlib-ng does not handle wbits for the zlib header correctly. - # TODO: latest zlib-ng works correctly. Should be fixed when new release - # TODO: of zlib-ng comes. - @unittest.expectedFailure def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 # Register "1.2.3" as "1.2.3.0" From f612ebeb7815fe16193e3391543954b55d4c50d0 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 20 Mar 2023 06:55:27 +0100 Subject: [PATCH 05/10] Update changelog with zlib-ng update --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 31ab2ef..a4ade92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changelog version 0.2.0-dev ----------------- ++ Update embedded zlib-ng version to 2.0.7 + Escape GIL for adler32 and crc32 functions. version 0.1.0 From 281f7ad02ee573cf1dc6683cab1ce757c1d44b5a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 20 Mar 2023 06:57:51 +0100 Subject: [PATCH 06/10] Require at least 2.0.7 version in source --- README.rst | 7 +------ src/zlib_ng/zlib_ngmodule.c | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index e1a551c..eef79dc 100644 --- a/README.rst +++ b/README.rst @@ -120,13 +120,8 @@ Differences with zlib and gzip modules .. differences start -+ Compression level 1 zlib_ng has a much worse compression rate than that in ++ Compression level 1 zlib_ng has a worse compression rate than that in zlib. For other compression levels zlib_ng compresses better. -+ Compression level 1 does not apply requested ``wbits`` correctly. For example - compressing with ``zlib_ng.compress(data, level=1, wbits=-9)`` results in - data that cannot be decompressed with ``zlib_ng.decompress(data, wbits=-9)`` - as this will throw an error mentioning invalid window sizes. This is a - bug in the included zlib-ng 2.0.6. + ``gzip_ng.open`` returns a class ``GzipNGFile`` instead of ``GzipFile``. Since there are differences between the compressed ratios between levels, a difference in naming was chosen to reflect this. diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index 3e5f7ae..5ecedb3 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -9,8 +9,8 @@ #include "stdbool.h" #include "stdint.h" -#if defined(ZLIBNG_VERNUM) && ZLIBNG_VERNUM < 0x02060 -#error "At least zlib-ng version 2.0.6 is required" +#if defined(ZLIBNG_VERNUM) && ZLIBNG_VERNUM < 0x02070 +#error "At least zlib-ng version 2.0.7 is required" #endif #define ENTER_ZLIB(obj) do { \ From 30b355547932807790997655cecfb5c0eb14f1f2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 21 Mar 2023 06:43:54 +0100 Subject: [PATCH 07/10] Update conda environment --- docs/conda-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conda-environment.yml b/docs/conda-environment.yml index e41f810..44ae3da 100644 --- a/docs/conda-environment.yml +++ b/docs/conda-environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - zlib-ng + - zlib-ng >=2.0.7 - python >=3.7 - sphinx - setuptools From 9724a924eecb85d2b2dbd90f5664f0abcf1c39ad Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 21 Mar 2023 06:49:38 +0100 Subject: [PATCH 08/10] Add pip as a dependency --- docs/conda-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conda-environment.yml b/docs/conda-environment.yml index 44ae3da..039ff60 100644 --- a/docs/conda-environment.yml +++ b/docs/conda-environment.yml @@ -7,6 +7,7 @@ dependencies: - python >=3.7 - sphinx - setuptools + - pip - pip: - sphinx-rtd-theme - sphinx-argparse \ No newline at end of file From 8c9eb6d3d79ce7d7274d648db6bad5f9ef12a54d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 21 Mar 2023 06:59:42 +0100 Subject: [PATCH 09/10] Use mamba on readthedocs --- .readthedocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2866eda..20eb6cd 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,6 +1,12 @@ version: 2 formats: [] # Do not build epub and pdf +# This adds mambaforge for the installation +build: + os: "ubuntu-20.04" + tools: + python: "mambaforge-4.10" + python: install: - method: pip From 11d65b6bf34736a69f815fd2324aa81b16b0e867 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 21 Mar 2023 07:03:45 +0100 Subject: [PATCH 10/10] Set stable version --- CHANGELOG.rst | 2 +- setup.py | 2 +- src/zlib_ng/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a4ade92..f726b1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 0.2.0-dev +version 0.2.0 ----------------- + Update embedded zlib-ng version to 2.0.7 + Escape GIL for adler32 and crc32 functions. diff --git a/setup.py b/setup.py index df5d966..3049718 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ def build_zlib_ng(): setup( name="zlib-ng", - version="0.2.0-dev", + version="0.2.0", description="Drop-in replacement for zlib and gzip modules using zlib-ng", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now diff --git a/src/zlib_ng/__init__.py b/src/zlib_ng/__init__.py index 847a3d8..6f754a5 100644 --- a/src/zlib_ng/__init__.py +++ b/src/zlib_ng/__init__.py @@ -5,4 +5,4 @@ # This file is part of python-zlib-ng which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. -__version__ = "0.2.0-dev" +__version__ = "0.2.0"