Merge pull request #12 from pycompression/release_0.2.0

Release 0.2.0

Author: rhpvorderman

.readthedocs.yml

@@ -1,6 +1,12 @@
 version: 2
 formats: []  # Do not build epub and pdf
 
+# This adds mambaforge for the installation
+build:
+  os: "ubuntu-20.04"
+  tools:
+    python: "mambaforge-4.10"
+
 python:
   install:
     - method: pip

CHANGELOG.rst

@@ -7,6 +7,11 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.2.0
+-----------------
++ Update embedded zlib-ng version to 2.0.7
++ Escape GIL for adler32 and crc32 functions.
+
 version 0.1.0
 -----------------
 + Build wheels for all three major operating systems.

README.rst

@@ -120,13 +120,8 @@ Differences with zlib and gzip modules
 
 .. differences start
 
-+ Compression level 1 zlib_ng has a much worse compression rate than that in
++ Compression level 1 zlib_ng has a worse compression rate than that in
   zlib. For other compression levels zlib_ng compresses better.
-+ Compression level 1 does not apply requested ``wbits`` correctly. For example
-  compressing with ``zlib_ng.compress(data, level=1, wbits=-9)`` results in
-  data that cannot be decompressed with ``zlib_ng.decompress(data, wbits=-9)``
-  as this will throw an error mentioning invalid window sizes. This is a
-  bug in the included zlib-ng 2.0.6.
 + ``gzip_ng.open`` returns a class ``GzipNGFile`` instead of ``GzipFile``. Since
   there are differences between the compressed ratios between levels, a
   difference in naming was chosen to reflect this.

docs/conda-environment.yml

@@ -3,10 +3,11 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - zlib-ng
+  - zlib-ng >=2.0.7
   - python >=3.7
   - sphinx
   - setuptools
+  - pip
   - pip:
     - sphinx-rtd-theme
     - sphinx-argparse

setup.py

@@ -121,7 +121,7 @@ def build_zlib_ng():
 
 setup(
     name="zlib-ng",
-    version="0.1.0",
+    version="0.2.0",
     description="Drop-in replacement for zlib and gzip modules using zlib-ng",
     author="Leiden University Medical Center",
     author_email="r.h.p.vorderman@lumc.nl",  # A placeholder for now

src/zlib_ng/__init__.py

@@ -5,4 +5,4 @@
 # This file is part of python-zlib-ng which is distributed under the
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

src/zlib_ng/zlib-ng

@@ -9,8 +9,8 @@
 #include "stdbool.h"
 #include "stdint.h"
 
-#if defined(ZLIBNG_VERNUM) && ZLIBNG_VERNUM < 0x02060
-#error "At least zlib-ng version 2.0.6 is required"
+#if defined(ZLIBNG_VERNUM) && ZLIBNG_VERNUM < 0x02070
+#error "At least zlib-ng version 2.0.7 is required"
 #endif
 
 #define ENTER_ZLIB(obj) do {                      \
@@ -1466,12 +1466,21 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 
     Py_ssize_t len = data.len ;
     uint8_t *buf = data.buf;
-    while ((size_t)len > UINT32_MAX) {
-        value = zng_adler32(value, buf, UINT32_MAX);
-        buf += (size_t) UINT32_MAX;
-        len -= (size_t) UINT32_MAX;
+
+    /* Do not drop GIL for small values as it increases overhead */
+    if (len > 1024 * 5) {
+        Py_BEGIN_ALLOW_THREADS
+        while ((size_t)len > UINT32_MAX) {
+            value = zng_adler32(value, buf, UINT32_MAX);
+            buf += (size_t) UINT32_MAX;
+            len -= (size_t) UINT32_MAX;
+        }
+        value = zng_adler32(value, buf, (uint32_t)len);
+        Py_END_ALLOW_THREADS
+    } else {
+        value = zng_adler32(value, buf, (uint32_t)len);
     }
-    value = zng_adler32(value, buf, (uint32_t)len);
+
     return_value = PyLong_FromUnsignedLong(value & 0xffffffffU);
     PyBuffer_Release(&data);
     return return_value;
@@ -1519,12 +1528,21 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 
     Py_ssize_t len = data.len ;
     uint8_t *buf = data.buf;
-    while ((size_t)len > UINT32_MAX) {
-        value = zng_crc32(value, buf, UINT32_MAX);
-        buf += (size_t) UINT32_MAX;
-        len -= (size_t) UINT32_MAX;
+
+    /* Do not drop GIL for small values as it increases overhead */
+    if (len > 1024 * 5) {
+        Py_BEGIN_ALLOW_THREADS
+        while ((size_t)len > UINT32_MAX) {
+            value = zng_crc32(value, buf, UINT32_MAX);
+            buf += (size_t) UINT32_MAX;
+            len -= (size_t) UINT32_MAX;
+        }
+        value = zng_crc32(value, buf, (uint32_t)len);
+        Py_END_ALLOW_THREADS
+    } else {
+        value = zng_crc32(value, buf, (uint32_t)len);
     }
-    value = zng_crc32(value, buf, (uint32_t)len);
+
     return_value = PyLong_FromUnsignedLong(value & 0xffffffffU);
     PyBuffer_Release(&data);
     return return_value;

src/zlib_ng/zlib_ngmodule.c

@@ -82,14 +82,7 @@ def test_adler32(data_size, value):
 def test_compress(data_size, level, wbits):
     data = DATA[:data_size]
     compressed = zlib_ng.compress(data, level=level, wbits=wbits)
-    try:
-        decompressed = zlib.decompress(compressed, wbits)
-    except zlib.error:
-        # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1.
-        if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and
-                level == 1 and
-                wbits & 0b1111 < 13):
-            pytest.xfail()
+    decompressed = zlib.decompress(compressed, wbits)
     assert decompressed == data
 
 
@@ -118,15 +111,7 @@ def test_decompress_wbits(data_size, level, wbits, memLevel, strategy):
 def test_decompress_zlib_ng(data_size, level, wbits):
     data = DATA[:data_size]
     compressed = zlib_ng.compress(data, level=level, wbits=wbits)
-    try:
-        decompressed = zlib_ng.decompress(compressed, wbits=wbits)
-    except zlib_ng.error:
-        # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1.
-        if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and
-                level == 1 and
-                wbits & 0b1111 < 13):
-            pytest.xfail()
-    assert decompressed == data
+    decompressed = zlib_ng.decompress(compressed, wbits=wbits)
     assert decompressed == data
 
 
@@ -139,14 +124,7 @@ def test_compress_compressobj(data_size, level, wbits, memLevel, strategy):
                                       memLevel=memLevel,
                                       strategy=strategy)
     compressed = compressobj.compress(data) + compressobj.flush()
-    try:
-        decompressed = zlib.decompress(compressed, wbits=wbits)
-    except zlib.error:
-        # Known bug in zlib-ng 2.0.6. Wbits is not correctly applied for level 1.
-        if (zlib_ng.ZLIBNG_VERSION == "2.0.6" and
-                level == 1 and
-                wbits & 0b1111 < 13):
-            pytest.xfail()
+    decompressed = zlib.decompress(compressed, wbits=wbits)
     assert data == decompressed
 
 

tests/test_compat.py

@@ -835,10 +835,6 @@ def test_large_unconsumed_tail(self, size):
         finally:
             comp = uncomp = data = None
 
-    # TODO: zlib-ng does not handle wbits for the zlib header correctly.
-    # TODO: latest zlib-ng works correctly. Should be fixed when new release
-    # TODO: of zlib-ng comes.
-    @unittest.expectedFailure
     def test_wbits(self):
         # wbits=0 only supported since zlib v1.2.3.5
         # Register "1.2.3" as "1.2.3.0"