Merge pull request #43 from pycompression/release_0.4.3

Release 0.4.3

Author: rhpvorderman

.github/workflows/ci.yml

@@ -62,9 +62,9 @@ jobs:
           - "pypy-3.10"
         os: ["ubuntu-latest"]
         include:
-          - os: "macos-latest"  # For m1 macos
-            python-version: "3.8"
-          - os: "macos-13"  # for x86 macos
+          - os: "macos-14"  # For m1 macos
+            python-version: "3.12"
+          - os: "macos-latest"  # for x86 macos
             python-version: "3.8"
           - os: "windows-latest"
             python-version: "3.8"

CHANGELOG.rst

@@ -7,6 +7,10 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.4.3
+-----------------
++ Fix a bug where files larger than 4GB could not be decompressed.
+
 version 0.4.2
 -----------------
 + Fix a reference counting error that happened on module initialization and

setup.py

@@ -124,7 +124,7 @@ def build_zlib_ng():
 
 setup(
     name="zlib-ng",
-    version="0.4.2",
+    version="0.4.3",
     description="Drop-in replacement for zlib and gzip modules using zlib-ng",
     author="Leiden University Medical Center",
     author_email="r.h.p.vorderman@lumc.nl",  # A placeholder for now

src/zlib_ng/__init__.py

@@ -5,4 +5,4 @@
 # This file is part of python-zlib-ng which is distributed under the
 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2.
 
-__version__ = "0.4.2"
+__version__ = "0.4.3"

src/zlib_ng/zlib_ngmodule.c

@@ -2590,8 +2590,9 @@ GzipReader_read_into_buffer(GzipReader *self, uint8_t *out_buffer, size_t out_bu
                         return -1;
                     }
                     uint32_t length = load_u32_le(current_pos);
-                    current_pos += 4; 
-                    if (length != self->zst.total_out) {
+                    current_pos += 4;
+                    // ISIZE is the length of the original data modulo 2^32
+                    if (length != (0xFFFFFFFFUL & self->zst.total_out)) {
                         Py_BLOCK_THREADS;
                         PyErr_SetString(BadGzipFile, "Incorrect length of data produced");
                         return -1;

tests/test_gzip_ng.py

@@ -292,6 +292,31 @@ def test_decompress_incorrect_length():
     error.match("Incorrect length of data produced")
 
 
+def test_decompress_on_long_input():
+    # Ensure that a compressed payload with length bigger than 2**32 (ISIZE is
+    # overflown) can be decompressed. To avoid writing the whole uncompressed payload
+    # into memory, the test writes the compressed data in chunks. The payload consists
+    # almost exclusively of zeros to achieve an exteremely efficient compression rate,
+    # so that the compressed data also fits in memory.
+
+    buffered_stream = io.BytesIO()
+    n = 20
+    block_size = 2**n
+    iterations = 2**(32 - n)
+    zeros_block = bytes(block_size)
+
+    # To avoid writing the whole compressed data, we will write the compressed data
+    with gzip_ng.open(buffered_stream, "wb") as gz:
+        for _ in range(iterations):
+            gz.write(zeros_block)
+        gz.write(b"\x01" * 123)
+    buffered_stream.seek(0)
+    with gzip_ng.open(buffered_stream, "rb") as gz:
+        for _ in range(iterations):
+            assert zeros_block == gz.read(block_size)
+        assert gz.read() == b"\x01" * 123
+
+
 def test_decompress_incorrect_checksum():
     # Create a wrong checksum by using a non-default seed.
     wrong_checksum = zlib.crc32(DATA, 50)