From 18e3c7c9eafd811f2739bb73f6036f68a4a37cd9 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 22 Dec 2023 11:59:29 +0100 Subject: [PATCH 1/3] Use pythondevmode and libasan to detect resource errors --- tox.ini | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tox.ini b/tox.ini index 5085404..ace675b 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ deps=pytest coverage passenv= PYTHON_ZLIB_NG_LINK_DYNAMIC +setenv= + PYTHONDEVMODE=1 commands = # Create HTML coverage report for humans and xml coverage report for external services. coverage run --branch --source=zlib_ng -m pytest tests @@ -17,6 +19,25 @@ commands = coverage html -i coverage xml -i +[testenv:asan] +setenv= + PYTHONDEVMODE=1 + PYTHONMALLOC=malloc + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer +allowlist_externals=bash +commands= + bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from zlib_ng import zlib_ng" && pytest tests' + +[testenv:compliance] +deps=pytest +commands= + pytest -v tests/test_zlib_compliance.py tests/test_gzip_compliance.py + +[testenv:compatibility] +deps=pytest +commands= + pytest tests/test_isal.py + [testenv:lint] deps=flake8 flake8-import-order @@ -56,21 +77,25 @@ commands= [testenv:benchmark-all] deps= +setenv = commands= python ./benchmark_scripts/benchmark.py --all [testenv:benchmark-functions] deps= +setenv = commands= python ./benchmark_scripts/benchmark.py --functions [testenv:benchmark-gzip] deps= +setenv = commands= python ./benchmark_scripts/benchmark.py --gzip [testenv:benchmark-checksums] deps= +setenv = commands= python ./benchmark_scripts/benchmark.py --checksums From 4da3bc540a13d3ebffc0f98ad0d6626e49127f7e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 22 Dec 2023 12:08:56 +0100 Subject: [PATCH 2/3] Fix resource warning when using the CLI --- src/zlib_ng/gzip_ng.py | 3 ++ tests/test_gzip_ng.py | 66 +++++++++++------------------------------- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index 70eede2..f29c24a 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -431,6 +431,7 @@ def main(): if yes_or_no not in {"y", "Y", "yes"}: sys.exit("not overwritten") + out_buffer = None if args.compress: if args.file is None: in_file = sys.stdin.buffer @@ -470,6 +471,8 @@ def main(): in_file.close() if out_file is not sys.stdout.buffer: out_file.close() + if out_buffer is not None and out_buffer is not sys.stdout.buffer: + out_buffer.close() if __name__ == "__main__": # pragma: no cover diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index 0a2ca86..fa6b68c 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -14,7 +14,6 @@ import os import re import shutil -import subprocess import sys import tempfile import zlib @@ -28,21 +27,6 @@ DATA = b'This is a simple test with gzip_ng' COMPRESSED_DATA = gzip.compress(DATA) TEST_FILE = str((Path(__file__).parent / "data" / "test.fastq.gz")) -PYPY = sys.implementation.name == "pypy" - - -def run_gzip_ng(*args, stdin=None): - """Calling gzip_ng externally seems to solve some issues on PyPy where - files would not be written properly when gzip_ng.main() was called. This is - probably due to some out of order execution that PyPy tries to pull. - Running the process externally is detrimental to the coverage report, - so this is only done for PyPy.""" - process = subprocess.Popen(["python", "-m", "zlib_ng.gzip_ng", *args], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - - return process.communicate(stdin) def test_repr(): @@ -121,12 +105,9 @@ def test_decompress_infile_outfile(tmp_path, capsysbinary): def test_compress_infile_outfile(tmp_path, capsysbinary): test_file = tmp_path / "test" test_file.write_bytes(DATA) - if PYPY: - out, err = run_gzip_ng(str(test_file)) - else: - sys.argv = ['', str(test_file)] - gzip_ng.main() - out, err = capsysbinary.readouterr() + sys.argv = ['', str(test_file)] + gzip_ng.main() + out, err = capsysbinary.readouterr() out_file = test_file.with_suffix(".gz") assert err == b'' assert out == b'' @@ -189,12 +170,9 @@ def test_compress_infile_out_file(tmp_path, capsysbinary): test.write_bytes(DATA) out_file = tmp_path / "compressed.gz" args = ['-o', str(out_file), str(test)] - if PYPY: - out, err = run_gzip_ng(*args) - else: - sys.argv = ['', *args] - gzip_ng.main() - out, err = capsysbinary.readouterr() + sys.argv = ['', *args] + gzip_ng.main() + out, err = capsysbinary.readouterr() assert gzip.decompress(out_file.read_bytes()) == DATA assert err == b'' assert out == b'' @@ -206,12 +184,9 @@ def test_compress_infile_out_file_force(tmp_path, capsysbinary): out_file = tmp_path / "compressed.gz" out_file.touch() args = ['-f', '-o', str(out_file), str(test)] - if PYPY: - out, err = run_gzip_ng(*args) - else: - sys.argv = ['', *args] - gzip_ng.main() - out, err = capsysbinary.readouterr() + sys.argv = ['', *args] + gzip_ng.main() + out, err = capsysbinary.readouterr() assert gzip.decompress(out_file.read_bytes()) == DATA assert err == b'' assert out == b'' @@ -254,14 +229,11 @@ def test_compress_infile_out_file_inmplicit_name_prompt_accept( test.write_bytes(DATA) out_file = tmp_path / "test.gz" out_file.touch() - if PYPY: - out, err = run_gzip_ng(str(test), stdin=b"y\n") - else: - sys.argv = ['', str(test)] - mock_stdin = io.BytesIO(b"y") - sys.stdin = io.TextIOWrapper(mock_stdin) - gzip_ng.main() - out, err = capsysbinary.readouterr() + sys.argv = ['', str(test)] + mock_stdin = io.BytesIO(b"y") + sys.stdin = io.TextIOWrapper(mock_stdin) + gzip_ng.main() + out, err = capsysbinary.readouterr() assert b"already exists; do you wish to overwrite" in out assert err == b"" assert gzip.decompress(out_file.read_bytes()) == DATA @@ -271,13 +243,9 @@ def test_compress_infile_out_file_no_name(tmp_path, capsysbinary): test = tmp_path / "test" test.write_bytes(DATA) out_file = tmp_path / "compressed.gz" - args = ['-n', '-o', str(out_file), str(test)] - if PYPY: - out, err = run_gzip_ng(*args) - else: - sys.argv = ['', '-n', '-o', str(out_file), str(test)] - gzip_ng.main() - out, err = capsysbinary.readouterr() + sys.argv = ['', '-n', '-o', str(out_file), str(test)] + gzip_ng.main() + out, err = capsysbinary.readouterr() output = out_file.read_bytes() assert gzip.decompress(output) == DATA assert err == b'' From 1c0c601f66efffe26395da09cfd45fd7cb394ed3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 22 Dec 2023 12:44:14 +0100 Subject: [PATCH 3/3] Update changelog with buffer errors patch --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a4eaa79..f38c87e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 0.4.0-dev +----------------- ++ Fix some unclosed buffer errors in the gzip_ng CLI. + version 0.3.0 ----------------- + Source distributions on Linux now default to building with configure and