Merge pull request #25 from pycompression/asan

Add address sanitizer tests.

Author: rhpvorderman

CHANGELOG.rst

@@ -7,6 +7,10 @@ Changelog
 .. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 0.4.0-dev
+-----------------
++ Fix some unclosed buffer errors in the gzip_ng CLI.
+
 version 0.3.0
 -----------------
 + Source distributions on Linux now default to building with configure and

src/zlib_ng/gzip_ng.py

@@ -431,6 +431,7 @@ def main():
             if yes_or_no not in {"y", "Y", "yes"}:
                 sys.exit("not overwritten")
 
+    out_buffer = None
     if args.compress:
         if args.file is None:
             in_file = sys.stdin.buffer
@@ -470,6 +471,8 @@ def main():
             in_file.close()
         if out_file is not sys.stdout.buffer:
             out_file.close()
+        if out_buffer is not None and out_buffer is not sys.stdout.buffer:
+            out_buffer.close()
 
 
 if __name__ == "__main__":  # pragma: no cover

tests/test_gzip_ng.py

@@ -14,7 +14,6 @@
 import os
 import re
 import shutil
-import subprocess
 import sys
 import tempfile
 import zlib
@@ -28,21 +27,6 @@
 DATA = b'This is a simple test with gzip_ng'
 COMPRESSED_DATA = gzip.compress(DATA)
 TEST_FILE = str((Path(__file__).parent / "data" / "test.fastq.gz"))
-PYPY = sys.implementation.name == "pypy"
-
-
-def run_gzip_ng(*args, stdin=None):
-    """Calling gzip_ng externally seems to solve some issues on PyPy where
-    files would not be written properly when gzip_ng.main() was called. This is
-     probably due to some out of order execution that PyPy tries to pull.
-     Running the process externally is detrimental to the coverage report,
-     so this is only done for PyPy."""
-    process = subprocess.Popen(["python", "-m", "zlib_ng.gzip_ng", *args],
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE,
-                               stdin=subprocess.PIPE)
-
-    return process.communicate(stdin)
 
 
 def test_repr():
@@ -121,12 +105,9 @@ def test_decompress_infile_outfile(tmp_path, capsysbinary):
 def test_compress_infile_outfile(tmp_path, capsysbinary):
     test_file = tmp_path / "test"
     test_file.write_bytes(DATA)
-    if PYPY:
-        out, err = run_gzip_ng(str(test_file))
-    else:
-        sys.argv = ['', str(test_file)]
-        gzip_ng.main()
-        out, err = capsysbinary.readouterr()
+    sys.argv = ['', str(test_file)]
+    gzip_ng.main()
+    out, err = capsysbinary.readouterr()
     out_file = test_file.with_suffix(".gz")
     assert err == b''
     assert out == b''
@@ -189,12 +170,9 @@ def test_compress_infile_out_file(tmp_path, capsysbinary):
     test.write_bytes(DATA)
     out_file = tmp_path / "compressed.gz"
     args = ['-o', str(out_file), str(test)]
-    if PYPY:
-        out, err = run_gzip_ng(*args)
-    else:
-        sys.argv = ['', *args]
-        gzip_ng.main()
-        out, err = capsysbinary.readouterr()
+    sys.argv = ['', *args]
+    gzip_ng.main()
+    out, err = capsysbinary.readouterr()
     assert gzip.decompress(out_file.read_bytes()) == DATA
     assert err == b''
     assert out == b''
@@ -206,12 +184,9 @@ def test_compress_infile_out_file_force(tmp_path, capsysbinary):
     out_file = tmp_path / "compressed.gz"
     out_file.touch()
     args = ['-f', '-o', str(out_file), str(test)]
-    if PYPY:
-        out, err = run_gzip_ng(*args)
-    else:
-        sys.argv = ['', *args]
-        gzip_ng.main()
-        out, err = capsysbinary.readouterr()
+    sys.argv = ['', *args]
+    gzip_ng.main()
+    out, err = capsysbinary.readouterr()
     assert gzip.decompress(out_file.read_bytes()) == DATA
     assert err == b''
     assert out == b''
@@ -254,14 +229,11 @@ def test_compress_infile_out_file_inmplicit_name_prompt_accept(
     test.write_bytes(DATA)
     out_file = tmp_path / "test.gz"
     out_file.touch()
-    if PYPY:
-        out, err = run_gzip_ng(str(test), stdin=b"y\n")
-    else:
-        sys.argv = ['', str(test)]
-        mock_stdin = io.BytesIO(b"y")
-        sys.stdin = io.TextIOWrapper(mock_stdin)
-        gzip_ng.main()
-        out, err = capsysbinary.readouterr()
+    sys.argv = ['', str(test)]
+    mock_stdin = io.BytesIO(b"y")
+    sys.stdin = io.TextIOWrapper(mock_stdin)
+    gzip_ng.main()
+    out, err = capsysbinary.readouterr()
     assert b"already exists; do you wish to overwrite" in out
     assert err == b""
     assert gzip.decompress(out_file.read_bytes()) == DATA
@@ -271,13 +243,9 @@ def test_compress_infile_out_file_no_name(tmp_path, capsysbinary):
     test = tmp_path / "test"
     test.write_bytes(DATA)
     out_file = tmp_path / "compressed.gz"
-    args = ['-n', '-o', str(out_file), str(test)]
-    if PYPY:
-        out, err = run_gzip_ng(*args)
-    else:
-        sys.argv = ['', '-n', '-o', str(out_file), str(test)]
-        gzip_ng.main()
-        out, err = capsysbinary.readouterr()
+    sys.argv = ['', '-n', '-o', str(out_file), str(test)]
+    gzip_ng.main()
+    out, err = capsysbinary.readouterr()
     output = out_file.read_bytes()
     assert gzip.decompress(output) == DATA
     assert err == b''

tox.ini

@@ -10,13 +10,34 @@ deps=pytest
      coverage
 passenv=
     PYTHON_ZLIB_NG_LINK_DYNAMIC
+setenv=
+    PYTHONDEVMODE=1
 commands =
     # Create HTML coverage report for humans and xml coverage report for external services.
     coverage run --branch --source=zlib_ng -m pytest tests
     # Ignore errors during report generation. Pypy does not generate proper coverage reports.
     coverage html -i
     coverage xml -i
 
+[testenv:asan]
+setenv=
+    PYTHONDEVMODE=1
+    PYTHONMALLOC=malloc
+    CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer
+allowlist_externals=bash
+commands=
+    bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from zlib_ng import zlib_ng" && pytest tests'
+
+[testenv:compliance]
+deps=pytest
+commands=
+    pytest -v tests/test_zlib_compliance.py tests/test_gzip_compliance.py
+
+[testenv:compatibility]
+deps=pytest
+commands=
+    pytest tests/test_isal.py
+
 [testenv:lint]
 deps=flake8
      flake8-import-order
@@ -56,21 +77,25 @@ commands=
 
 [testenv:benchmark-all]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --all
 
 [testenv:benchmark-functions]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --functions
 
 [testenv:benchmark-gzip]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --gzip
 
 [testenv:benchmark-checksums]
 deps=
+setenv =
 commands=
     python ./benchmark_scripts/benchmark.py --checksums