From 0e2821ac10100e9c1c10ae1d628f246ddea79f8b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:49:22 -0700 Subject: [PATCH 1/2] STY: Consolidate & add pre-commit checks --- .pre-commit-config.yaml | 36 ++++++------- pandas/tests/io/pytables/test_store.py | 74 ++++++++++---------------- pyproject.toml | 2 + 3 files changed, 44 insertions(+), 68 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9cd7528bcd2f..f90dc1fef5300 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,14 +46,22 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: + - id: check-ast + - id: check-case-conflict + - id: check-toml + - id: check-xml + - id: check-yaml + exclude: ^ci/meta.yaml$ - id: debug-statements - id: end-of-file-fixer exclude: \.txt$ - stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, - post-checkout, post-commit, post-merge, post-rewrite] + - id: mixed-line-ending + args: [--fix=auto] + exclude: ^pandas/tests/io/parser/data/utf16_ex.txt$ + - id: fix-byte-order-marker + - id: fix-encoding-pragma + args: [--remove] - id: trailing-whitespace - stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, - post-checkout, post-commit, post-merge, post-rewrite] - repo: https://github.com/cpplint/cpplint rev: 1.6.1 hooks: @@ -98,6 +106,8 @@ repos: - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore - id: rst-backticks - id: rst-directive-colons types: [text] # overwrite types: [rst] @@ -160,25 +170,17 @@ repos: language: pygrep types: [python] files: ^pandas/tests/ - exclude: | - (?x)^ - pandas/tests/io/pytables/test_store\.py$ - id: unwanted-patterns name: Unwanted patterns language: pygrep entry: | (?x) - # outdated annotation syntax, missing error codes + # outdated annotation syntax \#\ type:\ (?!ignore) - |\#\ type:\s?ignore(?!\[) # foo._class__ instead of type(foo) |\.__class__ - # np.bool/np.object instead of np.bool_/np.object_ - |np\.bool[^_8`] - |np\.object[^_8`] - # imports from collections.abc instead of `from collections import abc` |from\ collections\.abc\ import @@ -200,16 +202,8 @@ repos: # builtin filter function |(?obj`, not ` obj` - language: pygrep - entry: '[a-zA-Z0-9*]> ' - files: (\.pyx|\.pxi.in)$ - id: incorrect-backticks name: Check for backticks incorrectly rendering because of missing spaces language: pygrep diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 82330e1d63c9a..17b7eef011a0a 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -1,6 +1,6 @@ +import contextlib import datetime as dt import hashlib -import os import tempfile import time from warnings import ( @@ -27,7 +27,6 @@ from pandas.tests.io.pytables.common import ( _maybe_remove, ensure_clean_store, - safe_close, ) from pandas.io.pytables import ( @@ -832,53 +831,34 @@ def reader(path): tm.assert_frame_equal(df, result) -def test_copy(): - with catch_warnings(record=True): - - def do_copy(f, new_f=None, keys=None, propindexes=True, **kwargs): - if new_f is None: - fd, new_f = tempfile.mkstemp() - - try: - store = HDFStore(f, "r") - tstore = store.copy(new_f, keys=keys, propindexes=propindexes, **kwargs) +@pytest.mark.parametrize("propindexes", [True, False]) +def test_copy(propindexes): + df = tm.makeDataFrame() - # check keys - if keys is None: + with tm.ensure_clean() as path: + with HDFStore(path) as st: + st.append("df", df, data_columns=["A"]) + with tempfile.NamedTemporaryFile() as new_f: + with HDFStore(path) as store: + with contextlib.closing( + store.copy(new_f.name, keys=None, propindexes=propindexes) + ) as tstore: + # check keys keys = store.keys() - assert set(keys) == set(tstore.keys()) - - # check indices & nrows - for k in tstore.keys(): - if tstore.get_storer(k).is_table: - new_t = tstore.get_storer(k) - orig_t = store.get_storer(k) - - assert orig_t.nrows == new_t.nrows - - # check propindixes - if propindexes: - for a in orig_t.axes: - if a.is_indexed: - assert new_t[a.name].is_indexed - - finally: - safe_close(store) - safe_close(tstore) - try: - os.close(fd) - except (OSError, ValueError): - pass - os.remove(new_f) - - # new table - df = tm.makeDataFrame() - - with tm.ensure_clean() as path: - with HDFStore(path) as st: - st.append("df", df, data_columns=["A"]) - do_copy(f=path) - do_copy(f=path, propindexes=False) + assert set(keys) == set(tstore.keys()) + # check indices & nrows + for k in tstore.keys(): + if tstore.get_storer(k).is_table: + new_t = tstore.get_storer(k) + orig_t = store.get_storer(k) + + assert orig_t.nrows == new_t.nrows + + # check propindixes + if propindexes: + for a in orig_t.axes: + if a.is_indexed: + assert new_t[a.name].is_indexed def test_duplicate_column_name(tmp_path, setup_path): diff --git a/pyproject.toml b/pyproject.toml index a2ae269c26667..4dfdb1aa927f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,8 @@ select = [ "PGH", # Ruff-specific rules "RUF", + # flake8-bandit: exec-builtin + "S102" ] ignore = [ From b9509a51ae8d395395f8327251e240c7e63b97fa Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:00:13 -0700 Subject: [PATCH 2/2] Remove & consolidate pre-commit checks --- .pre-commit-config.yaml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f90dc1fef5300..1301683af5a84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -164,12 +164,6 @@ repos: exclude: ^pandas/core/interchange/ language: python types: [python] - - id: no-os-remove - name: Check code for instances of os.remove - entry: os\.remove - language: pygrep - types: [python] - files: ^pandas/tests/ - id: unwanted-patterns name: Unwanted patterns language: pygrep @@ -216,19 +210,6 @@ repos: entry: 'np\.random\.seed' files: ^asv_bench/benchmarks exclude: ^asv_bench/benchmarks/pandas_vb_common\.py - - id: np-testing-array-equal - name: Check for usage of numpy testing or array_equal - language: pygrep - entry: '(numpy|np)(\.testing|\.array_equal)' - files: ^pandas/tests/ - types: [python] - - id: invalid-ea-testing - name: Check for invalid EA testing - language: pygrep - entry: 'tm\.assert_(series|frame)_equal' - files: ^pandas/tests/extension/base - types: [python] - exclude: ^pandas/tests/extension/base/base\.py - id: unwanted-patterns-in-tests name: Unwanted patterns in tests language: pygrep @@ -262,6 +243,9 @@ repos: # pytest.warns (use tm.assert_produces_warning instead) |pytest\.warns + + # os.remove + |os\.remove files: ^pandas/tests/ types_or: [python, cython, rst] - id: unwanted-patterns-in-ea-tests