From b6722c5ac4d657564f6ee202a71ba4f5887ee54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Wed, 27 Mar 2024 17:23:18 +0100 Subject: [PATCH 1/6] Allow @np to be used within df.eval and df.query Add additional check to make sure that ndim of the provided variable is an int. This makes sure that the ndim guard doesn't trigger if it is something else (e.g., a function in case of a numpy). This allow for using @np in df.eval and df.query methods. --- pandas/core/computation/ops.py | 2 +- pandas/tests/computation/test_eval.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py index cd9aa1833d586..7d8e23abf43b6 100644 --- a/pandas/core/computation/ops.py +++ b/pandas/core/computation/ops.py @@ -115,7 +115,7 @@ def _resolve_name(self): res = self.env.resolve(local_name, is_local=is_local) self.update(res) - if hasattr(res, "ndim") and res.ndim > 2: + if hasattr(res, "ndim") and isinstance(res.ndim, int) and res.ndim > 2: raise NotImplementedError( "N-dimensional objects, where N > 2, are not supported with eval" ) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 8f14c562fa7c3..9d0b58da58e70 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -737,6 +737,13 @@ def test_and_logic_string_match(self): assert pd.eval(f"{event.str.match('hello').a}") assert pd.eval(f"{event.str.match('hello').a and event.str.match('hello').a}") + def test_using_numpy(self): + # GH 58041 + df = Series([0.2, 1.5, 2.8], name="a").to_frame() + res = df.eval("@np.floor(a)") + expected = np.floor(df["a"]) + tm.assert_series_equal(expected, res) + # ------------------------------------- # gh-12388: Typecasting rules consistency with python From 4051d3659d9219c42cd435fe7ac700132f59a665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Thu, 28 Mar 2024 23:58:50 +0100 Subject: [PATCH 2/6] Add whatsnew --- doc/source/whatsnew/v3.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 549d49aaa1853..2f32cb1dd48d2 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -318,6 +318,7 @@ Bug fixes ~~~~~~~~~ - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) - Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) +- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using numpy via ``@`` notation. (:issue:`58041`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) - Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`) From e0b1e72d33e4d099232fabea590d240f4fb281b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Fri, 29 Mar 2024 08:26:59 +0100 Subject: [PATCH 3/6] Fix typo Co-authored-by: Abdulaziz Aloqeely <52792999+Aloqeely@users.noreply.github.com> --- doc/source/whatsnew/v3.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 81cb522e74660..34e755cdd513d 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -325,7 +325,7 @@ Bug fixes - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) - Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) - Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`) -- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using numpy via ``@`` notation. (:issue:`58041 +- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using numpy via ``@`` notation. (:issue:`58041`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) - Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`) From 81d9c7f6ced934cddcfabdfa33cb54527f9f6635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Fri, 29 Mar 2024 09:59:04 +0100 Subject: [PATCH 4/6] Test: skip checking names due to inconsistencies between OSes --- pandas/tests/computation/test_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 9d0b58da58e70..769d1847e1a2d 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -742,7 +742,7 @@ def test_using_numpy(self): df = Series([0.2, 1.5, 2.8], name="a").to_frame() res = df.eval("@np.floor(a)") expected = np.floor(df["a"]) - tm.assert_series_equal(expected, res) + tm.assert_series_equal(expected, res, check_names=False) # ------------------------------------- From 1642cdbac615ba4ee766de4f1ca8adf2432368ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Tue, 2 Apr 2024 11:05:57 +0200 Subject: [PATCH 5/6] Elaborate futher on whatsnew message --- doc/source/whatsnew/v3.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 34e755cdd513d..8d0adccfd480d 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -325,7 +325,7 @@ Bug fixes - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`) - Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`) - Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`) -- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using numpy via ``@`` notation. (:issue:`58041`) +- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using NumPy attributes via ``@`` notation, e.g., ``df.eval("@np.floor(a)")``. (:issue:`58041`) - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`) - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`) - Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`) From dee12b3dd52dd1fda9995596c87ee6140f06c9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Smr=C5=BE?= Date: Tue, 2 Apr 2024 20:25:05 +0200 Subject: [PATCH 6/6] Fix the test by explicitly specifing engine. Also move the test to more appropriate file. --- pandas/tests/computation/test_eval.py | 7 ------- pandas/tests/frame/test_query_eval.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 769d1847e1a2d..8f14c562fa7c3 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -737,13 +737,6 @@ def test_and_logic_string_match(self): assert pd.eval(f"{event.str.match('hello').a}") assert pd.eval(f"{event.str.match('hello').a and event.str.match('hello').a}") - def test_using_numpy(self): - # GH 58041 - df = Series([0.2, 1.5, 2.8], name="a").to_frame() - res = df.eval("@np.floor(a)") - expected = np.floor(df["a"]) - tm.assert_series_equal(expected, res, check_names=False) - # ------------------------------------- # gh-12388: Typecasting rules consistency with python diff --git a/pandas/tests/frame/test_query_eval.py b/pandas/tests/frame/test_query_eval.py index d2e36eb6147e7..94e8e469f21e7 100644 --- a/pandas/tests/frame/test_query_eval.py +++ b/pandas/tests/frame/test_query_eval.py @@ -188,6 +188,16 @@ def test_eval_object_dtype_binop(self): expected = DataFrame({"a1": ["Y", "N"], "c": [True, False]}) tm.assert_frame_equal(res, expected) + def test_using_numpy(self, engine, parser): + # GH 58041 + skip_if_no_pandas_parser(parser) + df = Series([0.2, 1.5, 2.8], name="a").to_frame() + res = df.eval("@np.floor(a)", engine=engine, parser=parser) + expected = np.floor(df["a"]) + if engine == "numexpr": + expected.name = None # See GH 58069 + tm.assert_series_equal(expected, res) + class TestDataFrameQueryWithMultiIndex: def test_query_with_named_multiindex(self, parser, engine):