From 0e0b5553129cf99d6bbd2618f2e3d015c4d987d0 Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Sat, 21 May 2022 18:03:28 +0300 Subject: [PATCH 01/10] BUG: Fix eval's empty scopes not working --- pandas/core/computation/scope.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/computation/scope.py b/pandas/core/computation/scope.py index c661e225c26fa..05e188c873284 100644 --- a/pandas/core/computation/scope.py +++ b/pandas/core/computation/scope.py @@ -133,11 +133,11 @@ def __init__( # shallow copy here because we don't want to replace what's in # scope when we align terms (alignment accesses the underlying # numpy array of pandas objects) - scope_global = self.scope.new_child((global_dict or frame.f_globals).copy()) + scope_global = self.scope.new_child((global_dict if global_dict is not None else frame.f_globals).copy()) self.scope = DeepChainMap(scope_global) if not isinstance(local_dict, Scope): scope_local = self.scope.new_child( - (local_dict or frame.f_locals).copy() + (local_dict if local_dict is not None else frame.f_locals).copy() ) self.scope = DeepChainMap(scope_local) finally: From ab1ce4e7672f0bf331b07c2425c41756b147cae5 Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Sat, 21 May 2022 19:11:34 +0300 Subject: [PATCH 02/10] TST: Add tests for empty dicts in eval scopes --- pandas/tests/computation/test_eval.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 956e01ec5bde9..50d127955bc41 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -44,6 +44,7 @@ from pandas.core.computation.ops import ( ARITH_OPS_SYMS, SPECIAL_CASE_ARITH_OPS_SYMS, + UndefinedVariableError, _binary_math_ops, _binary_ops_dict, _unary_math_ops, @@ -1659,6 +1660,18 @@ def test_no_new_globals(self, engine, parser): gbls2 = globals().copy() assert gbls == gbls2 + def test_empty_locals(self, engine, parser): + x = 1 + msg = "name 'x' is not defined" + with pytest.raises(UndefinedVariableError, match=msg): + pd.eval("x + 1", engine=engine, parser=parser, local_dict={}) + + def test_empty_globals(self, engine, parser): + msg = "name '_var_s' is not defined" + e = "_var_s * 2" + with pytest.raises(UndefinedVariableError, match=msg): + pd.eval(e, engine=engine, parser=parser, global_dict={}) + @td.skip_if_no_ne def test_invalid_engine(): From 4b21c88efcb8efc2f121380c875103e5cddea741 Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Sun, 22 May 2022 13:52:32 +0000 Subject: [PATCH 03/10] Fixes from pre-commit [automated commit] --- pandas/core/computation/scope.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/computation/scope.py b/pandas/core/computation/scope.py index 05e188c873284..52169b034603d 100644 --- a/pandas/core/computation/scope.py +++ b/pandas/core/computation/scope.py @@ -133,7 +133,9 @@ def __init__( # shallow copy here because we don't want to replace what's in # scope when we align terms (alignment accesses the underlying # numpy array of pandas objects) - scope_global = self.scope.new_child((global_dict if global_dict is not None else frame.f_globals).copy()) + scope_global = self.scope.new_child( + (global_dict if global_dict is not None else frame.f_globals).copy() + ) self.scope = DeepChainMap(scope_global) if not isinstance(local_dict, Scope): scope_local = self.scope.new_child( From e0e6eae42c2a309078149c67ff19827a04a0a7df Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Mon, 23 May 2022 13:16:19 +0300 Subject: [PATCH 04/10] DOC: Add whatsnew entry --- doc/source/whatsnew/v1.5.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index b23dd5c2f05a6..aea69bd409ae2 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -824,6 +824,7 @@ Metadata Other ^^^^^ +- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) .. ***DO NOT USE THIS SECTION*** From ea824baa8562a3ffa97ca49c7c4ca0c652d7731b Mon Sep 17 00:00:00 2001 From: Alex-Blade <44120047+Alex-Blade@users.noreply.github.com> Date: Wed, 25 May 2022 00:34:28 +0300 Subject: [PATCH 05/10] DOC: Update whatsnew section --- doc/source/whatsnew/v1.5.0.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index aea69bd409ae2..ed4a1d1f71e51 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -824,11 +824,10 @@ Metadata Other ^^^^^ -- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) .. ***DO NOT USE THIS SECTION*** -- +- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) - .. --------------------------------------------------------------------------- From a4fd9c4fc265256860d868f53ddd9ef8c127c02d Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Wed, 25 May 2022 11:55:40 +0300 Subject: [PATCH 06/10] CLN: Cleanup whatsnew and comment tests --- doc/source/whatsnew/v1.5.0.rst | 3 ++- pandas/tests/computation/test_eval.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index ed4a1d1f71e51..3dddae6616834 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -675,6 +675,7 @@ Conversion - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`) - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`) - Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`) +- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) Strings ^^^^^^^ @@ -827,7 +828,7 @@ Other .. ***DO NOT USE THIS SECTION*** -- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) +- - .. --------------------------------------------------------------------------- diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 50d127955bc41..4812bbaae9bb4 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1661,12 +1661,14 @@ def test_no_new_globals(self, engine, parser): assert gbls == gbls2 def test_empty_locals(self, engine, parser): + # GH 47084 x = 1 msg = "name 'x' is not defined" with pytest.raises(UndefinedVariableError, match=msg): pd.eval("x + 1", engine=engine, parser=parser, local_dict={}) def test_empty_globals(self, engine, parser): + # GH 47084 msg = "name '_var_s' is not defined" e = "_var_s * 2" with pytest.raises(UndefinedVariableError, match=msg): From 6842b9abf894f04cfe6bd24f9f1408763b75f8ef Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Mon, 30 May 2022 18:37:28 +0300 Subject: [PATCH 07/10] CLN: Move whatsnew to 1.4.3 and fix tests --- doc/source/whatsnew/v1.4.3.rst | 6 ++++-- doc/source/whatsnew/v1.5.0.rst | 1 - pandas/tests/computation/test_eval.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v1.4.3.rst b/doc/source/whatsnew/v1.4.3.rst index 415a3ff4efda0..c683632b8066f 100644 --- a/doc/source/whatsnew/v1.4.3.rst +++ b/doc/source/whatsnew/v1.4.3.rst @@ -27,8 +27,10 @@ Fixed regressions Bug fixes ~~~~~~~~~ -- -- + +Conversion +^^^^^^^^^^ +- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) .. --------------------------------------------------------------------------- diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index 3dddae6616834..b23dd5c2f05a6 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -675,7 +675,6 @@ Conversion - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`) - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`) - Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`) -- Bug in :meth:`pd.eval`, :meth:`DataFrame.eval` and :meth:`DataFrame.query` where passing empty ``local_dict`` or ``global_dict`` was treated as passing ``None`` (:issue:`47084`) Strings ^^^^^^^ diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py index 4812bbaae9bb4..16f391eccbeb3 100644 --- a/pandas/tests/computation/test_eval.py +++ b/pandas/tests/computation/test_eval.py @@ -1662,7 +1662,7 @@ def test_no_new_globals(self, engine, parser): def test_empty_locals(self, engine, parser): # GH 47084 - x = 1 + x = 1 # noqa: F841 msg = "name 'x' is not defined" with pytest.raises(UndefinedVariableError, match=msg): pd.eval("x + 1", engine=engine, parser=parser, local_dict={}) From 03d1fd0a5288ff8f914ffb3f4d83e4da891c0660 Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Mon, 30 May 2022 22:27:58 +0300 Subject: [PATCH 08/10] BUG: PyTablesExpr relies on vague behavior --- pandas/core/computation/pytables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py index 138b05b33e375..f6bd72346d227 100644 --- a/pandas/core/computation/pytables.py +++ b/pandas/core/computation/pytables.py @@ -3,7 +3,7 @@ import ast from functools import partial -from typing import Any +from typing import Any, Optional import numpy as np @@ -563,7 +563,7 @@ def __init__( self._visitor = None # capture the environment if needed - local_dict: DeepChainMap[Any, Any] = DeepChainMap() + local_dict: Optional[DeepChainMap[Any, Any]] = None if isinstance(where, PyTablesExpr): local_dict = where.env.scope From be23c205355d871110f4209650e4d1e3028d0537 Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Mon, 30 May 2022 20:46:27 +0000 Subject: [PATCH 09/10] Fixes from pre-commit [automated commit] --- pandas/core/computation/pytables.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py index f6bd72346d227..98a6f60fce90b 100644 --- a/pandas/core/computation/pytables.py +++ b/pandas/core/computation/pytables.py @@ -3,7 +3,10 @@ import ast from functools import partial -from typing import Any, Optional +from typing import ( + Any, + Optional, +) import numpy as np @@ -563,7 +566,7 @@ def __init__( self._visitor = None # capture the environment if needed - local_dict: Optional[DeepChainMap[Any, Any]] = None + local_dict: DeepChainMap[Any, Any] | None = None if isinstance(where, PyTablesExpr): local_dict = where.env.scope From 3524ae2caa620835fa39cd6ca53f534ed82daeac Mon Sep 17 00:00:00 2001 From: Alexander Lakeev Date: Tue, 31 May 2022 00:09:55 +0300 Subject: [PATCH 10/10] CLN: Remove unused import --- pandas/core/computation/pytables.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py index 98a6f60fce90b..91a8505fad8c5 100644 --- a/pandas/core/computation/pytables.py +++ b/pandas/core/computation/pytables.py @@ -3,10 +3,7 @@ import ast from functools import partial -from typing import ( - Any, - Optional, -) +from typing import Any import numpy as np