Skip to content

Commit 3d3f7a6

Browse files
committed
TST: add numexpr fixture
1 parent d3f0e9a commit 3d3f7a6

File tree

5 files changed

+63
-40
lines changed

5 files changed

+63
-40
lines changed

pandas/conftest.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,6 +1248,52 @@ def utc_fixture(request):
12481248
utc_fixture2 = utc_fixture
12491249

12501250

1251+
@pytest.fixture
1252+
def is_using_numexpr():
1253+
"""
1254+
If True always uses numexpr for evaluations, if False never. If unsure, raise.
1255+
1256+
This fixture is meant to be used together with the ``autouse_numexpr`` fixture.
1257+
1258+
We are unsure whether we're using numexpr if `expr._USE_NUMEXPR` is True and
1259+
`expr._MIN_ELEMENTS` is != 0 (i.e. usage of numexpr depends on size of the data).
1260+
By default `expr._MIN_ELEMENTS` is 1_000_000.
1261+
"""
1262+
from pandas.core.computation import expressions as expr
1263+
1264+
if not expr.USE_NUMEXPR:
1265+
return False
1266+
elif expr._MIN_ELEMENTS == 0:
1267+
return True
1268+
else:
1269+
raise ValueError(
1270+
"expr.USE_NUMEXPR == True and expr._MIN_ELEMENTS =! 0. "
1271+
"Pandas can not determine if numexpr is used."
1272+
)
1273+
1274+
1275+
@pytest.fixture(params=[True, False], ids=["numexpr", "python"])
1276+
def _use_numexpr(request):
1277+
"""
1278+
If yields True, always use numexpr for evaluations, else never.
1279+
"""
1280+
from pandas.core.computation import expressions as expr
1281+
1282+
if request.param and not expr.NUMEXPR_INSTALLED:
1283+
pytest.skip("numexpr not installed")
1284+
1285+
OLD_USE_NUMEXPR = expr.USE_NUMEXPR
1286+
OLD_MIN_ELEMENTS = expr._MIN_ELEMENTS
1287+
1288+
expr.set_use_numexpr(request.param)
1289+
if request.param:
1290+
expr._MIN_ELEMENTS = 0
1291+
yield request.param
1292+
1293+
expr.set_use_numexpr(OLD_USE_NUMEXPR)
1294+
expr._MIN_ELEMENTS = OLD_MIN_ELEMENTS
1295+
1296+
12511297
# ----------------------------------------------------------------
12521298
# Dtypes
12531299
# ----------------------------------------------------------------

pandas/tests/arithmetic/conftest.py

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,15 @@
99
Int64Index,
1010
UInt64Index,
1111
)
12-
from pandas.core.computation import expressions as expr
1312

13+
# ------------------------------------------------------------------
1414

15-
@pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
16-
def switch_numexpr_min_elements(request):
17-
_MIN_ELEMENTS = expr._MIN_ELEMENTS
18-
expr._MIN_ELEMENTS = request.param
19-
yield request.param
20-
expr._MIN_ELEMENTS = _MIN_ELEMENTS
2115

16+
@pytest.fixture(autouse=True)
17+
def autouse_numexpr(_use_numexpr):
18+
yield _use_numexpr
2219

23-
# ------------------------------------------------------------------
2420

25-
# doctest with +SKIP for one fixture fails during setup with
26-
# 'DoctestItem' object has no attribute 'callspec'
27-
# due to switch_numexpr_min_elements fixture
2821
@pytest.fixture(params=[1, np.array(1, dtype=np.int64)])
2922
def one(request):
3023
"""
@@ -61,9 +54,6 @@ def one(request):
6154
zeros.extend([0, 0.0, -0.0])
6255

6356

64-
# doctest with +SKIP for zero fixture fails during setup with
65-
# 'DoctestItem' object has no attribute 'callspec'
66-
# due to switch_numexpr_min_elements fixture
6757
@pytest.fixture(params=zeros)
6858
def zero(request):
6959
"""

pandas/tests/arithmetic/test_numeric.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
Int64Index,
2828
UInt64Index,
2929
)
30-
from pandas.core.computation import expressions as expr
3130
from pandas.tests.arithmetic.common import (
3231
assert_invalid_addsub_type,
3332
assert_invalid_comparison,
@@ -392,7 +391,7 @@ def test_div_negative_zero(self, zero, numeric_idx, op):
392391
@pytest.mark.parametrize("dtype1", [np.int64, np.float64, np.uint64])
393392
def test_ser_div_ser(
394393
self,
395-
switch_numexpr_min_elements,
394+
is_using_numexpr,
396395
dtype1,
397396
any_real_numpy_dtype,
398397
):
@@ -412,7 +411,7 @@ def test_ser_div_ser(
412411
if first.dtype == "int64" and second.dtype == "float32":
413412
# when using numexpr, the casting rules are slightly different
414413
# and int64/float32 combo results in float32 instead of float64
415-
if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
414+
if is_using_numexpr:
416415
expected = expected.astype("float32")
417416

418417
result = first / second

pandas/tests/frame/test_arithmetic.py

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
)
2323
import pandas._testing as tm
2424
import pandas.core.common as com
25-
from pandas.core.computation import expressions as expr
2625
from pandas.core.computation.expressions import (
2726
_MIN_ELEMENTS,
2827
NUMEXPR_INSTALLED,
@@ -33,12 +32,9 @@
3332
)
3433

3534

36-
@pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
37-
def switch_numexpr_min_elements(request):
38-
_MIN_ELEMENTS = expr._MIN_ELEMENTS
39-
expr._MIN_ELEMENTS = request.param
40-
yield request.param
41-
expr._MIN_ELEMENTS = _MIN_ELEMENTS
35+
@pytest.fixture(autouse=True)
36+
def autouse_numexpr(_use_numexpr):
37+
yield _use_numexpr
4238

4339

4440
class DummyElement:
@@ -571,7 +567,7 @@ def test_arith_flex_frame_mixed(
571567
int_frame,
572568
mixed_int_frame,
573569
mixed_float_frame,
574-
switch_numexpr_min_elements,
570+
is_using_numexpr,
575571
):
576572
f = getattr(operator, op)
577573

@@ -585,7 +581,7 @@ def test_arith_flex_frame_mixed(
585581
dtype = {"B": "uint64", "C": None}
586582
elif op in ["__add__", "__mul__"]:
587583
dtype = {"C": None}
588-
if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
584+
if is_using_numexpr:
589585
# when using numexpr, the casting rules are slightly different:
590586
# in the `2 + mixed_int_frame` operation, int32 column becomes
591587
# and int64 column (not preserving dtype in operation with Python
@@ -1067,7 +1063,7 @@ def test_frame_with_frame_reindex(self):
10671063
],
10681064
ids=lambda x: x.__name__,
10691065
)
1070-
def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements, request):
1066+
def test_binop_other(self, op, value, dtype, is_using_numexpr):
10711067

10721068
skip = {
10731069
(operator.truediv, "bool"),
@@ -1101,11 +1097,7 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements, reques
11011097
msg = "ufunc 'remainder' not supported for the input types"
11021098
elif op is operator.sub:
11031099
msg = "numpy boolean subtract, the `-` operator, is "
1104-
if (
1105-
dtype == "bool"
1106-
and expr.USE_NUMEXPR
1107-
and switch_numexpr_min_elements == 0
1108-
):
1100+
if dtype == "bool" and is_using_numexpr:
11091101
warn = UserWarning # "evaluating in Python space because ..."
11101102
else:
11111103
msg = (
@@ -1120,7 +1112,7 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements, reques
11201112
elif (op, dtype) in skip:
11211113

11221114
if op in [operator.add, operator.mul]:
1123-
if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
1115+
if is_using_numexpr:
11241116
# "evaluating in Python space because ..."
11251117
warn = UserWarning
11261118
else:

pandas/tests/series/test_arithmetic.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,11 @@
3131
nanops,
3232
ops,
3333
)
34-
from pandas.core.computation import expressions as expr
3534

3635

37-
@pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
38-
def switch_numexpr_min_elements(request):
39-
_MIN_ELEMENTS = expr._MIN_ELEMENTS
40-
expr._MIN_ELEMENTS = request.param
41-
yield request.param
42-
expr._MIN_ELEMENTS = _MIN_ELEMENTS
36+
@pytest.fixture(autouse=True)
37+
def autouse_numexpr(_use_numexpr):
38+
yield _use_numexpr
4339

4440

4541
def _permute(obj):

0 commit comments

Comments
 (0)