Skip to content

BUG: Datetime and Timestamp are not well supported by pd.eval #52559

New issue
New issue
Open
Open
BUG: Datetime and Timestamp are not well supported by pd.eval#52559
Labels
Bugexpressionspd.eval, query
@semigodking

Description

@semigodking
semigodking
opened on Apr 10, 2023

Pandas version checks

  • I have checked that this issue has not already been reported.

  • I have confirmed this bug exists on the latest version of pandas.

  • I have confirmed this bug exists on the main branch of pandas.

Reproducible Example

import pandas as pd
df = pd.DataFrame([[pd.to_datetime('now', utc=True)]], columns=['time1'])
pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])

Issue Description

When I try to calculate datetimes in by eval as code sample above, exceptions are raised. I know such calculations can be done without eval. However, for some reason, I would prefer this operation can be done via expression.

Exceptions:

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[25], line 1 ----> 1 pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(year=1)", resolvers=[{'df': df, 'pd':pd}])

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
327 # get our (possibly passed-in) scope
328 env = ensure_scope(
329 level + 1,
330 global_dict=global_dict,
(...)
333 target=target,
334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
338 if engine == "numexpr" and (
339 is_extension_array_dtype(parsed_expr.terms.return_type)
340 or getattr(parsed_expr.terms, "operand_types", None) is not None
(...)
344 )
345 ):
346 warnings.warn(
347 "Engine has switched to 'python' because numexpr does not support "
348 "extension array dtypes. Please set your engine to python manually.",
349 RuntimeWarning,
350 stacklevel=find_stack_level(),
351 )

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:809, in Expr.init(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:421, in BaseExprVisitor.visit_Module(self, node, **kwargs)
419 raise SyntaxError("only a single expression is allowed")
420 expr = node.body[0]
--> 421 return self.visit(expr, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:424, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
423 def visit_Expr(self, node, **kwargs):
--> 424 return self.visit(node.value, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:719, in BaseExprVisitor.visit_Compare(self, node, **kwargs)
717 op = self.translate_In(ops[0])
718 binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 719 return self.visit(binop)
721 # recursive case: we have a chained comparison, a CMP b CMP c, etc.
722 left = node.left

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:535, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
534 def visit_BinOp(self, node, **kwargs):
--> 535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
537 return self._maybe_evaluate_binop(op, op_class, left, right)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:455, in BaseExprVisitor._maybe_transform_eq_ne(self, node, left, right)
453 def _maybe_transform_eq_ne(self, node, left=None, right=None):
454 if left is None:
--> 455 left = self.visit(node.left, side="left")
456 if right is None:
457 right = self.visit(node.right, side="right")

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:537, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
--> 537 return self._maybe_evaluate_binop(op, op_class, left, right)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:507, in BaseExprVisitor._maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
504 res = op(lhs, rhs)
506 if res.has_invalid_return_type:
--> 507 raise TypeError(
508 f"unsupported operand type(s) for {res.op}: "
509 f"'{lhs.type}' and '{rhs.type}'"
510 )
512 if self.engine != "pytables" and (
513 res.op in CMP_OPS_SYMS
514 and getattr(lhs, "is_datetime", False)
(...)
517 # all date ops must be done in python bc numexpr doesn't work
518 # well with NaT
519 return self._maybe_eval(res, self.binary_ops)

TypeError: unsupported operand type(s) for -: '<class 'pandas._libs.tslibs.timestamps.Timestamp'>' and 'datetime64[ns]'

pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])

--------------------------------------------------------------------------- TypeError Traceback (most recent call last) Cell In[27], line 1 ----> 1 pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
327 # get our (possibly passed-in) scope
328 env = ensure_scope(
329 level + 1,
330 global_dict=global_dict,
(...)
333 target=target,
334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
338 if engine == "numexpr" and (
339 is_extension_array_dtype(parsed_expr.terms.return_type)
340 or getattr(parsed_expr.terms, "operand_types", None) is not None
(...)
344 )
345 ):
346 warnings.warn(
347 "Engine has switched to 'python' because numexpr does not support "
348 "extension array dtypes. Please set your engine to python manually.",
349 RuntimeWarning,
350 stacklevel=find_stack_level(),
351 )

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:809, in Expr.init(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:421, in BaseExprVisitor.visit_Module(self, node, **kwargs)
419 raise SyntaxError("only a single expression is allowed")
420 expr = node.body[0]
--> 421 return self.visit(expr, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:424, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
423 def visit_Expr(self, node, **kwargs):
--> 424 return self.visit(node.value, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).name}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:537, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
--> 537 return self._maybe_evaluate_binop(op, op_class, left, right)

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:504, in BaseExprVisitor._maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
495 def _maybe_evaluate_binop(
496 self,
497 op,
(...)
502 maybe_eval_in_python=("==", "!=", "<", ">", "<=", ">="),
503 ):
--> 504 res = op(lhs, rhs)
506 if res.has_invalid_return_type:
507 raise TypeError(
508 f"unsupported operand type(s) for {res.op}: "
509 f"'{lhs.type}' and '{rhs.type}'"
510 )

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/ops.py:380, in BinOp.init(self, op, lhs, rhs)
376 self.rhs = rhs
378 self._disallow_scalar_only_bool_ops()
--> 380 self.convert_values()
382 try:
383 self.func = _binary_ops_dict[op]

File ~/venv/lib/python3.8/site-packages/pandas/core/computation/ops.py:478, in BinOp.convert_values(self)
476 if isinstance(v, (int, float)):
477 v = stringify(v)
--> 478 v = Timestamp(ensure_decoded(v))
479 if v.tz is not None:
480 v = v.tz_convert("UTC")

File ~/venv/lib/python3.8/site-packages/pandas/_libs/tslibs/timestamps.pyx:1667, in pandas._libs.tslibs.timestamps.Timestamp.new()

File ~/venv/lib/python3.8/site-packages/pandas/_libs/tslibs/conversion.pyx:336, in pandas._libs.tslibs.conversion.convert_to_tsobject()

TypeError: Cannot convert input [0 days 01:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp

Expected Behavior

The expressions given in example could pass and produce correct results.

Installed Versions

INSTALLED VERSIONS

commit : 478d340
python : 3.8.10.final.0
python-bits : 64
OS : Linux
OS-release : 5.19.10-rockchip64
Version : #22.08.2 SMP PREEMPT Wed Sep 21 19:15:09 UTC 2022
machine : aarch64
processor : aarch64
byteorder : little
LC_ALL : en_US.UTF-8
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8

pandas : 2.0.0
numpy : 1.24.2
pytz : 2023.3
dateutil : 2.8.2
setuptools : 44.0.0
pip : 23.0.1
Cython : None
pytest : None
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 3.1.2
IPython : 8.11.0
pandas_datareader: None
bs4 : 4.12.0
bottleneck : None
brotli : None
fastparquet : None
fsspec : None
gcsfs : None
matplotlib : None
numba : None
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : None
pyreadstat : None
pyxlsb : None
s3fs : None
scipy : None
snappy : None
sqlalchemy : None
tables : None
tabulate : None
xarray : None
xlrd : None
zstandard : None
tzdata : 2023.3
qtpy : None
pyqt5 : None

Metadata

Metadata

Assignees

No one assigned

    Labels

    Bugexpressionspd.eval, query

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions