Pandas version checks
-
[X] I have checked that this issue has not already been reported.
-
[X] I have confirmed this bug exists on the latest version of pandas.
-
[ ] I have confirmed this bug exists on the main branch of pandas.
Reproducible Example
import pandas as pd
df = pd.DataFrame([[pd.to_datetime('now', utc=True)]], columns=['time1'])
pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
Issue Description
When I try to calculate datetimes in by eval as code sample above, exceptions are raised. I know such calculations can be done without eval. However, for some reason, I would prefer this operation can be done via expression.
Exceptions:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[25], line 1
----> 1 pd.eval("pd.to_datetime('now', utc=True) - df['time1'] < pd.Timedelta(year=1)", resolvers=[{'df': df, 'pd':pd}])
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
327 # get our (possibly passed-in) scope
328 env = ensure_scope(
329 level + 1,
330 global_dict=global_dict,
(...)
333 target=target,
334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
338 if engine == "numexpr" and (
339 is_extension_array_dtype(parsed_expr.terms.return_type)
340 or getattr(parsed_expr.terms, "operand_types", None) is not None
(...)
344 )
345 ):
346 warnings.warn(
347 "Engine has switched to 'python' because numexpr does not support "
348 "extension array dtypes. Please set your engine to python manually.",
349 RuntimeWarning,
350 stacklevel=find_stack_level(),
351 )
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:809, in Expr.__init__(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:421, in BaseExprVisitor.visit_Module(self, node, **kwargs)
419 raise SyntaxError("only a single expression is allowed")
420 expr = node.body[0]
--> 421 return self.visit(expr, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:424, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
423 def visit_Expr(self, node, **kwargs):
--> 424 return self.visit(node.value, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:719, in BaseExprVisitor.visit_Compare(self, node, **kwargs)
717 op = self.translate_In(ops[0])
718 binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 719 return self.visit(binop)
721 # recursive case: we have a chained comparison, a CMP b CMP c, etc.
722 left = node.left
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:535, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
534 def visit_BinOp(self, node, **kwargs):
--> 535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
537 return self._maybe_evaluate_binop(op, op_class, left, right)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:455, in BaseExprVisitor._maybe_transform_eq_ne(self, node, left, right)
453 def _maybe_transform_eq_ne(self, node, left=None, right=None):
454 if left is None:
--> 455 left = self.visit(node.left, side="left")
456 if right is None:
457 right = self.visit(node.right, side="right")
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:537, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
--> 537 return self._maybe_evaluate_binop(op, op_class, left, right)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:507, in BaseExprVisitor._maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
504 res = op(lhs, rhs)
506 if res.has_invalid_return_type:
--> 507 raise TypeError(
508 f"unsupported operand type(s) for {res.op}: "
509 f"'{lhs.type}' and '{rhs.type}'"
510 )
512 if self.engine != "pytables" and (
513 res.op in CMP_OPS_SYMS
514 and getattr(lhs, "is_datetime", False)
(...)
517 # all date ops must be done in python bc numexpr doesn't work
518 # well with NaT
519 return self._maybe_eval(res, self.binary_ops)
TypeError: unsupported operand type(s) for -: '' and 'datetime64[ns]'
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
------------------------------------------------------------
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[27], line 1
----> 1 pd.eval("df['time1']-pd.Timedelta(hours=1)", resolvers=[{'df': df, 'pd':pd}])
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/eval.py:336, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
327 # get our (possibly passed-in) scope
328 env = ensure_scope(
329 level + 1,
330 global_dict=global_dict,
(...)
333 target=target,
334 )
--> 336 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
338 if engine == "numexpr" and (
339 is_extension_array_dtype(parsed_expr.terms.return_type)
340 or getattr(parsed_expr.terms, "operand_types", None) is not None
(...)
344 )
345 ):
346 warnings.warn(
347 "Engine has switched to 'python' because numexpr does not support "
348 "extension array dtypes. Please set your engine to python manually.",
349 RuntimeWarning,
350 stacklevel=find_stack_level(),
351 )
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:809, in Expr.__init__(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:421, in BaseExprVisitor.visit_Module(self, node, **kwargs)
419 raise SyntaxError("only a single expression is allowed")
420 expr = node.body[0]
--> 421 return self.visit(expr, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:424, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
423 def visit_Expr(self, node, **kwargs):
--> 424 return self.visit(node.value, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:415, in BaseExprVisitor.visit(self, node, **kwargs)
413 method = f"visit_{type(node).__name__}"
414 visitor = getattr(self, method)
--> 415 return visitor(node, **kwargs)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:537, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
--> 537 return self._maybe_evaluate_binop(op, op_class, left, right)
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/expr.py:504, in BaseExprVisitor._maybe_evaluate_binop(self, op, op_class, lhs, rhs, eval_in_python, maybe_eval_in_python)
495 def _maybe_evaluate_binop(
496 self,
497 op,
(...)
502 maybe_eval_in_python=("==", "!=", "<", ">", "<=", ">="),
503 ):
--> 504 res = op(lhs, rhs)
506 if res.has_invalid_return_type:
507 raise TypeError(
508 f"unsupported operand type(s) for {res.op}: "
509 f"'{lhs.type}' and '{rhs.type}'"
510 )
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/ops.py:380, in BinOp.__init__(self, op, lhs, rhs)
376 self.rhs = rhs
378 self._disallow_scalar_only_bool_ops()
--> 380 self.convert_values()
382 try:
383 self.func = _binary_ops_dict[op]
File ~/venv/lib/python3.8/site-packages/pandas/core/computation/ops.py:478, in BinOp.convert_values(self)
476 if isinstance(v, (int, float)):
477 v = stringify(v)
--> 478 v = Timestamp(ensure_decoded(v))
479 if v.tz is not None:
480 v = v.tz_convert("UTC")
File ~/venv/lib/python3.8/site-packages/pandas/_libs/tslibs/timestamps.pyx:1667, in pandas._libs.tslibs.timestamps.Timestamp.__new__()
File ~/venv/lib/python3.8/site-packages/pandas/_libs/tslibs/conversion.pyx:336, in pandas._libs.tslibs.conversion.convert_to_tsobject()
TypeError: Cannot convert input [0 days 01:00:00] of type to Timestamp
Expected Behavior
The expressions given in example could pass and produce correct results.
Installed Versions
INSTALLED VERSIONS
------------------
commit : 478d340667831908b5b4bf09a2787a11a14560c9
python : 3.8.10.final.0
python-bits : 64
OS : Linux
OS-release : 5.19.10-rockchip64
Version : #22.08.2 SMP PREEMPT Wed Sep 21 19:15:09 UTC 2022
machine : aarch64
processor : aarch64
byteorder : little
LC_ALL : en_US.UTF-8
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8
pandas : 2.0.0
numpy : 1.24.2
pytz : 2023.3
dateutil : 2.8.2
setuptools : 44.0.0
pip : 23.0.1
Cython : None
pytest : None
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 3.1.2
IPython : 8.11.0
pandas_datareader: None
bs4 : 4.12.0
bottleneck : None
brotli : None
fastparquet : None
fsspec : None
gcsfs : None
matplotlib : None
numba : None
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : None
pyreadstat : None
pyxlsb : None
s3fs : None
scipy : None
snappy : None
sqlalchemy : None
tables : None
tabulate : None
xarray : None
xlrd : None
zstandard : None
tzdata : 2023.3
qtpy : None
pyqt5 : None