Merge branch 'master' of https://github.com/pandas-dev/pandas into ERR/imporve_error_message_cut/qcut

Author: ryankarlos

ci/deps/azure-36-locale.yaml

@@ -9,6 +9,7 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
+  - pytest-asyncio
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-37-locale.yaml

@@ -8,6 +8,7 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
+  - pytest-asyncio
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

doc/source/whatsnew/v1.0.0.rst

@@ -1016,6 +1016,7 @@ Other
 - Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`)
 - :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`)
 - Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`)
+- Backtick quoting in :meth:`DataFrame.query` and :meth:`DataFrame.eval` can now also be used to use invalid identifiers like names that start with a digit, are python keywords, or are using single character operators. (:issue:`27017`)
 - Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`)
 - Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`)
 - Fix :class:`AbstractHolidayCalendar` to return correct results for

environment.yml

@@ -55,6 +55,7 @@ dependencies:
   - pytest>=5.0.1
   - pytest-cov
   - pytest-xdist>=1.21
+  - pytest-asyncio
 
   # downstream tests
   - seaborn

pandas/core/arrays/period.py

@@ -29,6 +29,7 @@
     is_datetime64_dtype,
     is_float_dtype,
     is_list_like,
+    is_object_dtype,
     is_period_dtype,
     pandas_dtype,
 )
@@ -41,6 +42,7 @@
 )
 from pandas.core.dtypes.missing import isna, notna
 
+from pandas.core import ops
 import pandas.core.algorithms as algos
 from pandas.core.arrays import datetimelike as dtl
 import pandas.core.common as com
@@ -92,22 +94,44 @@ def wrapper(self, other):
             self._check_compatible_with(other)
 
             result = ordinal_op(other.ordinal)
-        elif isinstance(other, cls):
-            self._check_compatible_with(other)
-
-            result = ordinal_op(other.asi8)
-
-            mask = self._isnan | other._isnan
-            if mask.any():
-                result[mask] = nat_result
 
-            return result
         elif other is NaT:
             result = np.empty(len(self.asi8), dtype=bool)
             result.fill(nat_result)
-        else:
+
+        elif not is_list_like(other):
             return invalid_comparison(self, other, op)
 
+        else:
+            if isinstance(other, list):
+                # TODO: could use pd.Index to do inference?
+                other = np.array(other)
+
+            if not isinstance(other, (np.ndarray, cls)):
+                return invalid_comparison(self, other, op)
+
+            if is_object_dtype(other):
+                with np.errstate(all="ignore"):
+                    result = ops.comp_method_OBJECT_ARRAY(
+                        op, self.astype(object), other
+                    )
+                o_mask = isna(other)
+
+            elif not is_period_dtype(other):
+                # e.g. is_timedelta64_dtype(other)
+                return invalid_comparison(self, other, op)
+
+            else:
+                assert isinstance(other, cls), type(other)
+
+                self._check_compatible_with(other)
+
+                result = ordinal_op(other.asi8)
+                o_mask = other._isnan
+
+            if o_mask.any():
+                result[o_mask] = nat_result
+
         if self._hasnans:
             result[self._isnan] = nat_result
 
@@ -215,12 +239,7 @@ def __init__(self, values, freq=None, dtype=None, copy=False):
 
         if isinstance(values, type(self)):
             if freq is not None and freq != values.freq:
-                msg = DIFFERENT_FREQ.format(
-                    cls=type(self).__name__,
-                    own_freq=values.freq.freqstr,
-                    other_freq=freq.freqstr,
-                )
-                raise IncompatibleFrequency(msg)
+                raise raise_on_incompatible(values, freq)
             values, freq = values._data, values.freq
 
         values = np.array(values, dtype="int64", copy=copy)
@@ -323,7 +342,7 @@ def _check_compatible_with(self, other):
         if other is NaT:
             return
         if self.freqstr != other.freqstr:
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
     # --------------------------------------------------------------------
     # Data / Attributes
@@ -682,7 +701,7 @@ def _add_offset(self, other):
         assert not isinstance(other, Tick)
         base = libfrequencies.get_base_alias(other.rule_code)
         if base != self.freq.rule_code:
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
         # Note: when calling parent class's _add_timedeltalike_scalar,
         #  it will call delta_to_nanoseconds(delta).  Because delta here
@@ -750,7 +769,7 @@ def _add_delta(self, other):
         """
         if not isinstance(self.freq, Tick):
             # We cannot add timedelta-like to non-tick PeriodArray
-            _raise_on_incompatible(self, other)
+            raise raise_on_incompatible(self, other)
 
         new_ordinals = super()._add_delta(other)
         return type(self)(new_ordinals, freq=self.freq)
@@ -802,28 +821,29 @@ def _check_timedeltalike_freq_compat(self, other):
             # by which will be added to self.
             return delta
 
-        _raise_on_incompatible(self, other)
+        raise raise_on_incompatible(self, other)
 
 
 PeriodArray._add_comparison_ops()
 
 
-def _raise_on_incompatible(left, right):
+def raise_on_incompatible(left, right):
     """
     Helper function to render a consistent error message when raising
     IncompatibleFrequency.
 
     Parameters
     ----------
     left : PeriodArray
-    right : DateOffset, Period, ndarray, or timedelta-like
+    right : None, DateOffset, Period, ndarray, or timedelta-like
 
-    Raises
+    Returns
     ------
     IncompatibleFrequency
+        Exception to be raised by the caller.
     """
     # GH#24283 error message format depends on whether right is scalar
-    if isinstance(right, np.ndarray):
+    if isinstance(right, np.ndarray) or right is None:
         other_freq = None
     elif isinstance(right, (ABCPeriodIndex, PeriodArray, Period, DateOffset)):
         other_freq = right.freqstr
@@ -833,7 +853,7 @@ def _raise_on_incompatible(left, right):
     msg = DIFFERENT_FREQ.format(
         cls=type(left).__name__, own_freq=left.freqstr, other_freq=other_freq
     )
-    raise IncompatibleFrequency(msg)
+    return IncompatibleFrequency(msg)
 
 
 # -------------------------------------------------------------------

pandas/core/computation/common.py

@@ -4,9 +4,6 @@
 
 from pandas._config import get_option
 
-# A token value Python's tokenizer probably will never use.
-_BACKTICK_QUOTED_STRING = 100
-
 
 def _ensure_decoded(s):
     """
@@ -29,16 +26,5 @@ def result_type_many(*arrays_and_dtypes):
         return reduce(np.result_type, arrays_and_dtypes)
 
 
-def _remove_spaces_column_name(name):
-    """
-    Check if name contains any spaces, if it contains any spaces
-    the spaces will be removed and an underscore suffix is added.
-    """
-    if not isinstance(name, str) or " " not in name:
-        return name
-
-    return name.replace(" ", "_") + "_BACKTICK_QUOTED_STRING"
-
-
 class NameResolutionError(NameError):
     pass

pandas/core/computation/eval.py

@@ -12,7 +12,8 @@
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.computation.engines import _engines
-from pandas.core.computation.expr import Expr, _parsers, tokenize_string
+from pandas.core.computation.expr import Expr, _parsers
+from pandas.core.computation.parsing import tokenize_string
 from pandas.core.computation.scope import ensure_scope
 
 from pandas.io.formats.printing import pprint_thing

pandas/core/computation/expr.py

@@ -3,19 +3,13 @@
 
 import ast
 from functools import partial, reduce
-from io import StringIO
-import itertools as it
-import operator
+from keyword import iskeyword
 import tokenize
 from typing import Optional, Type
 
 import numpy as np
 
 import pandas.core.common as com
-from pandas.core.computation.common import (
-    _BACKTICK_QUOTED_STRING,
-    _remove_spaces_column_name,
-)
 from pandas.core.computation.ops import (
     _LOCAL_TAG,
     BinOp,
@@ -34,38 +28,12 @@
     _unary_ops_syms,
     is_term,
 )
+from pandas.core.computation.parsing import clean_backtick_quoted_toks, tokenize_string
 from pandas.core.computation.scope import Scope
 
 import pandas.io.formats.printing as printing
 
 
-def tokenize_string(source: str):
-    """
-    Tokenize a Python source code string.
-
-    Parameters
-    ----------
-    source : str
-        A Python source code string
-    """
-    line_reader = StringIO(source).readline
-    token_generator = tokenize.generate_tokens(line_reader)
-
-    # Loop over all tokens till a backtick (`) is found.
-    # Then, take all tokens till the next backtick to form a backtick quoted
-    # string.
-    for toknum, tokval, _, _, _ in token_generator:
-        if tokval == "`":
-            tokval = " ".join(
-                it.takewhile(
-                    lambda tokval: tokval != "`",
-                    map(operator.itemgetter(1), token_generator),
-                )
-            )
-            toknum = _BACKTICK_QUOTED_STRING
-        yield toknum, tokval
-
-
 def _rewrite_assign(tok):
     """Rewrite the assignment operator for PyTables expressions that use ``=``
     as a substitute for ``==``.
@@ -133,31 +101,6 @@ def _replace_locals(tok):
     return toknum, tokval
 
 
-def _clean_spaces_backtick_quoted_names(tok):
-    """Clean up a column name if surrounded by backticks.
-
-    Backtick quoted string are indicated by a certain tokval value. If a string
-    is a backtick quoted token it will processed by
-    :func:`_remove_spaces_column_name` so that the parser can find this
-    string when the query is executed.
-    See also :meth:`NDFrame._get_space_character_free_column_resolver`.
-
-    Parameters
-    ----------
-    tok : tuple of int, str
-        ints correspond to the all caps constants in the tokenize module
-
-    Returns
-    -------
-    t : tuple of int, str
-        Either the input or token or the replacement values
-    """
-    toknum, tokval = tok
-    if toknum == _BACKTICK_QUOTED_STRING:
-        return tokenize.NAME, _remove_spaces_column_name(tokval)
-    return toknum, tokval
-
-
 def _compose2(f, g):
     """Compose 2 callables"""
     return lambda *args, **kwargs: f(g(*args, **kwargs))
@@ -172,10 +115,7 @@ def _compose(*funcs):
 def _preparse(
     source: str,
     f=_compose(
-        _replace_locals,
-        _replace_booleans,
-        _rewrite_assign,
-        _clean_spaces_backtick_quoted_names,
+        _replace_locals, _replace_booleans, _rewrite_assign, clean_backtick_quoted_toks
     ),
 ):
     """Compose a collection of tokenization functions
@@ -426,8 +366,6 @@ def visit(self, node, **kwargs):
             try:
                 node = ast.fix_missing_locations(ast.parse(clean))
             except SyntaxError as e:
-                from keyword import iskeyword
-
                 if any(iskeyword(x) for x in clean.split()):
                     e.msg = "Python keyword not valid identifier in numexpr query"
                 raise e
@@ -781,9 +719,7 @@ def __init__(
         parser,
         preparser=partial(
             _preparse,
-            f=_compose(
-                _replace_locals, _replace_booleans, _clean_spaces_backtick_quoted_names
-            ),
+            f=_compose(_replace_locals, _replace_booleans, clean_backtick_quoted_toks),
         ),
     ):
         super().__init__(env, engine, parser, preparser)