diff --git a/pandas/tests/computation/test_eval.py b/pandas/tests/computation/test_eval.py
index 5c614dac2bcb9..4ff4cb69d25ac 100644
--- a/pandas/tests/computation/test_eval.py
+++ b/pandas/tests/computation/test_eval.py
@@ -13,6 +13,7 @@
 
 from pandas.core.dtypes.common import (
     is_bool,
+    is_float,
     is_list_like,
     is_scalar,
 )
@@ -121,7 +122,10 @@ def _is_py3_complex_incompat(result, expected):
 
 
 # TODO: using range(5) here is a kludge
-@pytest.fixture(params=list(range(5)))
+@pytest.fixture(
+    params=list(range(5)),
+    ids=["DataFrame", "Series", "SeriesNaN", "DataFrameNaN", "float"],
+)
 def lhs(request):
 
     nan_df1 = DataFrame(np.random.rand(10, 5))
@@ -168,7 +172,12 @@ def current_engines(self):
     @pytest.mark.parametrize("binop", expr.BOOL_OPS_SYMS)
     def test_complex_cmp_ops(self, cmp1, cmp2, binop, lhs, rhs):
         if binop in self.exclude_bool:
-            pytest.skip()
+            # i.e. "&" and "|"
+            msg = "'BoolOp' nodes are not implemented"
+            with pytest.raises(NotImplementedError, match=msg):
+                ex = f"(lhs {cmp1} rhs) {binop} (lhs {cmp2} rhs)"
+                result = pd.eval(ex, engine=self.engine, parser=self.parser)
+            return
 
         lhs_new = _eval_single_bin(lhs, cmp1, rhs, self.engine)
         rhs_new = _eval_single_bin(lhs, cmp2, rhs, self.engine)
@@ -180,9 +189,6 @@ def test_complex_cmp_ops(self, cmp1, cmp2, binop, lhs, rhs):
 
     @pytest.mark.parametrize("cmp_op", expr.CMP_OPS_SYMS)
     def test_simple_cmp_ops(self, cmp_op):
-        if cmp_op in self.exclude_cmp:
-            pytest.skip()
-
         bool_lhses = (
             DataFrame(tm.randbool(size=(10, 5))),
             Series(tm.randbool((5,))),
@@ -193,6 +199,15 @@ def test_simple_cmp_ops(self, cmp_op):
             Series(tm.randbool((5,))),
             tm.randbool(),
         )
+
+        if cmp_op in self.exclude_cmp:
+            msg = "'(In|NotIn)' nodes are not implemented"
+            for lhs, rhs in product(bool_lhses, bool_rhses):
+
+                with pytest.raises(NotImplementedError, match=msg):
+                    self.check_simple_cmp_op(lhs, cmp_op, rhs)
+            return
+
         for lhs, rhs in product(bool_lhses, bool_rhses):
             self.check_simple_cmp_op(lhs, cmp_op, rhs)
 
@@ -213,15 +228,29 @@ def test_pow(self, lhs, rhs):
 
     @pytest.mark.parametrize("op", expr.CMP_OPS_SYMS)
     def test_single_invert_op(self, op, lhs):
-        if op in self.exclude_cmp:
-            pytest.skip()
-
         self.check_single_invert_op(lhs, op)
 
     @pytest.mark.parametrize("op", expr.CMP_OPS_SYMS)
-    def test_compound_invert_op(self, op, lhs, rhs):
+    def test_compound_invert_op(self, op, lhs, rhs, request):
         if op in self.exclude_cmp:
-            pytest.skip()
+
+            msg = "'(In|NotIn)' nodes are not implemented"
+            with pytest.raises(NotImplementedError, match=msg):
+                self.check_compound_invert_op(lhs, op, rhs)
+            return
+
+        if (
+            is_float(lhs)
+            and not is_float(rhs)
+            and op in ["in", "not in"]
+            and self.engine == "python"
+            and self.parser == "pandas"
+        ):
+            mark = pytest.mark.xfail(
+                reason="Looks like expected is negative, unclear whether "
+                "expected is incorrect or result is incorrect"
+            )
+            request.node.add_marker(mark)
 
         self.check_compound_invert_op(lhs, op, rhs)
 
@@ -753,8 +782,8 @@ def test_disallow_python_keywords(self):
 
 @td.skip_if_no_ne
 class TestEvalNumexprPython(TestEvalNumexprPandas):
-    exclude_cmp = ["in", "not in"]
-    exclude_bool = ["and", "or"]
+    exclude_cmp: list[str] = ["in", "not in"]
+    exclude_bool: list[str] = ["and", "or"]
 
     engine = "numexpr"
     parser = "python"
@@ -802,6 +831,8 @@ def check_alignment(self, result, nlhs, ghs, op):
 class TestEvalPythonPandas(TestEvalPythonPython):
     engine = "python"
     parser = "pandas"
+    exclude_bool: list[str] = []
+    exclude_cmp: list[str] = []
 
     def check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs):
         TestEvalNumexprPandas.check_chained_cmp_op(self, lhs, cmp1, mid, cmp2, rhs)