small fix

Author: Terji Petersen

pandas/core/indexes/base.py

@@ -5540,8 +5540,7 @@ def _should_fallback_to_positional(self) -> bool:
         if is_integer_dtype(self):
             return False
         elif is_object_dtype(self):
-            inferred_dtype = lib.infer_dtype(self._values, skipna=False)
-            return inferred_dtype not in ["integer", "mixed-integer"]
+            return self.inferred_type not in ["integer", "mixed-integer"]
         else:
             return True