diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 891ae95db65a0..3c735fc0309b6 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -4590,9 +4590,9 @@ def get_value(self, series: "Series", key):
             # If that fails, raise a KeyError if an integer
             # index, otherwise, see if key is an integer, and
             # try that
-            loc = self._engine.get_loc(key)
+            loc = self.get_loc(key)
         except KeyError:
-            if len(self) > 0 and (self.holds_integer() or self.is_boolean()):
+            if not self._should_fallback_to_positional():
                 raise
             elif is_integer(key):
                 # If the Index cannot hold integer, then this is unambiguously
@@ -4603,6 +4603,14 @@ def get_value(self, series: "Series", key):
 
         return self._get_values_for_loc(series, loc)
 
+    def _should_fallback_to_positional(self) -> bool:
+        """
+        If an integer key is not found, should we fall back to positional indexing?
+        """
+        if len(self) > 0 and (self.holds_integer() or self.is_boolean()):
+            return False
+        return True
+
     def _get_values_for_loc(self, series: "Series", loc):
         """
         Do a positional lookup on the given Series, returning either a scalar
diff --git a/pandas/core/indexes/extension.py b/pandas/core/indexes/extension.py
index c32889a9360bc..66b551f654bf1 100644
--- a/pandas/core/indexes/extension.py
+++ b/pandas/core/indexes/extension.py
@@ -1,7 +1,7 @@
 """
 Shared methods for Index subclasses backed by ExtensionArray.
 """
-from typing import TYPE_CHECKING, List
+from typing import List
 
 import numpy as np
 
@@ -11,7 +11,6 @@
 from pandas.core.dtypes.common import (
     ensure_platform_int,
     is_dtype_equal,
-    is_integer,
     is_object_dtype,
 )
 from pandas.core.dtypes.generic import ABCSeries
@@ -21,9 +20,6 @@
 from pandas.core.indexes.base import Index
 from pandas.core.ops import get_op_result_name
 
-if TYPE_CHECKING:
-    from pandas import Series
-
 
 def inherit_from_data(name: str, delegate, cache: bool = False, wrap: bool = False):
     """
@@ -297,26 +293,3 @@ def astype(self, dtype, copy=True):
         # pass copy=False because any copying will be done in the
         #  _data.astype call above
         return Index(new_values, dtype=new_values.dtype, name=self.name, copy=False)
-
-    # --------------------------------------------------------------------
-    # Indexing Methods
-
-    @Appender(Index.get_value.__doc__)
-    def get_value(self, series: "Series", key):
-        """
-        Fast lookup of value from 1-dimensional ndarray. Only use this if you
-        know what you're doing
-        """
-        try:
-            loc = self.get_loc(key)
-        except KeyError:
-            # e.g. DatetimeIndex doesn't hold integers
-            if is_integer(key) and not self.holds_integer():
-                # Fall back to positional
-                loc = key
-            else:
-                raise
-
-        return self._get_values_for_loc(series, loc)
-
-    # --------------------------------------------------------------------
diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py
index 0252a13665b84..9ec72df140c85 100644
--- a/pandas/core/indexes/interval.py
+++ b/pandas/core/indexes/interval.py
@@ -523,9 +523,10 @@ def is_overlapping(self) -> bool:
         # GH 23309
         return self._engine.is_overlapping
 
-    def holds_integer(self):
-        return self.dtype.subtype.kind not in ["m", "M"]
-        # TODO: There must already exist something for this?
+    def _should_fallback_to_positional(self):
+        # integer lookups in Series.__getitem__ are unambiguously
+        #  positional in this case
+        return self.dtype.subtype.kind in ["m", "M"]
 
     @Appender(Index._convert_scalar_indexer.__doc__)
     def _convert_scalar_indexer(self, key, kind=None):
diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
index ebfe50327b479..2f4c48cc2e5a5 100644
--- a/pandas/core/indexes/numeric.py
+++ b/pandas/core/indexes/numeric.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import numpy as np
 
@@ -32,12 +32,9 @@
 
 from pandas.core import algorithms
 import pandas.core.common as com
-from pandas.core.indexes.base import Index, InvalidIndexError, maybe_extract_name
+from pandas.core.indexes.base import Index, maybe_extract_name
 from pandas.core.ops import get_op_result_name
 
-if TYPE_CHECKING:
-    from pandas import Series
-
 _num_index_shared_docs = dict()
 
 
@@ -383,6 +380,13 @@ def astype(self, dtype, copy=True):
             return Int64Index(arr)
         return super().astype(dtype, copy=copy)
 
+    # ----------------------------------------------------------------
+    # Indexing Methods
+
+    @Appender(Index._should_fallback_to_positional.__doc__)
+    def _should_fallback_to_positional(self):
+        return False
+
     @Appender(Index._convert_scalar_indexer.__doc__)
     def _convert_scalar_indexer(self, key, kind=None):
         assert kind in ["loc", "getitem", "iloc", None]
@@ -401,6 +405,8 @@ def _convert_slice_indexer(self, key: slice, kind=None):
         # translate to locations
         return self.slice_indexer(key.start, key.stop, key.step, kind=kind)
 
+    # ----------------------------------------------------------------
+
     def _format_native_types(
         self, na_rep="", float_format=None, decimal=".", quoting=None, **kwargs
     ):
@@ -416,17 +422,6 @@ def _format_native_types(
         )
         return formatter.get_result_as_array()
 
-    @Appender(Index.get_value.__doc__)
-    def get_value(self, series: "Series", key):
-        """
-        We always want to get an index value, never a value.
-        """
-        if not is_scalar(key):
-            raise InvalidIndexError
-
-        loc = self.get_loc(key)
-        return self._get_values_for_loc(series, loc)
-
     def equals(self, other) -> bool:
         """
         Determines if two Index objects contain the same elements.