diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 14cd725c8f066..aabd0d2f1274a 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -2239,11 +2239,26 @@ def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
                     casted = element.astype(dtype)
                     comp = casted == element
                     if comp.all():
-                        return element
+                        # Return the casted values bc they can be passed to
+                        #  np.putmask, whereas the raw values cannot.
+                        #  see TestSetitemFloatNDarrayIntoIntegerSeries
+                        return casted
                     raise ValueError
 
                 # Anything other than integer we cannot hold
                 raise ValueError
+            elif (
+                dtype.kind == "u"
+                and isinstance(element, np.ndarray)
+                and element.dtype.kind == "i"
+            ):
+                # see test_where_uint64
+                casted = element.astype(dtype)
+                if (casted == element).all():
+                    # TODO: faster to check (element >=0).all()?  potential
+                    #  itemsize issues there?
+                    return casted
+                raise ValueError
             elif dtype.itemsize < tipo.itemsize:
                 if is_integer(element):
                     # e.g. test_setitem_series_int8 if we have a python int 1
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 6c7499a575fca..94b3c216cb14b 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -69,6 +69,7 @@
     find_common_type,
     infer_dtype_from,
     maybe_cast_pointwise_result,
+    np_can_hold_element,
 )
 from pandas.core.dtypes.common import (
     ensure_int64,
@@ -4912,6 +4913,13 @@ def _validate_fill_value(self, value):
         TypeError
             If the value cannot be inserted into an array of this dtype.
         """
+        dtype = self.dtype
+        if isinstance(dtype, np.dtype) and dtype.kind not in ["m", "M"]:
+            try:
+                return np_can_hold_element(dtype, value)
+            except ValueError as err:
+                # re-raise as TypeError for consistency
+                raise TypeError from err
         if not can_hold_element(self._values, value):
             raise TypeError
         return value
diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
index 5cd4cc9dfaec2..ec2ba7413f414 100644
--- a/pandas/core/indexes/numeric.py
+++ b/pandas/core/indexes/numeric.py
@@ -419,18 +419,6 @@ def asi8(self) -> npt.NDArray[np.int64]:
         )
         return self._values.view(self._default_dtype)
 
-    def _validate_fill_value(self, value):
-        # e.g. np.array([1.0]) we want np.array([1], dtype=self.dtype)
-        #  see TestSetitemFloatNDarrayIntoIntegerSeries
-        super()._validate_fill_value(value)
-        if hasattr(value, "dtype") and is_float_dtype(value.dtype):
-            converted = value.astype(self.dtype)
-            if (converted == value).all():
-                # See also: can_hold_element
-                return converted
-            raise TypeError
-        return value
-
 
 class Int64Index(IntegerIndex):
     _index_descr_args = {
@@ -461,16 +449,6 @@ class UInt64Index(IntegerIndex):
     _default_dtype = np.dtype(np.uint64)
     _dtype_validation_metadata = (is_unsigned_integer_dtype, "unsigned integer")
 
-    def _validate_fill_value(self, value):
-        # e.g. np.array([1]) we want np.array([1], dtype=np.uint64)
-        #  see test_where_uin64
-        super()._validate_fill_value(value)
-        if hasattr(value, "dtype") and is_signed_integer_dtype(value.dtype):
-            if (value >= 0).all():
-                return value.astype(self.dtype)
-            raise TypeError
-        return value
-
 
 class Float64Index(NumericIndex):
     _index_descr_args = {