diff --git a/pandas/plotting/_matplotlib/core.py b/pandas/plotting/_matplotlib/core.py
index 2a1503b84a634..a322e83f5f501 100644
--- a/pandas/plotting/_matplotlib/core.py
+++ b/pandas/plotting/_matplotlib/core.py
@@ -924,7 +924,9 @@ def plt(self):
     @final
     def _get_xticks(self):
         index = self.data.index
-        is_datetype = index.inferred_type in ("datetime", "date", "datetime64", "time")
+        is_datetype = self.data.dtypes.isin(
+            ["datetime64[ns]", "datetime", "date", "time"]
+        ).any()
 
         # TODO: be stricter about x?
         x: list[int] | np.ndarray
@@ -1247,9 +1249,14 @@ def __init__(self, data, x, y, **kwargs) -> None:
         MPLPlot.__init__(self, data, **kwargs)
         if x is None or y is None:
             raise ValueError(self._kind + " requires an x and y column")
-        if is_integer(x) and not self.data.columns._holds_integer():
+        if self.data.columns._holds_integer():
+            raise Warning(
+                """_holds_integer is deprecated and may be removed in a future version.
+                Please cast to a int dtype before plotting."""
+            )
+        if is_integer(x) and not is_integer_dtype(self.data.columns):
             x = self.data.columns[x]
-        if is_integer(y) and not self.data.columns._holds_integer():
+        if is_integer(y) and not is_integer_dtype(self.data.columns):
             y = self.data.columns[y]
 
         self.x = x
@@ -1319,7 +1326,12 @@ def __init__(
         self.norm = norm
 
         super().__init__(data, x, y, **kwargs)
-        if is_integer(c) and not self.data.columns._holds_integer():
+        if self.data.columns._holds_integer():
+            raise Warning(
+                """_holds_integer is deprecated and may be removed in a future version.
+                Please cast to a int dtype before plotting."""
+            )
+        if is_integer(c) and not is_integer_dtype(self.data.columns):
             c = self.data.columns[c]
         self.c = c
 
@@ -1435,7 +1447,12 @@ def _kind(self) -> Literal["hexbin"]:
 
     def __init__(self, data, x, y, C=None, *, colorbar: bool = True, **kwargs) -> None:
         super().__init__(data, x, y, **kwargs)
-        if is_integer(C) and not self.data.columns._holds_integer():
+        if self.data.columns._holds_integer():
+            raise Warning(
+                """_holds_integer is deprecated and may be removed in a future version.
+                Please cast to a int dtype before plotting."""
+            )
+        if is_integer(C) and not is_integer_dtype(self.data.columns):
             C = self.data.columns[C]
         self.C = C