diff --git a/pandas/core/construction.py b/pandas/core/construction.py
index f3133480108a6..9523c2e3e62e0 100644
--- a/pandas/core/construction.py
+++ b/pandas/core/construction.py
@@ -682,6 +682,13 @@ def _try_cast(
         subarr = construct_1d_object_array_from_listlike(arr)
         return subarr
 
+    if dtype is None and isinstance(arr, list):
+        # filter out cases that we _dont_ want to go through maybe_cast_to_datetime
+        varr = np.array(arr, copy=False)
+        if varr.dtype != object or varr.size == 0:
+            return varr
+        arr = varr
+
     try:
         # GH#15832: Check if we are requesting a numeric dtype and
         # that we can convert the data to the requested dtype.
diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index 46dc97214e2f6..3cc54fb912014 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -1475,7 +1475,9 @@ def maybe_castable(dtype: np.dtype) -> bool:
     return dtype.name not in POSSIBLY_CAST_DTYPES
 
 
-def maybe_infer_to_datetimelike(value: np.ndarray | list):
+def maybe_infer_to_datetimelike(
+    value: np.ndarray,
+) -> np.ndarray | DatetimeArray | TimedeltaArray:
     """
     we might have a array (or single object) that is datetime like,
     and no dtype is passed don't change the value unless we find a
@@ -1486,18 +1488,19 @@ def maybe_infer_to_datetimelike(value: np.ndarray | list):
 
     Parameters
     ----------
-    value : np.ndarray or list
+    value : np.ndarray[object]
+
+    Returns
+    -------
+    np.ndarray, DatetimeArray, or TimedeltaArray
 
     """
-    if not isinstance(value, (np.ndarray, list)):
+    if not isinstance(value, np.ndarray) or value.dtype != object:
+        # Caller is responsible for passing only ndarray[object]
         raise TypeError(type(value))  # pragma: no cover
 
     v = np.array(value, copy=False)
 
-    # we only care about object dtypes
-    if not is_object_dtype(v.dtype):
-        return value
-
     shape = v.shape
     if v.ndim != 1:
         v = v.ravel()
@@ -1575,6 +1578,8 @@ def maybe_cast_to_datetime(
     """
     try to cast the array/value to a datetimelike dtype, converting float
     nan to iNaT
+
+    We allow a list *only* when dtype is not None.
     """
     from pandas.core.arrays.datetimes import sequence_to_datetimes
     from pandas.core.arrays.timedeltas import sequence_to_td64ns
@@ -1666,11 +1671,10 @@ def maybe_cast_to_datetime(
             value = maybe_infer_to_datetimelike(value)
 
     elif isinstance(value, list):
-        # only do this if we have an array and the dtype of the array is not
-        # setup already we are not an integer/object, so don't bother with this
-        # conversion
-
-        value = maybe_infer_to_datetimelike(value)
+        # we only get here with dtype=None, which we do not allow
+        raise ValueError(
+            "maybe_cast_to_datetime allows a list *only* if dtype is not None"
+        )
 
     return value
 
diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py
index 00efc695ff04a..ba9d36467844c 100644
--- a/pandas/core/internals/construction.py
+++ b/pandas/core/internals/construction.py
@@ -357,8 +357,8 @@ def ndarray_to_mgr(
         if values.ndim == 2 and values.shape[0] != 1:
             # transpose and separate blocks
 
-            dvals_list = [maybe_infer_to_datetimelike(row) for row in values]
-            dvals_list = [ensure_block_shape(dval, 2) for dval in dvals_list]
+            dtlike_vals = [maybe_infer_to_datetimelike(row) for row in values]
+            dvals_list = [ensure_block_shape(dval, 2) for dval in dtlike_vals]
 
             # TODO: What about re-joining object columns?
             block_values = [