diff --git a/pandas/_libs/tslibs/c_timestamp.pyx b/pandas/_libs/tslibs/c_timestamp.pyx
index 906dabba09486..10ed2588deaca 100644
--- a/pandas/_libs/tslibs/c_timestamp.pyx
+++ b/pandas/_libs/tslibs/c_timestamp.pyx
@@ -269,15 +269,8 @@ cdef class _Timestamp(datetime):
             return self + neg_other
 
         typ = getattr(other, '_typ', None)
-
-        # a Timestamp-DatetimeIndex -> yields a negative TimedeltaIndex
-        if typ in ('datetimeindex', 'datetimearray'):
-            # timezone comparison is performed in DatetimeIndex._sub_datelike
-            return -other.__sub__(self)
-
-        # a Timestamp-TimedeltaIndex -> yields a negative TimedeltaIndex
-        elif typ in ('timedeltaindex', 'timedeltaarray'):
-            return (-other).__add__(self)
+        if typ is not None:
+            return NotImplemented
 
         elif other is NaT:
             return NaT
diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
index 1988726edc79b..bda5f8f4326f1 100644
--- a/pandas/core/arrays/datetimelike.py
+++ b/pandas/core/arrays/datetimelike.py
@@ -1300,7 +1300,7 @@ def __sub__(self, other):
         return result
 
     def __rsub__(self, other):
-        if is_datetime64_any_dtype(other) and is_timedelta64_dtype(self):
+        if is_datetime64_any_dtype(other) and is_timedelta64_dtype(self.dtype):
             # ndarray[datetime64] cannot be subtracted from self, so
             # we need to wrap in DatetimeArray/Index and flip the operation
             if not isinstance(other, DatetimeLikeArrayMixin):
@@ -1310,9 +1310,9 @@ def __rsub__(self, other):
                 other = DatetimeArray(other)
             return other - self
         elif (
-            is_datetime64_any_dtype(self)
+            is_datetime64_any_dtype(self.dtype)
             and hasattr(other, "dtype")
-            and not is_datetime64_any_dtype(other)
+            and not is_datetime64_any_dtype(other.dtype)
         ):
             # GH#19959 datetime - datetime is well-defined as timedelta,
             # but any other type - datetime is not well-defined.
@@ -1321,13 +1321,21 @@ def __rsub__(self, other):
                     cls=type(self).__name__, typ=type(other).__name__
                 )
             )
-        elif is_period_dtype(self) and is_timedelta64_dtype(other):
+        elif is_period_dtype(self.dtype) and is_timedelta64_dtype(other):
             # TODO: Can we simplify/generalize these cases at all?
             raise TypeError(
                 "cannot subtract {cls} from {dtype}".format(
                     cls=type(self).__name__, dtype=other.dtype
                 )
             )
+        elif is_timedelta64_dtype(self.dtype):
+            if lib.is_integer(other) or is_integer_dtype(other):
+                # need to subtract before negating, since that flips freq
+                # -self flips self.freq, messing up results
+                return -(self - other)
+
+            return (-self) + other
+
         return -(self - other)
 
     # FIXME: DTA/TDA/PA inplace methods should actually be inplace, GH#24115