diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index 9c259dfd11793..6035131e78aa8 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -395,9 +395,7 @@ def _union(self, other: "DatetimeIndex", sort):
             result = Index._union(this, other, sort=sort)
             if isinstance(result, DatetimeIndex):
                 assert result._data.dtype == this.dtype
-                if result.freq is None and (
-                    this.freq is not None or other.freq is not None
-                ):
+                if result.freq is None:
                     result._set_freq("infer")
             return result
 
diff --git a/pandas/tests/indexes/datetimes/test_setops.py b/pandas/tests/indexes/datetimes/test_setops.py
index f7960c114ec9d..78188c54b1d85 100644
--- a/pandas/tests/indexes/datetimes/test_setops.py
+++ b/pandas/tests/indexes/datetimes/test_setops.py
@@ -163,6 +163,21 @@ def test_union_freq_both_none(self, sort):
         tm.assert_index_equal(result, expected)
         assert result.freq is None
 
+    def test_union_freq_infer(self):
+        # When taking the union of two DatetimeIndexes, we infer
+        #  a freq even if the arguments don't have freq.  This matches
+        #  TimedeltaIndex behavior.
+        dti = pd.date_range("2016-01-01", periods=5)
+        left = dti[[0, 1, 3, 4]]
+        right = dti[[2, 3, 1]]
+
+        assert left.freq is None
+        assert right.freq is None
+
+        result = left.union(right)
+        tm.assert_index_equal(result, dti)
+        assert result.freq == "D"
+
     def test_union_dataframe_index(self):
         rng1 = date_range("1/1/1999", "1/1/2012", freq="MS")
         s1 = Series(np.random.randn(len(rng1)), rng1)
diff --git a/pandas/tests/indexes/timedeltas/test_setops.py b/pandas/tests/indexes/timedeltas/test_setops.py
index e6d5ca97a0bdf..0aa784cbb7710 100644
--- a/pandas/tests/indexes/timedeltas/test_setops.py
+++ b/pandas/tests/indexes/timedeltas/test_setops.py
@@ -78,6 +78,21 @@ def test_union_bug_4564(self):
         exp = TimedeltaIndex(sorted(set(left) | set(right)))
         tm.assert_index_equal(result, exp)
 
+    def test_union_freq_infer(self):
+        # When taking the union of two TimedeltaIndexes, we infer
+        #  a freq even if the arguments don't have freq.  This matches
+        #  DatetimeIndex behavior.
+        tdi = pd.timedelta_range("1 Day", periods=5)
+        left = tdi[[0, 1, 3, 4]]
+        right = tdi[[2, 3, 1]]
+
+        assert left.freq is None
+        assert right.freq is None
+
+        result = left.union(right)
+        tm.assert_index_equal(result, tdi)
+        assert result.freq == "D"
+
     def test_intersection_bug_1708(self):
         index_1 = timedelta_range("1 day", periods=4, freq="h")
         index_2 = index_1 + pd.offsets.Hour(5)