skip normalizing problematic timestamp values

#786

Author: dlawin

data_diff/databases/redshift.py

@@ -54,6 +54,15 @@ def md5_as_hex(self, s: str) -> str:
     def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
         if coltype.rounds:
             timestamp = f"{value}::timestamp(6)"
+
+            # redshift allows some problematic timestamp values, don't normalize those (epoch calcs fail)
+            case_start = (
+                f"case when "
+                + f"({timestamp} < '1901-01-01 00:00'::timestamp(6) or {timestamp} >= '2038-01-01'::timestamp(6))"
+                + f"then {timestamp}::varchar else"
+            )
+            case_end = f"end"
+
             # Get seconds since epoch. Redshift doesn't support milli- or micro-seconds.
             secs = f"timestamp 'epoch' + round(extract(epoch from {timestamp})::decimal(38)"
             # Get the milliseconds from timestamp.
@@ -65,8 +74,14 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
             timestamp6 = (
                 f"to_char({epoch}, -6+{coltype.precision}) * interval '0.000001 seconds', 'YYYY-mm-dd HH24:MI:SS.US')"
             )
+            padded = self._add_padding(coltype, timestamp6)
+            return f"{case_start} {padded} {case_end}"
         else:
             timestamp6 = f"to_char({value}::timestamp(6), 'YYYY-mm-dd HH24:MI:SS.US')"
+            padded = self._add_padding(coltype, timestamp6)
+            return padded
+
+    def _add_padding(self, coltype: TemporalType, timestamp6: str):
         return (
             f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')"
         )