From db8d920935cacb40252465e5b59520a667c924cd Mon Sep 17 00:00:00 2001 From: Chuck Cadman Date: Thu, 3 Nov 2022 11:59:40 -0700 Subject: [PATCH 1/2] TST: postgres tz-aware timestamps can be parsed by read_sql_table, not read_sql_query. --- pandas/tests/io/test_sql.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f89a06636a0c0..93fae76dc9c9b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1781,7 +1781,7 @@ def check(col): ) # GH11216 - df = read_sql_query("select * from types", self.conn) + df = read_sql_table("types", self.conn) if not hasattr(df, "DateColWithTz"): request.node.add_marker( pytest.mark.xfail(reason="no column with datetime with time zone") @@ -1793,9 +1793,7 @@ def check(col): col = df.DateColWithTz assert is_datetime64tz_dtype(col.dtype) - df = read_sql_query( - "select * from types", self.conn, parse_dates=["DateColWithTz"] - ) + df = read_sql_table("types", self.conn, parse_dates=["DateColWithTz"]) if not hasattr(df, "DateColWithTz"): request.node.add_marker( pytest.mark.xfail(reason="no column with datetime with time zone") @@ -1806,7 +1804,7 @@ def check(col): check(df.DateColWithTz) df = concat( - list(read_sql_query("select * from types", self.conn, chunksize=1)), + list(read_sql_table("types", self.conn, chunksize=1)), ignore_index=True, ) col = df.DateColWithTz From 021105dfa889dd9d1b5d389e5e4c90fd09447161 Mon Sep 17 00:00:00 2001 From: Chuck Cadman Date: Thu, 3 Nov 2022 12:04:09 -0700 Subject: [PATCH 2/2] BUG: Allow tz-aware Datetime SQL columns to be passed to parse_dates kwarg. --- pandas/io/sql.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 8bcee36cbef2b..8f01b05f6d280 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -1121,6 +1121,9 @@ def _harmonize_columns(self, parse_dates=None) -> None: try: df_col = self.frame[col_name] + # the type the dataframe column should have + col_type = self._get_dtype(sql_col.type) + # Handle date parsing upfront; don't try to convert columns # twice if col_name in parse_dates: @@ -1128,12 +1131,13 @@ def _harmonize_columns(self, parse_dates=None) -> None: fmt = parse_dates[col_name] except TypeError: fmt = None - self.frame[col_name] = _handle_date_column(df_col, format=fmt) + # Convert tz-aware Datetime SQL columns to UTC + utc = col_type is DatetimeTZDtype + self.frame[col_name] = _handle_date_column( + df_col, format=fmt, utc=utc + ) continue - # the type the dataframe column should have - col_type = self._get_dtype(sql_col.type) - if ( col_type is datetime or col_type is date