Address whatsnew changes & fix sql tests

Author: mroeschke

doc/source/whatsnew/v0.21.0.txt

@@ -303,6 +303,8 @@ length 2+ levels, so a :class:`MultiIndex` is always returned from all of the
 UTC Localization with Series
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+Previously, :func:`to_datetime` did not localize datetime ``Series`` data as when ``utc=True`` was passed. Now, :func:`to_datetime` will correctly localize `Series` with a `datetime64[ns, UTC]` data type. (:issue:`6415`).
+
   Previous Behavior
 
   .. ipython:: python
@@ -331,7 +333,7 @@ UTC Localization with Series
 
      pd.to_datetime(s, utc=True)
 
-This new behavior will also localize datetime columns in DataFrames returned from :func:`read_sql` which previously returned datetime columns as naive UTC.
+Additionally, DataFrames with datetime columns returned by :func:`read_sql` will also be localized to UTC only if the original SQL columns were timezone aware datetime columns.
 
 .. _whatsnew_0210.api:
 

pandas/io/sql.py

@@ -99,24 +99,24 @@ def _convert_params(sql, params):
     return args
 
 
-def _handle_date_column(col, format=None):
+def _handle_date_column(col, utc=None, format=None):
     if isinstance(format, dict):
         return to_datetime(col, errors='ignore', **format)
     else:
         if format in ['D', 's', 'ms', 'us', 'ns']:
-            return to_datetime(col, errors='coerce', unit=format, utc=True)
+            return to_datetime(col, errors='coerce', unit=format, utc=utc)
         elif (issubclass(col.dtype.type, np.floating) or
               issubclass(col.dtype.type, np.integer)):
             # parse dates as timestamp
             format = 's' if format is None else format
-            return to_datetime(col, errors='coerce', unit=format, utc=True)
+            return to_datetime(col, errors='coerce', unit=format, utc=utc)
         elif is_datetime64tz_dtype(col):
             # coerce to UTC timezone
             # GH11216
             return (to_datetime(col, errors='coerce')
                     .astype('datetime64[ns, UTC]'))
         else:
-            return to_datetime(col, errors='coerce', format=format, utc=True)
+            return to_datetime(col, errors='coerce', format=format, utc=utc)
 
 
 def _parse_date_columns(data_frame, parse_dates):
@@ -818,10 +818,14 @@ def _harmonize_columns(self, parse_dates=None):
                 df_col = self.frame[col_name]
                 # the type the dataframe column should have
                 col_type = self._get_dtype(sql_col.type)
-
                 if (col_type is datetime or col_type is date or
                         col_type is DatetimeTZDtype):
-                    self.frame[col_name] = _handle_date_column(df_col)
+                    if col_type is DatetimeTZDtype:
+                        # Convert SQL Datetime columns with tz to UTC
+                        self.frame[col_name] = _handle_date_column(df_col, 
+                                                                   utc=True)
+                    else:
+                        self.frame[col_name] = _handle_date_column(df_col)
 
                 elif col_type is float:
                     # floats support NA, can always convert!

pandas/tests/io/test_sql.py

@@ -606,15 +606,14 @@ def test_date_parsing(self):
         # No Parsing
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn)
         assert not issubclass(df.DateCol.dtype.type, np.datetime64)
-        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
-        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
+
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates=['DateCol'])
-        assert issubclass(df.DateCol.dtype.type, utc_dtype)
+        assert issubclass(df.DateCol.dtype.type, np.datetime64)
 
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates={'DateCol': '%Y-%m-%d %H:%M:%S'})
-        assert issubclass(df.DateCol.dtype.type, utc_dtype)
+        assert issubclass(df.DateCol.dtype.type, np.datetime64)
 
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 parse_dates=['IntDateCol'])
@@ -632,9 +631,8 @@ def test_date_and_index(self):
         df = sql.read_sql_query("SELECT * FROM types_test_data", self.conn,
                                 index_col='DateCol',
                                 parse_dates=['DateCol', 'IntDateCol'])
-        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
-        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
-        assert issubclass(df.index.dtype.type, utc_dtype)
+
+        assert issubclass(df.index.dtype.type, np.datetime64)
         assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
 
     def test_timedelta(self):
@@ -1323,15 +1321,14 @@ def check(col):
     def test_date_parsing(self):
         # No Parsing
         df = sql.read_sql_table("types_test_data", self.conn)
-        # Now that GH 6415 is fixed, dates are automatically parsed to UTC
-        utc_dtype = pd.core.dtypes.dtypes.DatetimeTZDtypeType
+
         df = sql.read_sql_table("types_test_data", self.conn,
                                 parse_dates=['DateCol'])
-        assert issubclass(df.DateCol.dtype.type, utc_dtype)
+        assert issubclass(df.DateCol.dtype.type, np.datetime64)
 
         df = sql.read_sql_table("types_test_data", self.conn,
                                 parse_dates={'DateCol': '%Y-%m-%d %H:%M:%S'})
-        assert issubclass(df.DateCol.dtype.type, utc_dtype)
+        assert issubclass(df.DateCol.dtype.type, np.datetime64)
 
         df = sql.read_sql_table("types_test_data", self.conn, parse_dates={
             'DateCol': {'format': '%Y-%m-%d %H:%M:%S'}})
@@ -1357,11 +1354,7 @@ def test_datetime(self):
         # with read_table -> type information from schema used
         result = sql.read_sql_table('test_datetime', self.conn)
         result = result.drop('index', axis=1)
-        # After GH 6415, dates outbound from a db will be localized to UTC
-        # xref GH 7364
-        expected = df.copy()
-        expected['A'] = expected['A'].dt.tz_localize('UTC')
-        tm.assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, df)
 
         # with read_sql -> no type information -> sqlite has no native
         result = sql.read_sql_query('SELECT * FROM test_datetime', self.conn)
@@ -1381,11 +1374,7 @@ def test_datetime_NaT(self):
 
         # with read_table -> type information from schema used
         result = sql.read_sql_table('test_datetime', self.conn)
-        # After GH 6415, dates outbound from a db will be localized to UTC
-        # xref GH 7364
-        expected = df.copy()
-        expected['A'] = expected['A'].dt.tz_localize('UTC')
-        tm.assert_frame_equal(result, expected)
+        tm.assert_frame_equal(result, df)
 
         # with read_sql -> no type information -> sqlite has no native
         result = sql.read_sql_query('SELECT * FROM test_datetime', self.conn)
@@ -1401,8 +1390,10 @@ def test_datetime_date(self):
         df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"])
         df.to_sql('test_date', self.conn, index=False)
         res = read_sql_table('test_date', self.conn)
-        # GH 6415 comes back as datetime64[ns, UTC]
-        tm.assert_series_equal(res['a'], to_datetime(df['a'], utc=True))
+        expected = res['a']
+        result = to_datetime(df['a'])
+        # comes back as datetime64
+        tm.assert_series_equal(result, expected)
 
     def test_datetime_time(self):
         # test support for datetime.time