Closed
Description
Pandas version checks
-
I have checked that this issue has not already been reported.
-
I have confirmed this bug exists on the latest version of pandas.
-
I have confirmed this bug exists on the main branch of pandas.
Reproducible Example
import pandas as pd
from datetime import datetime
df = pd.DataFrame({"integer": [1, 2, 3],
"date": [
(datetime(2022, 1, 1)),
(datetime(2022, 1, 2)),
(None)
],
"float":[4.0, 5.0, 6.0],
"strings": ["first", "two", "third"]
})
dfarrow = df.convert_dtypes(dtype_backend="pyarrow")
ENGINE_PATH = "sqlite:///mydb.db"
dfarrow.to_sql(name="MyTable_Arrow", con=ENGINE_PATH, if_exists="replace")Issue Description
Hello! The issue with pyarrow / to_sql / datetime has a new variant.
After #52046 was solved, I got today a new bug:
- I've installed Pandas from source, using branch 2.0.x and commit
7d5d123 - Using pyarrow backend
- If some collumn have dtype
datetimeand hasNonevalue in some line, to_sql crashes with:'NoneType' object has no attribute 'to_pydatetime'
Traceback
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/tmp/ipykernel_3158/3436422928.py in <cell line: 1>()
----> 1 dfarrow.to_sql(name="MyTable_Arrow", con=ENGINE_PATH, if_exists="replace")
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/core/generic.py in to_sql(self, name, con, schema, if_exists, index, index_label, chunksize, dtype, method)
2868 from pandas.io import sql
2869
-> 2870 return sql.to_sql(
2871 self,
2872 name,
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/io/sql.py in to_sql(frame, name, con, schema, if_exists, index, index_label, chunksize, dtype, method, engine, **engine_kwargs)
765
766 with pandasSQL_builder(con, schema=schema, need_transaction=True) as pandas_sql:
--> 767 return pandas_sql.to_sql(
768 frame,
769 name,
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/io/sql.py in to_sql(self, frame, name, if_exists, index, index_label, schema, chunksize, dtype, method, engine, **engine_kwargs)
1916 )
1917
-> 1918 total_inserted = sql_engine.insert_records(
1919 table=table,
1920 con=self.con,
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/io/sql.py in insert_records(self, table, con, frame, name, index, schema, chunksize, method, **engine_kwargs)
1457
1458 try:
-> 1459 return table.insert(chunksize=chunksize, method=method)
1460 except exc.StatementError as err:
1461 # GH34431
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/io/sql.py in insert(self, chunksize, method)
997 raise ValueError(f"Invalid parameter `method`: {method}")
998
--> 999 keys, data_list = self.insert_data()
1000
1001 nrows = len(self.frame)
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/io/sql.py in insert_data(self)
963 for i, (_, ser) in enumerate(temp.items()):
964 if ser.dtype.kind == "M":
--> 965 d = ser.dt.to_pydatetime()
966 elif ser.dtype.kind == "m":
967 vals = ser._values
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/core/indexes/accessors.py in to_pydatetime(self)
211
212 def to_pydatetime(self):
--> 213 return cast(ArrowExtensionArray, self._parent.array)._dt_to_pydatetime()
214
215 def isocalendar(self):
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/core/arrays/arrow/array.py in _dt_to_pydatetime(self)
2078 data = self._data.to_pylist()
2079 if self._dtype.pyarrow_dtype.unit == "ns":
-> 2080 data = [ts.to_pydatetime(warn=False) for ts in data]
2081 return np.array(data, dtype=object)
2082
~/projects/env/lib/python3.10/site-packages/pandas-2.0.0rc1+13.g7d5d123eba-py3.10-linux-x86_64.egg/pandas/core/arrays/arrow/array.py in <listcomp>(.0)
2078 data = self._data.to_pylist()
2079 if self._dtype.pyarrow_dtype.unit == "ns":
-> 2080 data = [ts.to_pydatetime(warn=False) for ts in data]
2081 return np.array(data, dtype=object)
2082
AttributeError: 'NoneType' object has no attribute 'to_pydatetime'Expected Behavior
I expected it to work, as pyarrow has support 'None values' for strings / int / float (double).
Installed Versions
INSTALLED VERSIONS
------------------
commit : 7d5d123ebac182ce08b40b8400df726c1d047be7
python : 3.10.6.final.0
python-bits : 64
OS : Linux
OS-release : 5.10.16.3-microsoft-standard-WSL2
Version : #1 SMP Fri Apr 2 22:23:49 UTC 2021
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : None
LANG : C.UTF-8
LOCALE : en_US.UTF-8
pandas : 2.0.0rc1+13.g7d5d123eba
numpy : 1.24.1
pytz : 2022.7
dateutil : 2.8.2
setuptools : 67.4.0
pip : 23.0.1
Cython : 0.29.33
pytest : 6.2.5
hypothesis : None
sphinx : None
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : 1.1
pymysql : None
psycopg2 : None
jinja2 : 3.1.2
IPython : 7.34.0
pandas_datareader: None
bs4 : None
bottleneck : None
brotli : None
fastparquet : None
fsspec : 2022.11.0
gcsfs : None
matplotlib : None
numba : None
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : 10.0.1
pyreadstat : None
pyxlsb : None
s3fs : None
scipy : None
snappy : None
sqlalchemy : 2.0.7
tables : None
tabulate : None
xarray : None
xlrd : None
zstandard : None
tzdata : 2023.2
qtpy : None
pyqt5 : None