Skip to content

BUG: fix the bad error raised by HDFStore.put() #38919

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jan 5, 2021
25 changes: 20 additions & 5 deletions pandas/io/pytables.py
Original file line number Diff line number Diff line change
Expand Up @@ -3916,6 +3916,7 @@ def get_blk_items(mgr, blocks):
nan_rep=nan_rep,
encoding=self.encoding,
errors=self.errors,
block_columns=b_items,
)
adj_name = _maybe_adjust_name(new_name, self.version)

Expand Down Expand Up @@ -4875,8 +4876,16 @@ def _unconvert_index(


def _maybe_convert_for_string_atom(
name: str, block, existing_col, min_itemsize, nan_rep, encoding, errors
name: str,
block,
existing_col,
min_itemsize,
nan_rep,
encoding,
errors,
block_columns: List[str] = [],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove the default arg, the annotation is enough

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the default [] is used to keep the interface consistency.
but since the function is only called once I think it's okay to remove, maybe
will remove it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes and never should you use mutable arguments

):
# block_columns(list[str]): the label of columns for debug info use.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you add a doc-string template e.g. Parameter / Returns (just list the arg names, and document the new one)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

got it.
but giving a private function-which is only internally called once-a docstring looks a bit deluxe...
will add docstring.

if not block.is_object:
return block.values

Expand Down Expand Up @@ -4908,14 +4917,20 @@ def _maybe_convert_for_string_atom(

# we cannot serialize this data, so report an exception on a column
# by column basis
for i in range(len(block.shape[0])):

# expected behaviour:
# search block for a non-string object column by column
for i in range(block.shape[0]):
col = block.iget(i)
inferred_type = lib.infer_dtype(col, skipna=False)
if inferred_type != "string":
iloc = block.mgr_locs.indexer[i]
error_column_label = (
block_columns[i] if len(block_columns) > i else f"No.{i}"
)
raise TypeError(
f"Cannot serialize the column [{iloc}] because\n"
f"its data contents are [{inferred_type}] object dtype"
f"Cannot serialize the column [{error_column_label}]\n"
f"because its data contents are not [string] but "
f"[{inferred_type}] object dtype"
)

# itemsize is the maximum length of a string (along any dimension)
Expand Down
10 changes: 8 additions & 2 deletions pandas/tests/io/pytables/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -2055,7 +2055,10 @@ def test_append_raise(self, setup_path):
df = tm.makeDataFrame()
df["invalid"] = [["a"]] * len(df)
assert df.dtypes["invalid"] == np.object_
msg = re.escape("object of type 'int' has no len()")
msg = re.escape(
"""Cannot serialize the column [invalid]
because its data contents are not [string] but [mixed] object dtype"""
)
with pytest.raises(TypeError, match=msg):
store.append("df", df)

Expand Down Expand Up @@ -2221,7 +2224,10 @@ def test_unimplemented_dtypes_table_columns(self, setup_path):

with ensure_clean_store(setup_path) as store:
# this fails because we have a date in the object block......
msg = "object of type 'int' has no len()"
msg = re.escape(
"""Cannot serialize the column [datetime1]
because its data contents are not [string] but [date] object dtype"""
)
with pytest.raises(TypeError, match=msg):
store.append("df_unimplemented", df)

Expand Down