REGR: roundtrip pandas<->pyarrow conversion for Period dtype broken for older pyarrow versions

[jorisvandenbossche]

To have an issue tracking #44681 (comment)

Reproducible snippet:

import pyarrow as pa

df = pd.DataFrame({"col": pd.period_range("2012", periods=3)})
pa.table(df).to_pandas()

No longer works with pyarrow < 4

[mroeschke]

Looks like we might have related xfailed tests already

pandas/pandas/tests/arrays/period/test_arrow_compat.py

Lines 74 to 132 in b8cce91

	@pytest.mark.xfail(
	pa_version_under4p0, reason="pyarrow incorrectly uses pandas internals API"
	)
	def test_arrow_table_roundtrip():
	from pandas.core.arrays._arrow_utils import ArrowPeriodType
	arr = PeriodArray([1, 2, 3], freq="D")
	arr[1] = pd.NaT
	df = pd.DataFrame({"a": arr})
	table = pa.table(df)
	assert isinstance(table.field("a").type, ArrowPeriodType)
	result = table.to_pandas()
	assert isinstance(result["a"].dtype, PeriodDtype)
	tm.assert_frame_equal(result, df)
	table2 = pa.concat_tables([table, table])
	result = table2.to_pandas()
	expected = pd.concat([df, df], ignore_index=True)
	tm.assert_frame_equal(result, expected)
	@pytest.mark.xfail(
	pa_version_under4p0, reason="pyarrow incorrectly uses pandas internals API"
	)
	def test_arrow_load_from_zero_chunks():
	# GH-41040
	from pandas.core.arrays._arrow_utils import ArrowPeriodType
	arr = PeriodArray([], freq="D")
	df = pd.DataFrame({"a": arr})
	table = pa.table(df)
	assert isinstance(table.field("a").type, ArrowPeriodType)
	table = pa.table(
	[pa.chunked_array([], type=table.column(0).type)], schema=table.schema
	)
	result = table.to_pandas()
	assert isinstance(result["a"].dtype, PeriodDtype)
	tm.assert_frame_equal(result, df)
	@pytest.mark.xfail(
	pa_version_under4p0, reason="pyarrow incorrectly uses pandas internals API"
	)
	def test_arrow_table_roundtrip_without_metadata():
	arr = PeriodArray([1, 2, 3], freq="H")
	arr[1] = pd.NaT
	df = pd.DataFrame({"a": arr})
	table = pa.table(df)
	# remove the metadata
	table = table.replace_schema_metadata()
	assert table.schema.metadata is None
	result = table.to_pandas()
	assert isinstance(result["a"].dtype, PeriodDtype)
	tm.assert_frame_equal(result, df)

[jorisvandenbossche]

Yes, but that only silences the tests, we should still fix the regression itself :)

The fix should be relatively straightforward I think, see #44681 (comment)