diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 479f6c4d8f69d..b1d721fb90bfb 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -257,6 +257,7 @@ Styler Other ^^^^^ - Bug in :func:`assert_almost_equal` now throwing assertion error for two unequal sets (:issue:`51727`) +- Bug in :meth:`Series.memory_usage` when ``deep=True`` throw an error with Series of objects and the returned value is incorrect, as it does not take into account GC corrections (:issue:`51858`) .. ***DO NOT USE THIS SECTION*** diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx index 88ea61a23a426..20e718cfe70a3 100644 --- a/pandas/_libs/lib.pyx +++ b/pandas/_libs/lib.pyx @@ -1,6 +1,7 @@ from collections import abc from decimal import Decimal from enum import Enum +from sys import getsizeof from typing import ( Literal, _GenericAlias, @@ -159,7 +160,7 @@ def memory_usage_of_objects(arr: object[:]) -> int64_t: n = len(arr) for i in range(n): - size += arr[i].__sizeof__() + size += getsizeof(arr[i]) return size diff --git a/pandas/_testing/__init__.py b/pandas/_testing/__init__.py index c49dda2763c83..410cf7c6cbe3a 100644 --- a/pandas/_testing/__init__.py +++ b/pandas/_testing/__init__.py @@ -177,6 +177,23 @@ np.uint32, ] +PYTHON_DATA_TYPES = [ + str, + int, + float, + complex, + list, + tuple, + range, + dict, + set, + frozenset, + bool, + bytes, + bytearray, + memoryview, +] + ENDIAN = {"little": "<", "big": ">"}[byteorder] NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")] diff --git a/pandas/conftest.py b/pandas/conftest.py index 68f3c575ee93d..22a38442dda59 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -760,6 +760,29 @@ def index_or_series_obj(request): return _index_or_series_objs[request.param].copy(deep=True) +_typ_objects_series = { + f"{dtype.__name__}-series": Series(dtype) for dtype in tm.PYTHON_DATA_TYPES +} + + +_index_or_series_memory_objs = { + **indices_dict, + **_series, + **_narrow_series, + **_typ_objects_series, +} + + +@pytest.fixture(params=_index_or_series_memory_objs.keys()) +def index_or_series_memory_obj(request): + """ + Fixture for tests on indexes, series, series with a narrow dtype and + series with empty objects type + copy to avoid mutation, e.g. setting .name + """ + return _index_or_series_memory_objs[request.param].copy(deep=True) + + # ---------------------------------------------------------------- # DataFrames # ---------------------------------------------------------------- diff --git a/pandas/tests/base/test_misc.py b/pandas/tests/base/test_misc.py index 01705ca31adcd..362df635c13fd 100644 --- a/pandas/tests/base/test_misc.py +++ b/pandas/tests/base/test_misc.py @@ -82,8 +82,8 @@ def test_ndarray_compat_properties(index_or_series_obj): @pytest.mark.skipif(PYPY, reason="not relevant for PyPy") -def test_memory_usage(index_or_series_obj): - obj = index_or_series_obj +def test_memory_usage(index_or_series_memory_obj): + obj = index_or_series_memory_obj # Clear index caches so that len(obj) == 0 report 0 memory usage if isinstance(obj, Series): is_ser = True