From 72440a5f4270f5313bb89abe10058d11bb43cc47 Mon Sep 17 00:00:00 2001 From: Tolker-KU Date: Sat, 24 Aug 2024 23:10:22 +0200 Subject: [PATCH 1/2] Improve efficency of _libs.internal --- pandas/_libs/internals.pyx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/internals.pyx b/pandas/_libs/internals.pyx index 05c4e7bd5e9dc..5ad1b9ebc6c73 100644 --- a/pandas/_libs/internals.pyx +++ b/pandas/_libs/internals.pyx @@ -1,9 +1,9 @@ from collections import defaultdict -import weakref cimport cython from cpython.pyport cimport PY_SSIZE_T_MAX from cpython.slice cimport PySlice_GetIndicesEx +from cpython.weakref cimport PyWeakref_NewRef from cython cimport Py_ssize_t import numpy as np @@ -746,7 +746,7 @@ cdef class BlockManager: # ------------------------------------------------------------------- # Block Placement - def _rebuild_blknos_and_blklocs(self) -> None: + cdef _rebuild_blknos_and_blklocs(self): """ Update mgr._blknos / mgr._blklocs. """ @@ -890,12 +890,12 @@ cdef class BlockValuesRefs: def __cinit__(self, blk: Block | None = None) -> None: if blk is not None: - self.referenced_blocks = [weakref.ref(blk)] + self.referenced_blocks = [PyWeakref_NewRef(blk, None)] else: self.referenced_blocks = [] self.clear_counter = 500 # set reasonably high - def _clear_dead_references(self, force=False) -> None: + cdef _clear_dead_references(self, bint force=False): # Use exponential backoff to decide when we want to clear references # if force=False. Clearing for every insertion causes slowdowns if # all these objects stay alive, e.g. df.items() for wide DataFrames @@ -910,7 +910,7 @@ cdef class BlockValuesRefs: elif nr_of_refs > self.clear_counter: self.clear_counter = max(self.clear_counter * 2, nr_of_refs) - def add_reference(self, blk: Block) -> None: + cpdef add_reference(self, Block blk): """Adds a new reference to our reference collection. Parameters @@ -919,7 +919,7 @@ cdef class BlockValuesRefs: The block that the new references should point to. """ self._clear_dead_references() - self.referenced_blocks.append(weakref.ref(blk)) + self.referenced_blocks.append(PyWeakref_NewRef(blk, None)) def add_index_reference(self, index: object) -> None: """Adds a new reference to our reference collection when creating an index. @@ -930,7 +930,7 @@ cdef class BlockValuesRefs: The index that the new reference should point to. """ self._clear_dead_references() - self.referenced_blocks.append(weakref.ref(index)) + self.referenced_blocks.append(PyWeakref_NewRef(index, None)) def has_reference(self) -> bool: """Checks if block has foreign references. From 8b78e707201a7af4936952db908f8ec043ccafc4 Mon Sep 17 00:00:00 2001 From: Tolker-KU Date: Sat, 24 Aug 2024 23:35:41 +0200 Subject: [PATCH 2/2] Re-add python version of ._rebuild_blknos_and_blklocs --- pandas/_libs/internals.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/internals.pyx b/pandas/_libs/internals.pyx index 5ad1b9ebc6c73..53e903aab9cb8 100644 --- a/pandas/_libs/internals.pyx +++ b/pandas/_libs/internals.pyx @@ -746,7 +746,7 @@ cdef class BlockManager: # ------------------------------------------------------------------- # Block Placement - cdef _rebuild_blknos_and_blklocs(self): + cpdef _rebuild_blknos_and_blklocs(self): """ Update mgr._blknos / mgr._blklocs. """