From 25eb51abbcdefa773b7ffb07e561aaaf11b7e3cf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 24 Oct 2023 11:25:51 -0600 Subject: [PATCH 1/8] Factor out interp_utils.*. --- Include/Python.h | 1 + Include/cpython/interp_utils.h | 80 ++++ Include/cpython/pystate.h | 77 ---- Include/interp_utils.h | 16 + Makefile.pre.in | 3 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 3 + PCbuild/pythoncore.vcxproj.filters | 9 + Python/interp_utils.c | 595 +++++++++++++++++++++++++ Python/pystate.c | 591 +----------------------- 10 files changed, 712 insertions(+), 666 deletions(-) create mode 100644 Include/cpython/interp_utils.h create mode 100644 Include/interp_utils.h create mode 100644 Python/interp_utils.c diff --git a/Include/Python.h b/Include/Python.h index 7312cc87d5cc33..2a18b6e409440c 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -84,6 +84,7 @@ #include "iterobject.h" #include "cpython/initconfig.h" #include "pystate.h" +#include "interp_utils.h" #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" diff --git a/Include/cpython/interp_utils.h b/Include/cpython/interp_utils.h new file mode 100644 index 00000000000000..b6f3b24e9f9d1d --- /dev/null +++ b/Include/cpython/interp_utils.h @@ -0,0 +1,80 @@ +#ifndef Py_CPYTHON_INTERP_UTILS_H +# error "this header file must not be included directly" +#endif + + +/* cross-interpreter data */ + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +typedef struct _xid _PyCrossInterpreterData; + +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +/* cross-interpreter data registry */ + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 995f02eab58635..ec99f90d669d12 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -258,80 +258,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interpid; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/interp_utils.h b/Include/interp_utils.h new file mode 100644 index 00000000000000..5dac666ad48229 --- /dev/null +++ b/Include/interp_utils.h @@ -0,0 +1,16 @@ +#ifndef Py_INTERP_UTILS_H +#define Py_INTERP_UTILS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_INTERP_UTILS_H +# include "cpython/interp_utils.h" +# undef Py_CPYTHON_INTERP_UTILS_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERP_UTILS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 733c68542ee885..d3d496643f8d04 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -428,6 +428,7 @@ PYTHON_OBJS= \ Python/importdl.o \ Python/initconfig.o \ Python/instrumentation.o \ + Python/interp_utils.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ Python/lock.o \ @@ -1675,6 +1676,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ $(srcdir)/Include/interpreteridobject.h \ + $(srcdir)/Include/interp_utils.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1746,6 +1748,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ + $(srcdir)/Include/cpython/interp_utils.h \ $(srcdir)/Include/cpython/listobject.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index f0afa7a6f53e58..22df7f421cdc79 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -220,6 +220,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 203d9c858173d7..3f3c7c417b2242 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -152,6 +152,7 @@ + @@ -292,6 +293,7 @@ + @@ -583,6 +585,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 09f4a2f27cdc06..6880f8d963bf8e 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -330,6 +330,9 @@ Include + + Include + Include @@ -489,6 +492,9 @@ Include + + Include\cpython + Include\cpython @@ -1346,6 +1352,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/interp_utils.c b/Python/interp_utils.c new file mode 100644 index 00000000000000..1b3d7edc40ad14 --- /dev/null +++ b/Python/interp_utils.c @@ -0,0 +1,595 @@ + +/* API for managing interpreters */ + +#include "Python.h" +#include "pycore_ceval.h" // _Py_simple_func +#include "pycore_pyerrors.h" // _PyErr_Clear() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_weakref.h" // _PyWeakref_GET_REF() + + +/**************************/ +/* cross-interpreter data */ +/**************************/ + +/* cross-interpreter data */ + +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + data->interpid = -1; +} + +static inline void +_xidata_clear(_PyCrossInterpreterData *data) +{ + // _PyCrossInterpreterData only has two members that need to be + // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. + // In both cases the original (owning) interpreter must be used, + // which is the caller's responsibility to ensure. + if (data->data != NULL) { + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + } + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + data->interpid = (interp != NULL) ? interp->id : -1; + data->new_object = new_object; +} + +int +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_RawMalloc(size); + if (data->data == NULL) { + return -1; + } + data->free = PyMem_RawFree; + return 0; +} + +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL || data->interpid == interp->id); + _xidata_clear(data); +} + +static int +_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) +{ + // data->data can be anything, including NULL, so we don't check it. + + // data->obj may be NULL, so we don't check it. + + if (data->interpid < 0) { + _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); + return -1; + } + + if (data->new_object == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); + return -1; + } + + // data->free may be NULL, so we don't check it. + + return 0; +} + +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + +int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + PyThreadState *tstate = _PyThreadState_GetCurrent(); +#ifdef Py_DEBUG + // The caller must hold the GIL + _Py_EnsureTstateNotNULL(tstate); +#endif + PyInterpreterState *interp = tstate->interp; + + // Reset data before re-populating. + *data = (_PyCrossInterpreterData){0}; + data->interpid = -1; + + // Call the "getdata" func for the object. + Py_INCREF(obj); + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + Py_DECREF(obj); + return -1; + } + int res = getdata(tstate, obj, data); + Py_DECREF(obj); + if (res != 0) { + return -1; + } + + // Fill in the blanks and validate the result. + data->interpid = interp->id; + if (_check_xidata(tstate, data) != 0) { + (void)_PyCrossInterpreterData_Release(data); + return -1; + } + + return 0; +} + +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +int +_Py_CallInInterpreter(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + return func(arg); + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); + return 0; +} + +int +_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + int res = func(arg); + PyMem_RawFree(arg); + return res; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); + return 0; +} + +static int +_call_clear_xidata(void *data) +{ + _xidata_clear((_PyCrossInterpreterData *)data); + return 0; +} + +static int +_xidata_release(_PyCrossInterpreterData *data, int rawfree) +{ + if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { + // Nothing to release! + if (rawfree) { + PyMem_RawFree(data); + } + else { + data->data = NULL; + } + return 0; + } + + // Switch to the original interpreter. + PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid); + if (interp == NULL) { + // The interpreter was already destroyed. + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + if (rawfree) { + PyMem_RawFree(data); + } + return -1; + } + + // "Release" the data and/or the object. + if (rawfree) { + return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); + } + else { + return _Py_CallInInterpreter(interp, _call_clear_xidata, data); + } +} + +int +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 0); +} + +int +_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 1); +} + +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +static int +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) +{ + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) { + return -1; + } + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } + } + newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } + xidregistry->head = newhead; + return 0; +} + +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_XDECREF(entry->weakref); + PyMem_RawFree(entry); + return next; +} + +// This is used in pystate.c (for now). +void +_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = _PyWeakref_GET_REF(cur->weakref); + if (registered == NULL) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; + } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + Py_DECREF(registered); + } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; + } + return NULL; +} + +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *xidregistry = &interp->runtime->xidregistry; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + assert(interp->xidregistry.mutex == xidregistry->mutex); + xidregistry = &interp->xidregistry; + } + return xidregistry; +} + +static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); + +static inline void +_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) +{ + if (xidregistry != &interp->xidregistry) { + assert(xidregistry == &interp->runtime->xidregistry); + if (xidregistry->head == NULL) { + _register_builtins_for_crossinterpreter_data(xidregistry); + } + } +} + +int +_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; + } + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: + PyThread_release_lock(xidregistry->mutex); + return res; +} + +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } + res = 1; + } + + PyThread_release_lock(xidregistry->mutex); + return res; +} + + +/* Cross-interpreter objects are looked up by exact match on the class. + We can reassess this policy when we move from a global registry to a + tp_* slot. */ + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + PyTypeObject *cls = Py_TYPE(obj); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + + PyThread_release_lock(xidregistry->mutex); + return func; +} + +/* cross-interpreter data for builtin types */ + +struct _shared_bytes_data { + char *bytes; + Py_ssize_t len; +}; + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); + return PyBytes_FromStringAndSize(shared->bytes, shared->len); +} + +static int +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; + if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); + return -1; + } + return 0; +} + +struct _shared_str_data { + int kind; + const void *buffer; + Py_ssize_t len; +}; + +static PyObject * +_new_str_object(_PyCrossInterpreterData *data) +{ + struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); + return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); +} + +static int +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; + shared->kind = PyUnicode_KIND(obj); + shared->buffer = PyUnicode_DATA(obj); + shared->len = PyUnicode_GET_LENGTH(obj); + return 0; +} + +static PyObject * +_new_long_object(_PyCrossInterpreterData *data) +{ + return PyLong_FromSsize_t((Py_ssize_t)(data->data)); +} + +static int +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + /* Note that this means the size of shareable ints is bounded by + * sys.maxsize. Hence on 32-bit architectures that is half the + * size of maximum shareable ints on 64-bit. + */ + Py_ssize_t value = PyLong_AsSsize_t(obj); + if (value == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); + } + return -1; + } + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL + return 0; +} + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + return Py_NewRef(Py_None); +} + +static int +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL + return 0; +} + +static void +_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) +{ + // None + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for cross-interpreter sharing"); + } + + // int + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + Py_FatalError("could not register int for cross-interpreter sharing"); + } + + // bytes + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for cross-interpreter sharing"); + } + + // str + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + Py_FatalError("could not register str for cross-interpreter sharing"); + } +} diff --git a/Python/pystate.c b/Python/pystate.c index c44a28ca6d3ac8..42855895a8f333 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -16,7 +16,6 @@ #include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_weakref.h" // _PyWeakref_GET_REF() /* -------------------------------------------------------------------------- CAUTION @@ -495,7 +494,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) return _PyStatus_OK(); } -static void _xidregistry_clear(struct _xidregistry *); +// This is defined in interp_utils.c. +void _xidregistry_clear(struct _xidregistry *); void _PyRuntimeState_Fini(_PyRuntimeState *runtime) @@ -2415,593 +2415,6 @@ PyGILState_Release(PyGILState_STATE oldstate) } -/**************************/ -/* cross-interpreter data */ -/**************************/ - -/* cross-interpreter data */ - -static inline void -_xidata_init(_PyCrossInterpreterData *data) -{ - // If the value is being reused - // then _xidata_clear() should have been called already. - assert(data->data == NULL); - assert(data->obj == NULL); - *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; -} - -static inline void -_xidata_clear(_PyCrossInterpreterData *data) -{ - // _PyCrossInterpreterData only has two members that need to be - // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. - // In both cases the original (owning) interpreter must be used, - // which is the caller's responsibility to ensure. - if (data->data != NULL) { - if (data->free != NULL) { - data->free(data->data); - } - data->data = NULL; - } - Py_CLEAR(data->obj); -} - -void -_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - void *shared, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(data != NULL); - assert(new_object != NULL); - _xidata_init(data); - data->data = shared; - if (obj != NULL) { - assert(interp != NULL); - // released in _PyCrossInterpreterData_Clear() - data->obj = Py_NewRef(obj); - } - // Ideally every object would know its owning interpreter. - // Until then, we have to rely on the caller to identify it - // (but we don't need it in all cases). - data->interpid = (interp != NULL) ? interp->id : -1; - data->new_object = new_object; -} - -int -_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, - PyInterpreterState *interp, - const size_t size, PyObject *obj, - xid_newobjectfunc new_object) -{ - assert(size > 0); - // For now we always free the shared data in the same interpreter - // where it was allocated, so the interpreter is required. - assert(interp != NULL); - _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); - data->data = PyMem_RawMalloc(size); - if (data->data == NULL) { - return -1; - } - data->free = PyMem_RawFree; - return 0; -} - -void -_PyCrossInterpreterData_Clear(PyInterpreterState *interp, - _PyCrossInterpreterData *data) -{ - assert(data != NULL); - // This must be called in the owning interpreter. - assert(interp == NULL || data->interpid == interp->id); - _xidata_clear(data); -} - -static int -_check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) -{ - // data->data can be anything, including NULL, so we don't check it. - - // data->obj may be NULL, so we don't check it. - - if (data->interpid < 0) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing interp"); - return -1; - } - - if (data->new_object == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func"); - return -1; - } - - // data->free may be NULL, so we don't check it. - - return 0; -} - -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); - -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) -{ - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; -} - -int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - return -1; - } - return 0; -} - -int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) -{ - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = current_fast_get(runtime); -#ifdef Py_DEBUG - // The caller must hold the GIL - _Py_EnsureTstateNotNULL(tstate); -#endif - PyInterpreterState *interp = tstate->interp; - - // Reset data before re-populating. - *data = (_PyCrossInterpreterData){0}; - data->interpid = -1; - - // Call the "getdata" func for the object. - Py_INCREF(obj); - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - Py_DECREF(obj); - return -1; - } - int res = getdata(tstate, obj, data); - Py_DECREF(obj); - if (res != 0) { - return -1; - } - - // Fill in the blanks and validate the result. - data->interpid = interp->id; - if (_check_xidata(tstate, data) != 0) { - (void)_PyCrossInterpreterData_Release(data); - return -1; - } - - return 0; -} - -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); -} - -int -_Py_CallInInterpreter(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - return func(arg); - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); - return 0; -} - -int -_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == current_fast_get(interp->runtime)->interp) { - int res = func(arg); - PyMem_RawFree(arg); - return res; - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); - return 0; -} - -static int -_call_clear_xidata(void *data) -{ - _xidata_clear((_PyCrossInterpreterData *)data); - return 0; -} - -static int -_xidata_release(_PyCrossInterpreterData *data, int rawfree) -{ - if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { - // Nothing to release! - if (rawfree) { - PyMem_RawFree(data); - } - else { - data->data = NULL; - } - return 0; - } - - // Switch to the original interpreter. - PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid); - if (interp == NULL) { - // The interpreter was already destroyed. - // This function shouldn't have been called. - // XXX Someone leaked some memory... - assert(PyErr_Occurred()); - if (rawfree) { - PyMem_RawFree(data); - } - return -1; - } - - // "Release" the data and/or the object. - if (rawfree) { - return _Py_CallInInterpreterAndRawFree(interp, _call_clear_xidata, data); - } - else { - return _Py_CallInInterpreter(interp, _call_clear_xidata, data); - } -} - -int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 0); -} - -int -_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) -{ - return _xidata_release(data, 1); -} - -/* registry of {type -> crossinterpdatafunc} */ - -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -static int -_xidregistry_add_type(struct _xidregistry *xidregistry, - PyTypeObject *cls, crossinterpdatafunc getdata) -{ - struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) { - return -1; - } - *newhead = (struct _xidregitem){ - // We do not keep a reference, to avoid keeping the class alive. - .cls = cls, - .refcount = 1, - .getdata = getdata, - }; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - // XXX Assign a callback to clear the entry from the registry? - newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); - if (newhead->weakref == NULL) { - PyMem_RawFree(newhead); - return -1; - } - } - newhead->next = xidregistry->head; - if (newhead->next != NULL) { - newhead->next->prev = newhead; - } - xidregistry->head = newhead; - return 0; -} - -static struct _xidregitem * -_xidregistry_remove_entry(struct _xidregistry *xidregistry, - struct _xidregitem *entry) -{ - struct _xidregitem *next = entry->next; - if (entry->prev != NULL) { - assert(entry->prev->next == entry); - entry->prev->next = next; - } - else { - assert(xidregistry->head == entry); - xidregistry->head = next; - } - if (next != NULL) { - next->prev = entry->prev; - } - Py_XDECREF(entry->weakref); - PyMem_RawFree(entry); - return next; -} - -static void -_xidregistry_clear(struct _xidregistry *xidregistry) -{ - struct _xidregitem *cur = xidregistry->head; - xidregistry->head = NULL; - while (cur != NULL) { - struct _xidregitem *next = cur->next; - Py_XDECREF(cur->weakref); - PyMem_RawFree(cur); - cur = next; - } -} - -static struct _xidregitem * -_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) -{ - struct _xidregitem *cur = xidregistry->head; - while (cur != NULL) { - if (cur->weakref != NULL) { - // cur is/was a heap type. - PyObject *registered = _PyWeakref_GET_REF(cur->weakref); - if (registered == NULL) { - // The weakly ref'ed object was freed. - cur = _xidregistry_remove_entry(xidregistry, cur); - continue; - } - assert(PyType_Check(registered)); - assert(cur->cls == (PyTypeObject *)registered); - assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); - Py_DECREF(registered); - } - if (cur->cls == cls) { - return cur; - } - cur = cur->next; - } - return NULL; -} - -static inline struct _xidregistry * -_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) -{ - struct _xidregistry *xidregistry = &interp->runtime->xidregistry; - if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { - assert(interp->xidregistry.mutex == xidregistry->mutex); - xidregistry = &interp->xidregistry; - } - return xidregistry; -} - -static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); - -static inline void -_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) -{ - if (xidregistry != &interp->xidregistry) { - assert(xidregistry == &interp->runtime->xidregistry); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); - } - } -} - -int -_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) -{ - if (!PyType_Check(cls)) { - PyErr_Format(PyExc_ValueError, "only classes may be registered"); - return -1; - } - if (getdata == NULL) { - PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); - return -1; - } - - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->getdata == getdata); - matched->refcount += 1; - goto finally; - } - - res = _xidregistry_add_type(xidregistry, cls, getdata); - -finally: - PyThread_release_lock(xidregistry->mutex); - return res; -} - -int -_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) -{ - int res = 0; - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - if (matched != NULL) { - assert(matched->refcount > 0); - matched->refcount -= 1; - if (matched->refcount == 0) { - (void)_xidregistry_remove_entry(xidregistry, matched); - } - res = 1; - } - - PyThread_release_lock(xidregistry->mutex); - return res; -} - - -/* Cross-interpreter objects are looked up by exact match on the class. - We can reassess this policy when we move from a global registry to a - tp_* slot. */ - -crossinterpdatafunc -_PyCrossInterpreterData_Lookup(PyObject *obj) -{ - PyTypeObject *cls = Py_TYPE(obj); - - PyInterpreterState *interp = _PyInterpreterState_GET(); - struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); - PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - - _ensure_builtins_xid(interp, xidregistry); - - struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); - crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; - - PyThread_release_lock(xidregistry->mutex); - return func; -} - -/* cross-interpreter data for builtin types */ - -struct _shared_bytes_data { - char *bytes; - Py_ssize_t len; -}; - -static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) -{ - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data); - return PyBytes_FromStringAndSize(shared->bytes, shared->len); -} - -static int -_bytes_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_bytes_data), obj, - _new_bytes_object - ) < 0) - { - return -1; - } - struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; - if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { - _PyCrossInterpreterData_Clear(tstate->interp, data); - return -1; - } - return 0; -} - -struct _shared_str_data { - int kind; - const void *buffer; - Py_ssize_t len; -}; - -static PyObject * -_new_str_object(_PyCrossInterpreterData *data) -{ - struct _shared_str_data *shared = (struct _shared_str_data *)(data->data); - return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len); -} - -static int -_str_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - if (_PyCrossInterpreterData_InitWithSize( - data, tstate->interp, sizeof(struct _shared_str_data), obj, - _new_str_object - ) < 0) - { - return -1; - } - struct _shared_str_data *shared = (struct _shared_str_data *)data->data; - shared->kind = PyUnicode_KIND(obj); - shared->buffer = PyUnicode_DATA(obj); - shared->len = PyUnicode_GET_LENGTH(obj); - return 0; -} - -static PyObject * -_new_long_object(_PyCrossInterpreterData *data) -{ - return PyLong_FromSsize_t((Py_ssize_t)(data->data)); -} - -static int -_long_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - /* Note that this means the size of shareable ints is bounded by - * sys.maxsize. Hence on 32-bit architectures that is half the - * size of maximum shareable ints on 64-bit. - */ - Py_ssize_t value = PyLong_AsSsize_t(obj); - if (value == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_SetString(PyExc_OverflowError, "try sending as bytes"); - } - return -1; - } - _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, - _new_long_object); - // data->obj and data->free remain NULL - return 0; -} - -static PyObject * -_new_none_object(_PyCrossInterpreterData *data) -{ - // XXX Singleton refcounts are problematic across interpreters... - return Py_NewRef(Py_None); -} - -static int -_none_shared(PyThreadState *tstate, PyObject *obj, - _PyCrossInterpreterData *data) -{ - _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, - _new_none_object); - // data->data, data->obj and data->free remain NULL - return 0; -} - -static void -_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) -{ - // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { - Py_FatalError("could not register None for cross-interpreter sharing"); - } - - // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { - Py_FatalError("could not register int for cross-interpreter sharing"); - } - - // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { - Py_FatalError("could not register bytes for cross-interpreter sharing"); - } - - // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { - Py_FatalError("could not register str for cross-interpreter sharing"); - } -} - - /*************/ /* Other API */ /*************/ From 7870af1502ec8c16196b2c8fe81e593ecb79317e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 24 Oct 2023 11:43:19 -0600 Subject: [PATCH 2/8] interp_utils -> crossinterp --- Include/Python.h | 2 +- Include/cpython/crossinterp.h | 80 ++++++++++++++++++++++++ Include/cpython/interp_utils.h | 2 +- Include/crossinterp.h | 16 +++++ Include/interp_utils.h | 16 ----- Makefile.pre.in | 6 +- PCbuild/_freeze_module.vcxproj.filters | 2 +- PCbuild/pythoncore.vcxproj | 6 +- PCbuild/pythoncore.vcxproj.filters | 6 +- Python/{interp_utils.c => crossinterp.c} | 0 Python/pystate.c | 2 +- 11 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 Include/cpython/crossinterp.h create mode 100644 Include/crossinterp.h delete mode 100644 Include/interp_utils.h rename Python/{interp_utils.c => crossinterp.c} (100%) diff --git a/Include/Python.h b/Include/Python.h index 2a18b6e409440c..94e8e91180c776 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -84,7 +84,7 @@ #include "iterobject.h" #include "cpython/initconfig.h" #include "pystate.h" -#include "interp_utils.h" +#include "crossinterp.h" #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" diff --git a/Include/cpython/crossinterp.h b/Include/cpython/crossinterp.h new file mode 100644 index 00000000000000..d155c46018a1aa --- /dev/null +++ b/Include/cpython/crossinterp.h @@ -0,0 +1,80 @@ +#ifndef Py_CPYTHON_CROSSINTERP_H +# error "this header file must not be included directly" +#endif + + +/* cross-interpreter data */ + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. +typedef struct _xid _PyCrossInterpreterData; + +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +/* cross-interpreter data registry */ + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/cpython/interp_utils.h b/Include/cpython/interp_utils.h index b6f3b24e9f9d1d..d155c46018a1aa 100644 --- a/Include/cpython/interp_utils.h +++ b/Include/cpython/interp_utils.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_INTERP_UTILS_H +#ifndef Py_CPYTHON_CROSSINTERP_H # error "this header file must not be included directly" #endif diff --git a/Include/crossinterp.h b/Include/crossinterp.h new file mode 100644 index 00000000000000..1c6bf0b3545ffd --- /dev/null +++ b/Include/crossinterp.h @@ -0,0 +1,16 @@ +#ifndef Py_CROSSINTERP_H +#define Py_CROSSINTERP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_CROSSINTERP_H +# include "cpython/crossinterp.h" +# undef Py_CPYTHON_CROSSINTERP_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CROSSINTERP_H */ diff --git a/Include/interp_utils.h b/Include/interp_utils.h deleted file mode 100644 index 5dac666ad48229..00000000000000 --- a/Include/interp_utils.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_INTERP_UTILS_H -#define Py_INTERP_UTILS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERP_UTILS_H -# include "cpython/interp_utils.h" -# undef Py_CPYTHON_INTERP_UTILS_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERP_UTILS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index d3d496643f8d04..225eaafcef20a8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -428,7 +428,7 @@ PYTHON_OBJS= \ Python/importdl.o \ Python/initconfig.o \ Python/instrumentation.o \ - Python/interp_utils.o \ + Python/crossinterp.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ Python/lock.o \ @@ -1676,7 +1676,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/interp_utils.h \ + $(srcdir)/Include/crossinterp.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1748,7 +1748,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/interp_utils.h \ + $(srcdir)/Include/cpython/crossinterp.h \ $(srcdir)/Include/cpython/listobject.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 22df7f421cdc79..123b36c81d5de0 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -220,7 +220,7 @@ Source Files - + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3f3c7c417b2242..6bf3143d28a7de 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -152,7 +152,7 @@ - + @@ -293,7 +293,7 @@ - + @@ -585,7 +585,7 @@ - + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6880f8d963bf8e..f6854f3327cdc4 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -330,7 +330,7 @@ Include - + Include @@ -492,7 +492,7 @@ Include - + Include\cpython @@ -1352,7 +1352,7 @@ Source Files - + Source Files diff --git a/Python/interp_utils.c b/Python/crossinterp.c similarity index 100% rename from Python/interp_utils.c rename to Python/crossinterp.c diff --git a/Python/pystate.c b/Python/pystate.c index 42855895a8f333..4f05bcd5798171 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -494,7 +494,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) return _PyStatus_OK(); } -// This is defined in interp_utils.c. +// This is defined in crossinterp.c (for now). void _xidregistry_clear(struct _xidregistry *); void From 8450753e4abc916aef3bce9b433195756140b58e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 24 Oct 2023 12:01:29 -0600 Subject: [PATCH 3/8] Add pycore_crossinterp.h. --- Include/cpython/crossinterp.h | 40 +------------------ Include/internal/pycore_crossinterp.h | 55 ++++++++++++++++++++++++++ Makefile.pre.in | 7 ++-- Modules/_testcapimodule.c | 9 ++--- Modules/_xxinterpchannelsmodule.c | 1 + Modules/_xxsubinterpretersmodule.c | 1 + PCbuild/_freeze_module.vcxproj.filters | 6 +-- PCbuild/pythoncore.vcxproj | 7 ++-- PCbuild/pythoncore.vcxproj.filters | 21 +++++----- Python/crossinterp.c | 26 +++++++++++- 10 files changed, 110 insertions(+), 63 deletions(-) create mode 100644 Include/internal/pycore_crossinterp.h diff --git a/Include/cpython/crossinterp.h b/Include/cpython/crossinterp.h index d155c46018a1aa..55a923b4f00d22 100644 --- a/Include/cpython/crossinterp.h +++ b/Include/cpython/crossinterp.h @@ -13,44 +13,8 @@ typedef struct _xid _PyCrossInterpreterData; typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); typedef void (*xid_freefunc)(void *); -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interpid; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; +PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); +PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); PyAPI_FUNC(void) _PyCrossInterpreterData_Init( _PyCrossInterpreterData *data, diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h new file mode 100644 index 00000000000000..a0ff0ed675c6f4 --- /dev/null +++ b/Include/internal/pycore_crossinterp.h @@ -0,0 +1,55 @@ +#ifndef Py_INTERNAL_CROSSINTERP_H +#define Py_INTERNAL_CROSSINTERP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case the code that sets the field, + // likely a registered "crossinterpdatafunc", is responsible for + // ensuring it owns the reference (i.e. incref). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. Note that IDs are never re-used, so + // each one will always correspond to a specific interpreter + // (whether still alive or not). + int64_t interpid; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + xid_newobjectfunc new_object; + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + xid_freefunc free; +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CROSSINTERP_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 225eaafcef20a8..2a05f7b16534c1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -409,6 +409,7 @@ PYTHON_OBJS= \ Python/codecs.o \ Python/compile.o \ Python/context.o \ + Python/crossinterp.o \ Python/dynamic_annotations.o \ Python/errors.o \ Python/executor.o \ @@ -428,7 +429,6 @@ PYTHON_OBJS= \ Python/importdl.o \ Python/initconfig.o \ Python/instrumentation.o \ - Python/crossinterp.o \ Python/intrinsics.o \ Python/legacy_tracing.o \ Python/lock.o \ @@ -1665,6 +1665,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ + $(srcdir)/Include/crossinterp.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dynamic_annotations.h \ @@ -1676,7 +1677,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ $(srcdir)/Include/interpreteridobject.h \ - $(srcdir)/Include/crossinterp.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1737,6 +1737,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/compile.h \ $(srcdir)/Include/cpython/complexobject.h \ $(srcdir)/Include/cpython/context.h \ + $(srcdir)/Include/cpython/crossinterp.h \ $(srcdir)/Include/cpython/descrobject.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ @@ -1748,7 +1749,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ - $(srcdir)/Include/cpython/crossinterp.h \ $(srcdir)/Include/cpython/listobject.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ @@ -1803,6 +1803,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_complexobject.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_crossinterp.h \ $(srcdir)/Include/internal/pycore_dict.h \ $(srcdir)/Include/internal/pycore_dict_state.h \ $(srcdir)/Include/internal/pycore_descrobject.h \ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index dc9a25b6c9ff3e..cfecc8543f5870 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1452,7 +1452,7 @@ _xid_capsule_destructor(PyObject *capsule) (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); if (data != NULL) { assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); + _PyCrossInterpreterData_Free(data); } } @@ -1464,19 +1464,18 @@ get_crossinterp_data(PyObject *self, PyObject *args) return NULL; } - _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); if (data == NULL) { - PyErr_NoMemory(); return NULL; } if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - PyMem_Free(data); + _PyCrossInterpreterData_Free(data); return NULL; } PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); if (capsule == NULL) { assert(_PyCrossInterpreterData_Release(data) == 0); - PyMem_Free(data); + _PyCrossInterpreterData_Free(data); } return capsule; } diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 11fe8cd01fc47b..1c9ae3b87adf7c 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -7,6 +7,7 @@ #include "Python.h" #include "interpreteridobject.h" +#include "pycore_crossinterp.h" // struct _xid #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 69078dc74d5c37..640fd69061d929 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,6 +6,7 @@ #endif #include "Python.h" +#include "pycore_crossinterp.h" // struct _xid #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 123b36c81d5de0..d6cbd2d3d47361 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -103,6 +103,9 @@ Source Files + + Source Files + Source Files @@ -220,9 +223,6 @@ Source Files - - Source Files - Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 6bf3143d28a7de..f4147e41e3b86c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -142,6 +142,7 @@ + @@ -152,7 +153,6 @@ - @@ -187,6 +187,7 @@ + @@ -217,6 +218,7 @@ + @@ -293,7 +295,6 @@ - @@ -563,6 +564,7 @@ + @@ -585,7 +587,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f6854f3327cdc4..ce8b30f8f2e220 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -72,6 +72,9 @@ Include + + Include + Include @@ -330,9 +333,6 @@ Include - - Include - Include @@ -378,6 +378,9 @@ Include\cpython + + Include\cpython + Include\cpython @@ -492,9 +495,6 @@ Include - - Include\cpython - Include\cpython @@ -585,6 +585,9 @@ Include\internal + + Include\internal + Include\internal @@ -1286,6 +1289,9 @@ Python + + Source Files + Python @@ -1352,9 +1358,6 @@ Source Files - - Source Files - Source Files diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 1b3d7edc40ad14..eb3cf2638ac783 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1,8 +1,9 @@ -/* API for managing interpreters */ +/* API for managing interactions between isolated interpreters */ #include "Python.h" #include "pycore_ceval.h" // _Py_simple_func +#include "pycore_crossinterp.h" // struct _xid #include "pycore_pyerrors.h" // _PyErr_Clear() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -41,6 +42,25 @@ _xidata_clear(_PyCrossInterpreterData *data) Py_CLEAR(data->obj); } +_PyCrossInterpreterData * +_PyCrossInterpreterData_New(void) +{ + _PyCrossInterpreterData *xid = PyMem_RawMalloc( + sizeof(_PyCrossInterpreterData)); + if (xid == NULL) { + PyErr_NoMemory(); + } + return xid; +} + +void +_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyCrossInterpreterData_Clear(interp, xid); + PyMem_RawFree(xid); +} + void _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, PyInterpreterState *interp, @@ -88,7 +108,9 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp, { assert(data != NULL); // This must be called in the owning interpreter. - assert(interp == NULL || data->interpid == interp->id); + assert(interp == NULL + || data->interpid == -1 + || data->interpid == interp->id); _xidata_clear(data); } From 2f788d8caa361efb0ac2edd9bf855fb60d3545ad Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Oct 2023 12:48:41 -0600 Subject: [PATCH 4/8] Move everything to the internal C-API for now. --- Include/Python.h | 1 - Include/cpython/crossinterp.h | 44 --------------- Include/cpython/interp_utils.h | 80 --------------------------- Include/crossinterp.h | 16 ------ Include/internal/pycore_crossinterp.h | 63 +++++++++++++++++++++ Include/internal/pycore_interp.h | 23 +------- Include/internal/pycore_runtime.h | 1 + Lib/test/test__xxsubinterpreters.py | 12 ++-- Makefile.pre.in | 2 - Modules/_testcapimodule.c | 53 ------------------ Modules/_testinternalcapi.c | 54 ++++++++++++++++++ PCbuild/pythoncore.vcxproj | 2 - PCbuild/pythoncore.vcxproj.filters | 6 -- 13 files changed, 125 insertions(+), 232 deletions(-) delete mode 100644 Include/cpython/crossinterp.h delete mode 100644 Include/cpython/interp_utils.h delete mode 100644 Include/crossinterp.h diff --git a/Include/Python.h b/Include/Python.h index 94e8e91180c776..7312cc87d5cc33 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -84,7 +84,6 @@ #include "iterobject.h" #include "cpython/initconfig.h" #include "pystate.h" -#include "crossinterp.h" #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" diff --git a/Include/cpython/crossinterp.h b/Include/cpython/crossinterp.h deleted file mode 100644 index 55a923b4f00d22..00000000000000 --- a/Include/cpython/crossinterp.h +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef Py_CPYTHON_CROSSINTERP_H -# error "this header file must not be included directly" -#endif - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); -PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/cpython/interp_utils.h b/Include/cpython/interp_utils.h deleted file mode 100644 index d155c46018a1aa..00000000000000 --- a/Include/cpython/interp_utils.h +++ /dev/null @@ -1,80 +0,0 @@ -#ifndef Py_CPYTHON_CROSSINTERP_H -# error "this header file must not be included directly" -#endif - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interpid; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/crossinterp.h b/Include/crossinterp.h deleted file mode 100644 index 1c6bf0b3545ffd..00000000000000 --- a/Include/crossinterp.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_CROSSINTERP_H -#define Py_CROSSINTERP_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_CROSSINTERP_H -# include "cpython/crossinterp.h" -# undef Py_CPYTHON_CROSSINTERP_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_CROSSINTERP_H */ diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index a0ff0ed675c6f4..1bfc366e4c4412 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -9,6 +9,17 @@ extern "C" { #endif +/**************************/ +/* cross-interpreter data */ +/**************************/ + +typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass safely between interpreters in the same process. struct _xid { // data is the cross-interpreter-safe derivation of a Python object // (see _PyObject_GetCrossInterpreterData). It will be NULL if the @@ -48,6 +59,58 @@ struct _xid { xid_freefunc free; }; +PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); +PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); + +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +/* cross-interpreter data registry */ + +// For now we use a global registry of shareable classes. An +// alternative would be to add a tp_* slot for a class's +// crossinterpdatafunc. It would be simpler and more efficient. + +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); + +struct _xidregitem; + +struct _xidregitem { + struct _xidregitem *prev; + struct _xidregitem *next; + /* This can be a dangling pointer, but only if weakref is set. */ + PyTypeObject *cls; + /* This is NULL for builtin types. */ + PyObject *weakref; + size_t refcount; + crossinterpdatafunc getdata; +}; + +struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; +}; + +PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index fc27aad48b5831..a067a60eca05df 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -15,6 +15,7 @@ extern "C" { #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state #include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state @@ -41,28 +42,6 @@ struct _Py_long_state { /* cross-interpreter data registry */ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ - -struct _xidregitem; - -struct _xidregitem { - struct _xidregitem *prev; - struct _xidregitem *next; - /* This can be a dangling pointer, but only if weakref is set. */ - PyTypeObject *cls; - /* This is NULL for builtin types. */ - PyObject *weakref; - size_t refcount; - crossinterpdatafunc getdata; -}; - -struct _xidregistry { - PyThread_type_lock mutex; - struct _xidregitem *head; -}; - /* interpreter state */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index ec026d42f6d10b..320e5bbedc068a 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_atexit.h" // struct _atexit_runtime_state #include "pycore_ceval_state.h" // struct _ceval_runtime_state +#include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index e3c917aa2eb19d..f0b9101f0a20bc 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -7,7 +7,7 @@ import threading import unittest -import _testcapi +import _testinternalcapi from test import support from test.support import import_helper from test.support import script_helper @@ -146,8 +146,8 @@ class ShareableTypeTests(unittest.TestCase): def _assert_values(self, values): for obj in values: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) self.assertEqual(got, obj) self.assertIs(type(got), type(obj)) @@ -155,8 +155,8 @@ def _assert_values(self, values): def test_singletons(self): for obj in [None]: with self.subTest(obj): - xid = _testcapi.get_crossinterp_data(obj) - got = _testcapi.restore_crossinterp_data(xid) + xid = _testinternalcapi.get_crossinterp_data(obj) + got = _testinternalcapi.restore_crossinterp_data(xid) # XXX What about between interpreters? self.assertIs(got, obj) @@ -187,7 +187,7 @@ def test_non_shareable_int(self): for i in ints: with self.subTest(i): with self.assertRaises(OverflowError): - _testcapi.get_crossinterp_data(i) + _testinternalcapi.get_crossinterp_data(i) class ModuleTests(TestBase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 2a05f7b16534c1..f2b252cce9775c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1665,7 +1665,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/crossinterp.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dynamic_annotations.h \ @@ -1737,7 +1736,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/compile.h \ $(srcdir)/Include/cpython/complexobject.h \ $(srcdir)/Include/cpython/context.h \ - $(srcdir)/Include/cpython/crossinterp.h \ $(srcdir)/Include/cpython/descrobject.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cfecc8543f5870..fc9dc746095b72 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1445,57 +1445,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static void -_xid_capsule_destructor(PyObject *capsule) -{ - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data != NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); - } -} - -static PyObject * -get_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *obj = NULL; - if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { - return NULL; - } - - _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); - if (data == NULL) { - return NULL; - } - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { - _PyCrossInterpreterData_Free(data); - return NULL; - } - PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); - if (capsule == NULL) { - assert(_PyCrossInterpreterData_Release(data) == 0); - _PyCrossInterpreterData_Free(data); - } - return capsule; -} - -static PyObject * -restore_crossinterp_data(PyObject *self, PyObject *args) -{ - PyObject *capsule = NULL; - if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { - return NULL; - } - - _PyCrossInterpreterData *data = \ - (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); - if (data == NULL) { - return NULL; - } - return _PyCrossInterpreterData_NewObject(data); -} - static PyMethodDef ml; static PyObject * @@ -3281,8 +3230,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, - {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1869f48c2b1fbf..a71e7e1dcc1256 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1466,6 +1466,58 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static void +_xid_capsule_destructor(PyObject *capsule) +{ + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data != NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } +} + +static PyObject * +get_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *obj = NULL; + if (!PyArg_ParseTuple(args, "O:get_crossinterp_data", &obj)) { + return NULL; + } + + _PyCrossInterpreterData *data = _PyCrossInterpreterData_New(); + if (data == NULL) { + return NULL; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + _PyCrossInterpreterData_Free(data); + return NULL; + } + PyObject *capsule = PyCapsule_New(data, NULL, _xid_capsule_destructor); + if (capsule == NULL) { + assert(_PyCrossInterpreterData_Release(data) == 0); + _PyCrossInterpreterData_Free(data); + } + return capsule; +} + +static PyObject * +restore_crossinterp_data(PyObject *self, PyObject *args) +{ + PyObject *capsule = NULL; + if (!PyArg_ParseTuple(args, "O:restore_crossinterp_data", &capsule)) { + return NULL; + } + + _PyCrossInterpreterData *data = \ + (_PyCrossInterpreterData *)PyCapsule_GetPointer(capsule, NULL); + if (data == NULL) { + return NULL; + } + return _PyCrossInterpreterData_NewObject(data); +} + + /*[clinic input] _testinternalcapi.write_unraisable_exc exception as exc: object @@ -1645,6 +1697,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, + {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, + {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, _TESTINTERNALCAPI_WRITE_UNRAISABLE_EXC_METHODDEF _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {NULL, NULL} /* sentinel */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index f4147e41e3b86c..954a59a0bc7019 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -142,7 +142,6 @@ - @@ -187,7 +186,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ce8b30f8f2e220..2f8b206f973f34 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -72,9 +72,6 @@ Include - - Include - Include @@ -378,9 +375,6 @@ Include\cpython - - Include\cpython - Include\cpython From f77c5c904f281c989761eb509b5c026a0617aa0a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Oct 2023 11:33:13 -0600 Subject: [PATCH 5/8] Group the API and code appropriately. --- Include/internal/pycore_ceval.h | 10 --- Include/internal/pycore_crossinterp.h | 23 +++++- Python/crossinterp.c | 102 ++++++++++++++------------ 3 files changed, 78 insertions(+), 57 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index d8afee90a32e2d..339ced3c87a43e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -54,16 +54,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( void *arg, int flags); -typedef int (*_Py_simple_func)(void *); -extern int _Py_CallInInterpreter( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); -extern int _Py_CallInInterpreterAndRawFree( - PyInterpreterState *interp, - _Py_simple_func func, - void *arg); - extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 1bfc366e4c4412..59e4cd9ece780d 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -9,6 +9,21 @@ extern "C" { #endif +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +typedef int (*_Py_simple_func)(void *); +extern int _Py_CallInInterpreter( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); +extern int _Py_CallInInterpreterAndRawFree( + PyInterpreterState *interp, + _Py_simple_func func, + void *arg); + + /**************************/ /* cross-interpreter data */ /**************************/ @@ -62,6 +77,9 @@ struct _xid { PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void); PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data); + +/* defining cross-interpreter data */ + PyAPI_FUNC(void) _PyCrossInterpreterData_Init( _PyCrossInterpreterData *data, PyInterpreterState *interp, void *shared, PyObject *obj, @@ -73,12 +91,15 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( PyInterpreterState *, _PyCrossInterpreterData *); + +/* using cross-interpreter data */ + +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); /* cross-interpreter data registry */ diff --git a/Python/crossinterp.c b/Python/crossinterp.c index eb3cf2638ac783..d7c2b705a2d011 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -9,11 +9,62 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() +/***************************/ +/* cross-interpreter calls */ +/***************************/ + +int +_Py_CallInInterpreter(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + return func(arg); + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); + return 0; +} + +int +_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, + _Py_simple_func func, void *arg) +{ + if (interp == _PyThreadState_GetCurrent()->interp) { + int res = func(arg); + PyMem_RawFree(arg); + return res; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); + return 0; +} + + /**************************/ /* cross-interpreter data */ /**************************/ -/* cross-interpreter data */ +_PyCrossInterpreterData * +_PyCrossInterpreterData_New(void) +{ + _PyCrossInterpreterData *xid = PyMem_RawMalloc( + sizeof(_PyCrossInterpreterData)); + if (xid == NULL) { + PyErr_NoMemory(); + } + return xid; +} + +void +_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyCrossInterpreterData_Clear(interp, xid); + PyMem_RawFree(xid); +} + + +/* defining cross-interpreter data */ static inline void _xidata_init(_PyCrossInterpreterData *data) @@ -42,25 +93,6 @@ _xidata_clear(_PyCrossInterpreterData *data) Py_CLEAR(data->obj); } -_PyCrossInterpreterData * -_PyCrossInterpreterData_New(void) -{ - _PyCrossInterpreterData *xid = PyMem_RawMalloc( - sizeof(_PyCrossInterpreterData)); - if (xid == NULL) { - PyErr_NoMemory(); - } - return xid; -} - -void -_PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid) -{ - PyInterpreterState *interp = PyInterpreterState_Get(); - _PyCrossInterpreterData_Clear(interp, xid); - PyMem_RawFree(xid); -} - void _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, PyInterpreterState *interp, @@ -114,6 +146,9 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp, _xidata_clear(data); } + +/* using cross-interpreter data */ + static int _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) { @@ -203,32 +238,6 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -int -_Py_CallInInterpreter(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == _PyThreadState_GetCurrent()->interp) { - return func(arg); - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, (_Py_pending_call_func)func, arg, 0); - return 0; -} - -int -_Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, - _Py_simple_func func, void *arg) -{ - if (interp == _PyThreadState_GetCurrent()->interp) { - int res = func(arg); - PyMem_RawFree(arg); - return res; - } - // XXX Emit a warning if this fails? - _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_RAWFREE); - return 0; -} - static int _call_clear_xidata(void *data) { @@ -284,6 +293,7 @@ _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) return _xidata_release(data, 1); } + /* registry of {type -> crossinterpdatafunc} */ /* For now we use a global registry of shareable classes. An From e85f29e2e80343c42312110b050b7ac4b416b198 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Oct 2023 11:48:12 -0600 Subject: [PATCH 6/8] Fix a smelly symbol. --- Python/crossinterp.c | 2 +- Python/pystate.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index d7c2b705a2d011..74f1d6ecef1329 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -353,7 +353,7 @@ _xidregistry_remove_entry(struct _xidregistry *xidregistry, // This is used in pystate.c (for now). void -_xidregistry_clear(struct _xidregistry *xidregistry) +_Py_xidregistry_clear(struct _xidregistry *xidregistry) { struct _xidregitem *cur = xidregistry->head; xidregistry->head = NULL; diff --git a/Python/pystate.c b/Python/pystate.c index 4f05bcd5798171..d97a03caf491c4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -495,7 +495,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) } // This is defined in crossinterp.c (for now). -void _xidregistry_clear(struct _xidregistry *); +extern void _Py_xidregistry_clear(struct _xidregistry *); void _PyRuntimeState_Fini(_PyRuntimeState *runtime) @@ -505,7 +505,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif - _xidregistry_clear(&runtime->xidregistry); + _Py_xidregistry_clear(&runtime->xidregistry); if (gilstate_tss_initialized(runtime)) { gilstate_tss_fini(runtime); @@ -948,7 +948,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); - _xidregistry_clear(&interp->xidregistry); + _Py_xidregistry_clear(&interp->xidregistry); /* The lock is owned by the runtime, so we don't free it here. */ interp->xidregistry.mutex = NULL; From cc3c84468acb3573c0f438fbefcd8e1fc078d030 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Oct 2023 11:56:10 -0600 Subject: [PATCH 7/8] Add a missing include. --- Objects/abstract.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/abstract.c b/Objects/abstract.c index 806ca6584bda95..070e762c58a192 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -5,6 +5,7 @@ #include "pycore_pybuffer.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() +#include "pycore_crossinterp.h" // _Py_CallInInterpreter() #include "pycore_object.h" // _Py_CheckSlotResult() #include "pycore_long.h" // _Py_IsNegative #include "pycore_pyerrors.h" // _PyErr_Occurred() From 953c39e3c35f54174e68f67c9afbcca1a5a165d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Oct 2023 11:59:24 -0600 Subject: [PATCH 8/8] Fix the _freeze_module build on Windows. --- PCbuild/_freeze_module.vcxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 5ca10588f09710..98088435b603ba 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -209,6 +209,7 @@ +