From 2337b1f6ad1844c6658283f7f7b0df84b7ddcb6f Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 14 Nov 2024 09:41:15 +0300 Subject: [PATCH 1/5] Add PyLong Import/Export API PyPy is not supported, as well as Py2. In the later case it's possible, but hardly worth code complications: most real-word potential consumers (e.g. Sage, gmpy2 or python-flint) support only Python 3. --- docs/api.rst | 36 ++++++ docs/changelog.rst | 12 ++ pythoncapi_compat.h | 181 ++++++++++++++++++++++++++++ tests/test_pythoncapi_compat_cext.c | 44 +++++++ 4 files changed, 273 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 7c743f8..07b303a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -29,6 +29,42 @@ Latest version of the header file: Python 3.14 ----------- +.. c:struct:: PyLongLayout + + See `PyLongLayout documentation `__. + +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + See `PyLong_GetNativeLayout() documentation `__. + +.. c:struct:: PyLongExport + + See `PyLongExport documentation `__. + +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) + + See `PyLong_Export() documentation `__. + +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) + + See `PyLong_FreeExport() documentation `__. + +.. c:struct:: PyLongWriter + + See `PyLongWriter documentation `__. + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) + + See `PyLongWriter_Create() documentation `__. + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + See `PyLongWriter_Finish() documentation `__. + +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + See `PyLongWriter_Discard() documentation `__. + .. c:function:: int PyLong_IsPositive(PyObject *obj) See `PyLong_IsPositive() documentation `__. diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a616f7..0383e49 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,18 @@ Changelog ========= +* XXXX-XX-XX: Add functions and structs: + + * ``PyLongLayout`` + * ``PyLong_GetNativeLayout()`` + * ``PyLongExport`` + * ``PyLong_Export()`` + * ``PyLong_FreeExport()`` + * ``PyLongWriter`` + * ``PyLongWriter_Create()`` + * ``PyLongWriter_Finish()`` + * ``PyLongWriter_Discard()`` + * 2024-11-12: Add functions: * ``PyLong_IsPositive()`` diff --git a/pythoncapi_compat.h b/pythoncapi_compat.h index c51dd6b..15f72f6 100644 --- a/pythoncapi_compat.h +++ b/pythoncapi_compat.h @@ -1720,6 +1720,187 @@ static inline int PyLong_AsUInt64(PyObject *obj, uint64_t *pvalue) #endif +// gh-102471 added import and export API for integers to 3.14.0a2. +#if PY_VERSION_HEX < 0x030E00A2 && PY_VERSION_HEX >= 0x03000000 && !defined(PYPY_VERSION) +// Helpers to access PyLongObject internals. +static inline void +_PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) +{ +#if PY_VERSION_HEX >= 0x030C0000 + op->long_value.lv_tag = (uintptr_t)(1 - sign) | ((uintptr_t)(size) << 3); +#elif PY_VERSION_HEX >= 0x030900A4 + Py_SET_SIZE(op, sign*size); +#else + Py_SIZE(op) = sign*size; +#endif +} + +static inline Py_ssize_t +_PyLong_DigitCount(const PyLongObject *op) +{ +#if PY_VERSION_HEX >= 0x030C0000 + return (Py_ssize_t)(op->long_value.lv_tag >> 3); +#else + return _PyLong_Sign((PyObject*)op) < 0 ? -Py_SIZE(op) : Py_SIZE(op); +#endif +} + +static inline digit* +_PyLong_GetDigits(const PyLongObject *op) +{ +#if PY_VERSION_HEX >= 0x030C0000 + return (digit*)(op->long_value.ob_digit); +#else + return (digit*)(op->ob_digit); +#endif +} + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +static const PyLongLayout PyLong_LAYOUT = { + PyLong_SHIFT, + sizeof(digit), + -1, // least significant first + PY_LITTLE_ENDIAN ? -1 : 1, +}; + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + Py_uintptr_t _reserved; +} PyLongExport; + +typedef struct PyLongWriter PyLongWriter; + +static inline const PyLongLayout* +PyLong_GetNativeLayout(void) +{ + return &PyLong_LAYOUT; +} + +static inline int +PyLong_Export(PyObject *obj, PyLongExport *export_long) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expected int, got %s", + Py_TYPE(obj)->tp_name); + return -1; + } + + // Fast-path: try to convert to a int64_t + PyLongObject *self = (PyLongObject*)obj; + int overflow; +#if SIZEOF_LONG == 8 + long value = PyLong_AsLongAndOverflow(obj, &overflow); +#else + // Windows has 32-bit long, so use 64-bit long long instead + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); +#endif + Py_BUILD_ASSERT(sizeof(value) == sizeof(int64_t)); + // the function cannot fail since obj is a PyLongObject + assert(!(value == -1 && PyErr_Occurred())); + + if (!overflow) { + export_long->value = value; + export_long->negative = 0; + export_long->ndigits = 0; + export_long->digits = 0; + export_long->_reserved = 0; + } + else { + export_long->value = 0; + export_long->negative = _PyLong_Sign(obj) < 0; + export_long->ndigits = _PyLong_DigitCount(self); + if (export_long->ndigits == 0) { + export_long->ndigits = 1; + } + export_long->digits = _PyLong_GetDigits(self); + export_long->_reserved = (Py_uintptr_t)Py_NewRef(obj); + } + return 0; +} + +static inline void +PyLong_FreeExport(PyLongExport *export_long) +{ + PyObject *obj = (PyObject*)export_long->_reserved; + + if (obj) { + export_long->_reserved = 0; + Py_DECREF(obj); + } +} + +static inline PyLongWriter* +PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) +{ + if (ndigits < 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); + return NULL; + } + assert(digits != NULL); + + PyLongObject *obj = _PyLong_New(ndigits); + if (obj == NULL) { + return NULL; + } + if (ndigits == 0) { + assert(_PyLong_GetDigits(obj)[0] == 0); + } + _PyLong_SetSignAndDigitCount(obj, negative?-1:1, ndigits); + + *digits = _PyLong_GetDigits(obj); + return (PyLongWriter*)obj; +} + +static inline void +PyLongWriter_Discard(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + + assert(Py_REFCNT(obj) == 1); + Py_DECREF(obj); +} + +static inline PyObject* +PyLongWriter_Finish(PyLongWriter *writer) +{ + PyObject *obj = (PyObject *)writer; + PyLongObject *self = (PyLongObject*)obj; + Py_ssize_t j = _PyLong_DigitCount(self); + Py_ssize_t i = j; + int sign = _PyLong_Sign(obj); + + assert(Py_REFCNT(obj) == 1); + + // Normalize and get singleton if possible + while (i > 0 && _PyLong_GetDigits(self)[i-1] == 0) { + --i; + } + if (i != j) { + if (i == 0) { + sign = 0; + } + _PyLong_SetSignAndDigitCount(self, sign, i); + } + if (i <= 1) { + long val = sign*(long)(_PyLong_GetDigits(self)[0]); + Py_DECREF(obj); + return PyLong_FromLong(val); + } + + return obj; +} +#endif + + #ifdef __cplusplus } #endif diff --git a/tests/test_pythoncapi_compat_cext.c b/tests/test_pythoncapi_compat_cext.c index b8df8a6..e5c342e 100644 --- a/tests/test_pythoncapi_compat_cext.c +++ b/tests/test_pythoncapi_compat_cext.c @@ -1426,6 +1426,50 @@ test_long_api(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) assert(PyLong_IsNegative(obj) == 0); assert(PyLong_IsZero(obj) == 0); +#if defined(PYTHON3) && !defined(PYPY_VERSION) + // test import/export API + digit *digits; + PyLongWriter *writer; + static PyLongExport long_export; + + writer = PyLongWriter_Create(1, 1, (void**)&digits); + PyLongWriter_Discard(writer); + + writer = PyLongWriter_Create(1, 1, (void**)&digits); + digits[0] = 123; + obj = PyLongWriter_Finish(writer); + + PyLong_Export(obj, &long_export); + assert(long_export.value == -123); + assert(long_export.digits == NULL); + PyLong_FreeExport(&long_export); + Py_DECREF(obj); + + writer = PyLongWriter_Create(0, 5, (void**)&digits); + digits[0] = 1; + digits[1] = 0; + digits[2] = 0; + digits[3] = 0; + digits[4] = 1; + obj = PyLongWriter_Finish(writer); + + PyLong_Export(obj, &long_export); + assert(long_export.value == 0); + digits = (digit*)long_export.digits; + assert(digits[0] == 1); + assert(digits[1] == 0); + assert(digits[2] == 0); + assert(digits[3] == 0); + assert(digits[4] == 1); + PyLong_FreeExport(&long_export); + Py_DECREF(obj); + + const PyLongLayout *layout = PyLong_GetNativeLayout(); + + assert(layout->digits_order == -1); + assert(layout->digit_size == sizeof(digit)); +#endif // defined(PYTHON3) && !defined(PYPY_VERSION) + Py_RETURN_NONE; } From 629882182de8fee60ea44a4d934d89fa6500cc81 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Dec 2024 16:41:50 +0300 Subject: [PATCH 2/5] Apply suggestions from code review Co-authored-by: Victor Stinner --- pythoncapi_compat.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythoncapi_compat.h b/pythoncapi_compat.h index 15f72f6..8fb7504 100644 --- a/pythoncapi_compat.h +++ b/pythoncapi_compat.h @@ -1729,9 +1729,9 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) #if PY_VERSION_HEX >= 0x030C0000 op->long_value.lv_tag = (uintptr_t)(1 - sign) | ((uintptr_t)(size) << 3); #elif PY_VERSION_HEX >= 0x030900A4 - Py_SET_SIZE(op, sign*size); + Py_SET_SIZE(op, sign * size); #else - Py_SIZE(op) = sign*size; + Py_SIZE(op) = sign * size; #endif } @@ -1891,7 +1891,7 @@ PyLongWriter_Finish(PyLongWriter *writer) _PyLong_SetSignAndDigitCount(self, sign, i); } if (i <= 1) { - long val = sign*(long)(_PyLong_GetDigits(self)[0]); + long val = sign * (long)(_PyLong_GetDigits(self)[0]); Py_DECREF(obj); return PyLong_FromLong(val); } From a1cb729644da7d10c1591aea803222eab3ec623b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Dec 2024 16:56:06 +0300 Subject: [PATCH 3/5] Apply suggestions from code review --- docs/changelog.rst | 2 +- tests/test_pythoncapi_compat_cext.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0383e49..ec263a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -* XXXX-XX-XX: Add functions and structs: +* 2024-12-13: Add functions and structs: * ``PyLongLayout`` * ``PyLong_GetNativeLayout()`` diff --git a/tests/test_pythoncapi_compat_cext.c b/tests/test_pythoncapi_compat_cext.c index e5c342e..d8a27bd 100644 --- a/tests/test_pythoncapi_compat_cext.c +++ b/tests/test_pythoncapi_compat_cext.c @@ -1465,7 +1465,6 @@ test_long_api(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_DECREF(obj); const PyLongLayout *layout = PyLong_GetNativeLayout(); - assert(layout->digits_order == -1); assert(layout->digit_size == sizeof(digit)); #endif // defined(PYTHON3) && !defined(PYPY_VERSION) From cc05a9ab67af13cfe3ec2119d6f94b23c96039ca Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Dec 2024 17:04:29 +0300 Subject: [PATCH 4/5] address review --- pythoncapi_compat.h | 14 +++++++------- tests/test_pythoncapi_compat_cext.c | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pythoncapi_compat.h b/pythoncapi_compat.h index 03ac58e..c8552db 100644 --- a/pythoncapi_compat.h +++ b/pythoncapi_compat.h @@ -1762,13 +1762,6 @@ typedef struct PyLongLayout { int8_t digit_endianness; } PyLongLayout; -static const PyLongLayout PyLong_LAYOUT = { - PyLong_SHIFT, - sizeof(digit), - -1, // least significant first - PY_LITTLE_ENDIAN ? -1 : 1, -}; - typedef struct PyLongExport { int64_t value; uint8_t negative; @@ -1782,6 +1775,13 @@ typedef struct PyLongWriter PyLongWriter; static inline const PyLongLayout* PyLong_GetNativeLayout(void) { + static const PyLongLayout PyLong_LAYOUT = { + PyLong_SHIFT, + sizeof(digit), + -1, // least significant first + PY_LITTLE_ENDIAN ? -1 : 1, + }; + return &PyLong_LAYOUT; } diff --git a/tests/test_pythoncapi_compat_cext.c b/tests/test_pythoncapi_compat_cext.c index eb556b2..28663d2 100644 --- a/tests/test_pythoncapi_compat_cext.c +++ b/tests/test_pythoncapi_compat_cext.c @@ -1439,6 +1439,7 @@ test_long_api(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) digits[0] = 123; obj = PyLongWriter_Finish(writer); + check_int(obj, -123); PyLong_Export(obj, &long_export); assert(long_export.value == -123); assert(long_export.digits == NULL); From eb966bfccce213995de5d2e7b26692ebe59401de Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Fri, 13 Dec 2024 17:09:34 +0300 Subject: [PATCH 5/5] sync with CPython pr --- pythoncapi_compat.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pythoncapi_compat.h b/pythoncapi_compat.h index c8552db..5e22e7d 100644 --- a/pythoncapi_compat.h +++ b/pythoncapi_compat.h @@ -1789,6 +1789,7 @@ static inline int PyLong_Export(PyObject *obj, PyLongExport *export_long) { if (!PyLong_Check(obj)) { + memset(export_long, 0, sizeof(*export_long)); PyErr_Format(PyExc_TypeError, "expected int, got %s", Py_TYPE(obj)->tp_name); return -1; @@ -1841,7 +1842,7 @@ PyLong_FreeExport(PyLongExport *export_long) static inline PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) { - if (ndigits < 0) { + if (ndigits <= 0) { PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); return NULL; } @@ -1851,9 +1852,6 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) if (obj == NULL) { return NULL; } - if (ndigits == 0) { - assert(_PyLong_GetDigits(obj)[0] == 0); - } _PyLong_SetSignAndDigitCount(obj, negative?-1:1, ndigits); *digits = _PyLong_GetDigits(obj);