From b158d3f510419eb63edf2e771783f8e53b641006 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 23:14:27 +0100 Subject: [PATCH 01/10] Rename inited to initialized, convert to bool --- Modules/_zstd/_zstdmodule.h | 10 +++++----- Modules/_zstd/compressor.c | 10 +++++----- Modules/_zstd/decompressor.c | 9 +++++---- Modules/_zstd/zstddict.c | 9 +++++---- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index d5283ddcf56fe8..a0be7180fda5f4 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -11,6 +11,8 @@ Python module. #include "zstd.h" #include "zdict.h" +#include // bool + /* Forward declaration of module state */ typedef struct _zstd_state _zstd_state; @@ -61,7 +63,7 @@ typedef struct { uint32_t dict_id; /* __init__ has been called, 0 or 1. */ - int inited; + bool initialized; } ZstdDict; typedef struct { @@ -83,7 +85,7 @@ typedef struct { int compression_level; /* __init__ has been called, 0 or 1. */ - int inited; + bool initialized; } ZstdCompressor; typedef struct { @@ -120,7 +122,7 @@ typedef struct { char _unused_char_for_align; /* __init__ has been called, 0 or 1. */ - int inited; + bool initialized; } ZstdDecompressor; typedef enum { @@ -163,5 +165,3 @@ set_zstd_error(const _zstd_state* const state, extern void set_parameter_error(const _zstd_state* const state, int is_compress, int key_v, int value_v); - -static const char init_twice_msg[] = "__init__ method is called twice."; diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b92083cf78fb26..dc25292d86b93c 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -20,6 +20,7 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_ty #include "buffer.h" +#include // bool #include // offsetof() @@ -305,7 +306,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->use_multithread = 0; @@ -372,12 +373,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, PyObject *options, PyObject *zstd_dict) /*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ { - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; if (level != Py_None && options != Py_None) { PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index d2b637205be64f..03abefc488992d 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -19,6 +19,7 @@ class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompres #include "buffer.h" +#include // bool #include // offsetof() #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) @@ -616,7 +617,7 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - self->inited = 0; + self->initialized = 0; self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; @@ -695,11 +696,11 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Load dictionary to decompression context */ if (zstd_dict != Py_None) { diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 402fb65fb0042a..d007ed2a33e158 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -17,6 +17,7 @@ class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" #include "_zstdmodule.h" +#include // bool #include // offsetof() #define ZstdDict_CAST(op) ((ZstdDict *)op) @@ -31,7 +32,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U } self->dict_content = NULL; - self->inited = 0; + self->initialized = 0; self->d_dict = NULL; /* ZSTD_CDict dict */ @@ -92,11 +93,11 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, /*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ { /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); + if (self->initialized) { + PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); return -1; } - self->inited = 1; + self->initialized = 1; /* Check dict_content's type */ self->dict_content = PyBytes_FromObject(dict_content); From ae9e303fc4b05088e21666d46ac36129599aa091 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 23:15:00 +0100 Subject: [PATCH 02/10] Move mt_continue_should_break() --- Modules/_zstd/_zstdmodule.h | 6 ------ Modules/_zstd/compressor.c | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index a0be7180fda5f4..032c9fea515246 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -151,12 +151,6 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; -static inline int -mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) -{ - return in->size == in->pos && out->size != out->pos; -} - /* Format error message and set ZstdError. */ extern void set_zstd_error(const _zstd_state* const state, diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index dc25292d86b93c..e6314dae22f612 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -488,6 +488,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, return NULL; } +static inline int +mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) +{ + return in->size == in->pos && out->size != out->pos; +} + static PyObject * compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) { From d2e189fc04852f71eb26656effbc6e6da26e9fa1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 9 May 2025 23:15:57 +0100 Subject: [PATCH 03/10] Move enum decompress_type --- Modules/_zstd/_zstdmodule.h | 5 ----- Modules/_zstd/decompressor.c | 15 ++++++--------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 032c9fea515246..b2a285c5805e20 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -125,11 +125,6 @@ typedef struct { bool initialized; } ZstdDecompressor; -typedef enum { - TYPE_DECOMPRESSOR, // , ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // , decompress() function -} decompress_type; - typedef enum { ERR_DECOMPRESS, ERR_COMPRESS, diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 03abefc488992d..b783e7afe9183f 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -216,17 +216,14 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) return 0; } - +typedef enum { + TYPE_DECOMPRESSOR, // , ZstdDecompressor class + TYPE_ENDLESS_DECOMPRESSOR, // , decompress() function +} decompress_type; /* - Given the two types of decompressors (defined in _zstdmodule.h): - - typedef enum { - TYPE_DECOMPRESSOR, // , ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // , decompress() function - } decompress_type; - - Decompress implementation for , , pseudo code: + Given the two types of decompressors (defined above), + decompress implementation for , , pseudo code: initialize_output_buffer while True: From 8e5e1ca0b003ba2dc63598a2f15c97686191b1c6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:15:26 +0100 Subject: [PATCH 04/10] Add guard defines --- Modules/_zstd/_zstdmodule.h | 5 ++++- Modules/_zstd/buffer.h | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index b2a285c5805e20..0309612a581049 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,4 +1,3 @@ -#pragma once /* Low level interface to Meta's zstd library for use in the compression.zstd Python module. @@ -6,6 +5,8 @@ Python module. /* Declarations shared between different parts of the _zstd module*/ +#ifndef ZSTD_MODULE_H +#define ZSTD_MODULE_H #include "Python.h" #include "zstd.h" @@ -154,3 +155,5 @@ set_zstd_error(const _zstd_state* const state, extern void set_parameter_error(const _zstd_state* const state, int is_compress, int key_v, int value_v); + +#endif diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 319b1214833fcf..d6b3e4dda13c26 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -3,7 +3,10 @@ Low level interface to Meta's zstd library for use in the compression.zstd Python module. */ +#ifndef ZSTD_BUFFER_H +#define ZSTD_BUFFER_H #include "_zstdmodule.h" +#include "Python.h" #include "pycore_blocks_output_buffer.h" /* Blocks output buffer wrapper code */ @@ -102,3 +105,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } + +#endif From 161b826a7704ddfea434789948ab03c21a98e865 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:20:20 +0100 Subject: [PATCH 05/10] Use angle-bracket includes --- Modules/_zstd/_zstdmodule.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 0309612a581049..544f4d9c7499b7 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -9,10 +9,9 @@ Python module. #define ZSTD_MODULE_H #include "Python.h" -#include "zstd.h" -#include "zdict.h" - #include // bool +#include +#include /* Forward declaration of module state */ From 942be5b81acba16f8accba8e03031ccb20a1f976 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:24:26 +0100 Subject: [PATCH 06/10] Reduce transitive includes --- Modules/_zstd/buffer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index d6b3e4dda13c26..7f9ada97b420fa 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -5,10 +5,11 @@ Python module. #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H -#include "_zstdmodule.h" #include "Python.h" #include "pycore_blocks_output_buffer.h" +#include // ZSTD_outBuffer + /* Blocks output buffer wrapper code */ /* Initialize the buffer, and grow the buffer. From a71aa3cac4953d1614fb2dc282506a2072c57649 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:28:16 +0100 Subject: [PATCH 07/10] Remove Zstd{Compressor,Decompressor,Dict} from _zstdmodule.h --- Makefile.pre.in | 2 +- Modules/_zstd/_zstdmodule.c | 6 +++ Modules/_zstd/_zstdmodule.h | 83 ----------------------------------- Modules/_zstd/compressor.c | 35 ++++++++++++--- Modules/_zstd/decompressor.c | 50 ++++++++++++++++++--- Modules/_zstd/zstddict.c | 13 +++--- Modules/_zstd/zstddict.h | 32 ++++++++++++++ PCbuild/_zstd.vcxproj | 1 + PCbuild/_zstd.vcxproj.filters | 3 ++ 9 files changed, 121 insertions(+), 104 deletions(-) create mode 100644 Modules/_zstd/zstddict.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 17e0c9904cc3aa..b9ae6037afbd13 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3341,7 +3341,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h -MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h +MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 483e88d98b78ba..4e5646e59660f7 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -7,7 +7,13 @@ Python module. # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" + +#include // ZSTD_*() +#include // ZDICT_*() /*[clinic input] module _zstd diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 544f4d9c7499b7..56f24d37a0f732 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -9,11 +9,6 @@ Python module. #define ZSTD_MODULE_H #include "Python.h" -#include // bool -#include -#include - - /* Forward declaration of module state */ typedef struct _zstd_state _zstd_state; @@ -47,84 +42,6 @@ struct _zstd_state { PyTypeObject *DParameter_type; }; -typedef struct { - PyObject_HEAD - - /* Reusable compress/decompress dictionary, they are created once and - can be shared by multiple threads concurrently, since its usage is - read-only. - c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ - ZSTD_DDict *d_dict; - PyObject *c_dicts; - - /* Content of the dictionary, bytes object. */ - PyObject *dict_content; - /* Dictionary id */ - uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - bool initialized; -} ZstdDict; - -typedef struct { - PyObject_HEAD - - /* Compression context */ - ZSTD_CCtx *cctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Last mode, initialized to ZSTD_e_end */ - int last_mode; - - /* (nbWorker >= 1) ? 1 : 0 */ - int use_multithread; - - /* Compression level */ - int compression_level; - - /* __init__ has been called, 0 or 1. */ - bool initialized; -} ZstdCompressor; - -typedef struct { - PyObject_HEAD - - /* Decompression context */ - ZSTD_DCtx *dctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Unconsumed input data */ - char *input_buffer; - size_t input_buffer_size; - size_t in_begin, in_end; - - /* Unused data */ - PyObject *unused_data; - - /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ - char needs_input; - - /* For decompress(), 0 or 1. - 1 when both input and output streams are at a frame edge, means a - frame is completely decoded and fully flushed, or the decompressor - just be initialized. */ - char at_frame_edge; - - /* For ZstdDecompressor, 0 or 1. - 1 means the end of the first frame has been reached. */ - char eof; - - /* Used for fast reset above three variables */ - char _unused_char_for_align; - - /* __init__ has been called, 0 or 1. */ - bool initialized; -} ZstdDecompressor; - typedef enum { ERR_DECOMPRESS, ERR_COMPRESS, diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index e6314dae22f612..a6edb067535b19 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -7,25 +7,50 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type" +class _zstd.ZstdCompressor "ZstdCompressor *" "zstd_compressor_type_spec" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ - #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" #include // bool #include // offsetof() +#include // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Compression context */ + ZSTD_CCtx *cctx; + + /* ZstdDict object in use */ + PyObject *dict; + /* Last mode, initialized to ZSTD_e_end */ + int last_mode; + + /* (nbWorker >= 1) ? 1 : 0 */ + int use_multithread; + + /* Compression level */ + int compression_level; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) +#include "clinic/compressor.c.h" + static int _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, const char *arg_name, const char* arg_type) @@ -293,10 +318,6 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) return 0; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/compressor.c.h" -#undef clinic_state - static PyObject * _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) { diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index b783e7afe9183f..52ab7b4d994773 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -7,7 +7,7 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type" +class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ @@ -15,15 +15,57 @@ class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompres # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "zstddict.h" #include // bool #include // offsetof() +#include // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Decompression context */ + ZSTD_DCtx *dctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Unconsumed input data */ + char *input_buffer; + size_t input_buffer_size; + size_t in_begin, in_end; + + /* Unused data */ + PyObject *unused_data; + + /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ + char needs_input; + + /* For decompress(), 0 or 1. + 1 when both input and output streams are at a frame edge, means a + frame is completely decoded and fully flushed, or the decompressor + just be initialized. */ + char at_frame_edge; + + /* For ZstdDecompressor, 0 or 1. + 1 means the end of the first frame has been reached. */ + char eof; + + /* Used for fast reset above three variables */ + char _unused_char_for_align; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) +#include "clinic/decompressor.c.h" + static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { @@ -800,10 +842,6 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, return ret; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/decompressor.c.h" -#undef clinic_state - static PyMethodDef ZstdDecompressor_methods[] = { _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF {NULL, NULL} diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index d007ed2a33e158..34b31deaa9f7e1 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -7,7 +7,7 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" +class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ @@ -15,10 +15,13 @@ class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "zstddict.h" +#include "clinic/zstddict.c.h" -#include // bool -#include // offsetof() +#include // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() #define ZstdDict_CAST(op) ((ZstdDict *)op) @@ -136,10 +139,6 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, return 0; } -#define clinic_state() (get_zstd_state(type)) -#include "clinic/zstddict.c.h" -#undef clinic_state - PyDoc_STRVAR(ZstdDict_dictid_doc, "ID of zstd dictionary, a 32-bit unsigned int value.\n\n" "Non-zero means ordinary dictionary, was created by zstd functions, follow\n" diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h new file mode 100644 index 00000000000000..91454ff6a2d181 --- /dev/null +++ b/Modules/_zstd/zstddict.h @@ -0,0 +1,32 @@ +/* +Low level interface to Meta's zstd library for use in the compression.zstd +Python module. +*/ + +#ifndef ZSTD_DICT_H +#define ZSTD_DICT_H +#include "Python.h" + +#include // bool +#include // ZSTD_DDict + +typedef struct { + PyObject_HEAD + + /* Reusable compress/decompress dictionary, they are created once and + can be shared by multiple threads concurrently, since its usage is + read-only. + c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ + ZSTD_DDict *d_dict; + PyObject *c_dicts; + + /* Content of the dictionary, bytes object. */ + PyObject *dict_content; + /* Dictionary id */ + uint32_t dict_id; + + /* __init__ has been called, 0 or 1. */ + bool initialized; +} ZstdDict; + +#endif diff --git a/PCbuild/_zstd.vcxproj b/PCbuild/_zstd.vcxproj index b53f93a6af8d11..6f91b8d05cca20 100644 --- a/PCbuild/_zstd.vcxproj +++ b/PCbuild/_zstd.vcxproj @@ -137,6 +137,7 @@ + diff --git a/PCbuild/_zstd.vcxproj.filters b/PCbuild/_zstd.vcxproj.filters index d4d36063c85d09..eec666e5eaf439 100644 --- a/PCbuild/_zstd.vcxproj.filters +++ b/PCbuild/_zstd.vcxproj.filters @@ -128,6 +128,9 @@ Header Files + + Header Files + Header Files\zstd From 9548b6a15923416403b09d0590881433a0c4cad4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:28:55 +0100 Subject: [PATCH 08/10] Make _zstdmodule static --- Modules/_zstd/_zstdmodule.c | 2 +- Modules/_zstd/_zstdmodule.h | 26 +++++--------------------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4e5646e59660f7..bddef0d0b864cd 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -733,7 +733,7 @@ static struct PyModuleDef_Slot _zstd_slots[] = { {0, NULL}, }; -struct PyModuleDef _zstdmodule = { +static struct PyModuleDef _zstdmodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zstd", .m_doc = "Implementation module for Zstandard compression.", diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 56f24d37a0f732..bfacea925cb4e5 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -9,38 +9,22 @@ Python module. #define ZSTD_MODULE_H #include "Python.h" -/* Forward declaration of module state */ -typedef struct _zstd_state _zstd_state; - -/* Forward reference of module def */ -extern PyModuleDef _zstdmodule; - -/* For clinic type calculations */ -static inline _zstd_state * -get_zstd_state_from_type(PyTypeObject *type) -{ - PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule); - if (module == NULL) { - return NULL; - } - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_zstd_state *)state; -} - +/* Type specs */ extern PyType_Spec zstd_dict_type_spec; extern PyType_Spec zstd_compressor_type_spec; extern PyType_Spec zstd_decompressor_type_spec; -struct _zstd_state { +typedef struct { + /* Module heap types. */ PyTypeObject *ZstdDict_type; PyTypeObject *ZstdCompressor_type; PyTypeObject *ZstdDecompressor_type; PyObject *ZstdError; + /* enum types set by set_parameter_types. */ PyTypeObject *CParameter_type; PyTypeObject *DParameter_type; -}; +} _zstd_state; typedef enum { ERR_DECOMPRESS, From 223216d9047f57358d6530ff96f065b0756550c1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 01:52:09 +0100 Subject: [PATCH 09/10] regen-all --- Modules/_zstd/compressor.c | 2 +- Modules/_zstd/decompressor.c | 2 +- Modules/_zstd/zstddict.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index a6edb067535b19..9aee81bd92782a 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -9,7 +9,7 @@ Python module. module _zstd class _zstd.ZstdCompressor "ZstdCompressor *" "zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=842f2c4f4520ab26]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 52ab7b4d994773..2ed88cd3f231f1 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -9,7 +9,7 @@ Python module. module _zstd class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 34b31deaa9f7e1..99976cef85a210 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -9,7 +9,7 @@ Python module. module _zstd class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 From bb204455b4ff78402b9c4918897a0cf1e2773589 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sat, 10 May 2025 15:43:45 +0100 Subject: [PATCH 10/10] =?UTF-8?q?B=C3=A9n=C3=A9dikt's=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Modules/_zstd/_zstdmodule.h | 3 +-- Modules/_zstd/buffer.h | 4 ++-- Modules/_zstd/compressor.c | 4 ++-- Modules/_zstd/zstddict.h | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index bfacea925cb4e5..00e0d2177f31f6 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -7,7 +7,6 @@ Python module. #ifndef ZSTD_MODULE_H #define ZSTD_MODULE_H -#include "Python.h" /* Type specs */ extern PyType_Spec zstd_dict_type_spec; @@ -56,4 +55,4 @@ extern void set_parameter_error(const _zstd_state* const state, int is_compress, int key_v, int value_v); -#endif +#endif // !ZSTD_MODULE_H diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 7f9ada97b420fa..c902eef4f8e037 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -5,7 +5,7 @@ Python module. #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H -#include "Python.h" + #include "pycore_blocks_output_buffer.h" #include // ZSTD_outBuffer @@ -107,4 +107,4 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } -#endif +#endif // !ZSTD_BUFFER_H diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 9aee81bd92782a..e70eb637b29f3e 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -7,9 +7,9 @@ Python module. /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "zstd_compressor_type_spec" +class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=842f2c4f4520ab26]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index 91454ff6a2d181..e82498f5dd1901 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -5,7 +5,6 @@ Python module. #ifndef ZSTD_DICT_H #define ZSTD_DICT_H -#include "Python.h" #include // bool #include // ZSTD_DDict @@ -29,4 +28,4 @@ typedef struct { bool initialized; } ZstdDict; -#endif +#endif // !ZSTD_DICT_H