From 9c191331595bf96a45793a663dcd7e331d8560fb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 12:50:04 +0900 Subject: [PATCH 1/8] ci: add Python 3.13 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1faeb0ca..530238c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} From 4206af8f83ff74b8ba956377df1e2d28075f948b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 12:59:24 +0900 Subject: [PATCH 2/8] add /std:c++20 flag on win32 --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index a13bd81d..e3c80901 100644 --- a/setup.py +++ b/setup.py @@ -51,10 +51,12 @@ def __init__(self, *args, **kwargs): libraries = [] macros = [] +extra_compile_args = [] if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] + extra_compile_args = ["/std:c++20"] ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): @@ -65,6 +67,7 @@ def __init__(self, *args, **kwargs): libraries=libraries, include_dirs=["."], define_macros=macros, + extra_compile_args=extra_compile_args, ) ) del libraries, macros From 8e813d5c0932630b8a7e07ab68a6a8c1b8e4cea5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 13:47:00 +0900 Subject: [PATCH 3/8] use CFLAGS envvar instead of extra_compile_args --- setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e3c80901..2b553061 100644 --- a/setup.py +++ b/setup.py @@ -51,12 +51,17 @@ def __init__(self, *args, **kwargs): libraries = [] macros = [] -extra_compile_args = [] if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] - extra_compile_args = ["/std:c++20"] + cflags = os.environ.get("CFLAGS") + cxx20flag = "/std:c++20" + if cflags is None: + cflags = cxx20flag + elif cxx20flag not in cflags: + cflags += " " + cxx20flag + os.environ["CFLAGS"] = cflags ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): @@ -67,7 +72,6 @@ def __init__(self, *args, **kwargs): libraries=libraries, include_dirs=["."], define_macros=macros, - extra_compile_args=extra_compile_args, ) ) del libraries, macros From 6177331ac34cccfbd62abb4debb01dba06a07dc1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 14:04:54 +0900 Subject: [PATCH 4/8] test --- setup.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 2b553061..0498fe86 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,23 @@ #!/usr/bin/env python import os import sys + + +libraries = [] +macros = [] + +if sys.platform == "win32": + libraries.append("ws2_32") + macros = [("__LITTLE_ENDIAN__", "1")] + cflags = os.environ.get("CFLAGS") + cxx20flag = "/std:c++20" + if cflags is None: + cflags = cxx20flag + elif cxx20flag not in cflags: + cflags += " " + cxx20flag + os.environ["CFLAGS"] = cflags + + from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist @@ -49,20 +66,6 @@ def __init__(self, *args, **kwargs): sdist.__init__(self, *args, **kwargs) -libraries = [] -macros = [] - -if sys.platform == "win32": - libraries.append("ws2_32") - macros = [("__LITTLE_ENDIAN__", "1")] - cflags = os.environ.get("CFLAGS") - cxx20flag = "/std:c++20" - if cflags is None: - cflags = cxx20flag - elif cxx20flag not in cflags: - cflags += " " + cxx20flag - os.environ["CFLAGS"] = cflags - ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( From 6af994d7f5eeb65465c7d14184d18884c98a2673 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 14:16:45 +0900 Subject: [PATCH 5/8] CXXFLAGS --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0498fe86..ccea0350 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,13 @@ if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] - cflags = os.environ.get("CFLAGS") + cflags = os.environ.get("CXXFLAGS") cxx20flag = "/std:c++20" if cflags is None: cflags = cxx20flag elif cxx20flag not in cflags: cflags += " " + cxx20flag - os.environ["CFLAGS"] = cflags + os.environ["CXXFLAGS"] = cflags from setuptools import setup, Extension From 1203eb79a8c86d4fdf32588b2bca8a596955f0e0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 15:44:04 +0900 Subject: [PATCH 6/8] stop using c++ --- Makefile | 2 +- msgpack/_unpacker.pyx | 2 +- msgpack/unpack_container_header.h | 51 +++++++++++++++++++ msgpack/unpack_template.h | 85 ++++++++----------------------- setup.py | 28 ++++------ 5 files changed, 84 insertions(+), 84 deletions(-) create mode 100644 msgpack/unpack_container_header.h diff --git a/Makefile b/Makefile index 5c1863c7..3ce178f2 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ pyupgrade: .PHONY: cython cython: - cython --cplus msgpack/_cmsgpack.pyx + cython msgpack/_cmsgpack.pyx .PHONY: test test: cython diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 56126f43..2771e7bf 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -35,7 +35,7 @@ cdef extern from "unpack.h": PyObject* timestamp_t PyObject *giga; PyObject *utc; - char *unicode_errors + const char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len Py_ssize_t max_array_len diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h new file mode 100644 index 00000000..c14a3c2b --- /dev/null +++ b/msgpack/unpack_container_header.h @@ -0,0 +1,51 @@ +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8b9fcc19..0fd00586 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -75,8 +75,7 @@ static inline void unpack_clear(unpack_context *ctx) Py_CLEAR(ctx->stack[0].obj); } -template -static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -386,6 +385,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef construct_cb } +#undef NEXT_CS #undef SWITCH_RANGE_BEGIN #undef SWITCH_RANGE #undef SWITCH_RANGE_DEFAULT @@ -397,68 +397,27 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize #undef again_fixed_trail_if_zero #undef start_container -template -static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; - -#define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; -#ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: -#else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: -#endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; +static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(true, ctx, data, len, off); +} +static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(false, ctx, data, len, off); } -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END - -static const execute_fn unpack_construct = &unpack_execute; -static const execute_fn unpack_skip = &unpack_execute; -static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; - -#undef NEXT_CS +#define unpack_container_header read_array_header +#define fixed_offset 0x90 +#define var_offset 0xdc +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset + +#define unpack_container_header read_map_header +#define fixed_offset 0x80 +#define var_offset 0xde +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset /* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/setup.py b/setup.py index ccea0350..dc14a26e 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,6 @@ #!/usr/bin/env python import os import sys - - -libraries = [] -macros = [] - -if sys.platform == "win32": - libraries.append("ws2_32") - macros = [("__LITTLE_ENDIAN__", "1")] - cflags = os.environ.get("CXXFLAGS") - cxx20flag = "/std:c++20" - if cflags is None: - cflags = cxx20flag - elif cxx20flag not in cflags: - cflags += " " + cxx20flag - os.environ["CXXFLAGS"] = cflags - - from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist @@ -42,7 +25,7 @@ def cythonize(src): if not have_cython: raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src], cplus=True) + cython_compiler.compile([src]) def ensure_source(src): @@ -66,12 +49,19 @@ def __init__(self, *args, **kwargs): sdist.__init__(self, *args, **kwargs) +libraries = [] +macros = [] ext_modules = [] + +if sys.platform == "win32": + libraries.append("ws2_32") + macros = [("__LITTLE_ENDIAN__", "1")] + if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", - sources=["msgpack/_cmsgpack.cpp"], + sources=["msgpack/_cmsgpack.c"], libraries=libraries, include_dirs=["."], define_macros=macros, From 86260c787592eba11a2630ad07f9b86d3d6e2541 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 15:52:05 +0900 Subject: [PATCH 7/8] define bool --- msgpack/pack.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/pack.h b/msgpack/pack.h index 2453428c..901fad7b 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,6 +24,8 @@ #ifdef __cplusplus extern "C" { +#else +#define bool char #endif typedef struct msgpack_packer { From b0c44290cfcd4cb0a19fbfdfba9dd58eec40b3b7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 15:54:26 +0900 Subject: [PATCH 8/8] use 1/0 instead of true/false --- msgpack/unpack_template.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 0fd00586..cce29e7a 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -398,10 +398,10 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char #undef start_container static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(true, ctx, data, len, off); + return unpack_execute(1, ctx, data, len, off); } static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(false, ctx, data, len, off); + return unpack_execute(0, ctx, data, len, off); } #define unpack_container_header read_array_header