From ae4dac4cfd3effa354498c7dc30d55ce7f8f777b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 06:41:43 +0000 Subject: [PATCH 1/7] fix Unpacker --- msgpack/_unpacker.pyx | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 27facc0a..33b94aba 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -455,19 +455,13 @@ cdef class Unpacker(object): cdef object obj cdef Py_ssize_t prev_head - if self.buf_head >= self.buf_tail and self.file_like is not None: - self.read_from_file() - while 1: prev_head = self.buf_head - if prev_head >= self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head + if prev_head < self.buf_tail: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head + else: + ret = 0 if ret == 1: obj = unpack_data(&self.ctx) @@ -477,10 +471,11 @@ cdef class Unpacker(object): if self.file_like is not None: self.read_from_file() continue - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") + if prev_head == self.buf_tail: + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") elif ret == -2: raise FormatError elif ret == -3: From f233d0bcec6ac0f6ccf72512374ad47479a25f30 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 07:20:59 +0000 Subject: [PATCH 2/7] Unpacker raises BufferFull when max_buffer_size is not enough. --- msgpack/_unpacker.pyx | 21 +++++++++++---------- test/test_sequnpack.py | 11 ++++++++++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 33b94aba..8b06661e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -440,15 +440,17 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) + cdef int read_from_file(self) except -1: + cdef Py_ssize_t remains = self.max_buffer_size - (self.buf_tail - self.buf_head) + if remains <= 0: + raise BufferFull + + next_bytes = self.file_like_read(min(self.read_size, remains)) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) else: self.file_like = None + return 0 cdef object _unpack(self, execute_fn execute, bint iter=0): cdef int ret @@ -471,11 +473,10 @@ cdef class Unpacker(object): if self.file_like is not None: self.read_from_file() continue - if prev_head == self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") elif ret == -2: raise FormatError elif ret == -3: diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9f20c07e..afd692e2 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -2,7 +2,7 @@ # coding: utf-8 import io from msgpack import Unpacker, BufferFull -from msgpack import pack +from msgpack import pack, packb from msgpack.exceptions import OutOfData from pytest import raises @@ -78,6 +78,15 @@ def test_maxbuffersize(): assert ord("b") == next(unpacker) +def test_maxbuffersize_file(): + buff = io.BytesIO(packb(b"a"*10) + packb(b"a"*100)) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=99) + assert unpacker.unpack() == b"a"*10 + #assert unpacker.unpack() == b"a"*100 + with raises(BufferFull): + unpacker.unpack() == b"a"*100 + + def test_readbytes(): unpacker = Unpacker(read_size=3) unpacker.feed(b"foobar") From 1531b7980befdd6cbde376501ba35d53a15e2327 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 07:22:25 +0000 Subject: [PATCH 3/7] black --- test/test_sequnpack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index afd692e2..6de4814b 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -79,12 +79,12 @@ def test_maxbuffersize(): def test_maxbuffersize_file(): - buff = io.BytesIO(packb(b"a"*10) + packb(b"a"*100)) + buff = io.BytesIO(packb(b"a" * 10) + packb(b"a" * 100)) unpacker = Unpacker(buff, read_size=1, max_buffer_size=99) - assert unpacker.unpack() == b"a"*10 - #assert unpacker.unpack() == b"a"*100 + assert unpacker.unpack() == b"a" * 10 + # assert unpacker.unpack() == b"a"*100 with raises(BufferFull): - unpacker.unpack() == b"a"*100 + unpacker.unpack() == b"a" * 100 def test_readbytes(): From 1436dfb53e55682b9338c14c0ee6a121102ab8ef Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 07:32:23 +0000 Subject: [PATCH 4/7] fix test --- test/test_sequnpack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6de4814b..300b94d9 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -79,12 +79,12 @@ def test_maxbuffersize(): def test_maxbuffersize_file(): - buff = io.BytesIO(packb(b"a" * 10) + packb(b"a" * 100)) - unpacker = Unpacker(buff, read_size=1, max_buffer_size=99) + buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20]*2)) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=19) assert unpacker.unpack() == b"a" * 10 - # assert unpacker.unpack() == b"a"*100 + #assert unpacker.unpack() == [b"a" * 20]*2 with raises(BufferFull): - unpacker.unpack() == b"a" * 100 + print(unpacker.unpack()) def test_readbytes(): From 8ef34c383f9272261a71f83c1846dfc8384da698 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 07:36:05 +0000 Subject: [PATCH 5/7] black --- test/test_sequnpack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 300b94d9..4e49c260 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -79,10 +79,10 @@ def test_maxbuffersize(): def test_maxbuffersize_file(): - buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20]*2)) + buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2)) unpacker = Unpacker(buff, read_size=1, max_buffer_size=19) assert unpacker.unpack() == b"a" * 10 - #assert unpacker.unpack() == [b"a" * 20]*2 + # assert unpacker.unpack() == [b"a" * 20]*2 with raises(BufferFull): print(unpacker.unpack()) From b1f06c33b9cdf0aa314c058a3a807f19f3d7b401 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 07:49:46 +0000 Subject: [PATCH 6/7] fix test --- test/test_sequnpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 4e49c260..c091076b 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -80,7 +80,7 @@ def test_maxbuffersize(): def test_maxbuffersize_file(): buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2)) - unpacker = Unpacker(buff, read_size=1, max_buffer_size=19) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=19, max_bin_len=20) assert unpacker.unpack() == b"a" * 10 # assert unpacker.unpack() == [b"a" * 20]*2 with raises(BufferFull): From 9f1aceb669bbcf7591da165662722d55c34ce5b1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 08:50:59 +0000 Subject: [PATCH 7/7] fallback raise BufferFull --- msgpack/fallback.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5f215e95..f560c7b5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -423,6 +423,8 @@ def _reserve(self, n, raise_outofdata=True): # Read from file remain_bytes = -remain_bytes + if remain_bytes + len(self._buffer) > self._max_buffer_size: + raise BufferFull while remain_bytes > 0: to_read_bytes = max(self._read_size, remain_bytes) read_data = self.file_like.read(to_read_bytes)