From dd0571d4bd7eed025ebb3c624b8abebd9b807282 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 2 Feb 2021 09:17:42 -0600 Subject: [PATCH] Update to brotlidecpy v1.0.2 --- included_dependencies/brotlidecpy/__init__.py | 2 +- .../brotlidecpy/bit_reader.py | 61 ++++++++++--------- included_dependencies/brotlidecpy/decode.py | 21 ++++--- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/included_dependencies/brotlidecpy/__init__.py b/included_dependencies/brotlidecpy/__init__.py index 8e10a688..595239e8 100644 --- a/included_dependencies/brotlidecpy/__init__.py +++ b/included_dependencies/brotlidecpy/__init__.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__version__ = "1.0.1" +__version__ = "1.0.2" # noinspection PyUnresolvedReferences from .decode import brotli_decompress_buffer as decompress diff --git a/included_dependencies/brotlidecpy/bit_reader.py b/included_dependencies/brotlidecpy/bit_reader.py index 8126de9f..6dd641a8 100644 --- a/included_dependencies/brotlidecpy/bit_reader.py +++ b/included_dependencies/brotlidecpy/bit_reader.py @@ -2,16 +2,17 @@ # Distributed under MIT license. # See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -kBitMask = [ - 0x000000, 0x000001, 0x000003, 0x000007, 0x00000f, 0x00001f, 0x00003f, 0x00007f, - 0x0000ff, 0x0001ff, 0x0003ff, 0x0007ff, 0x000fff, 0x001fff, 0x003fff, 0x007fff, - 0x00ffff, 0x01ffff, 0x03ffff, 0x07ffff, 0x0fffff, 0x1fffff, 0x3fffff, 0x7fffff, - 0xffffff -] - class BrotliBitReader: """Wrap a bytes buffer to enable reading 0 < n <=24 bits at a time, or transfer of arbitrary number of bytes""" + + kBitMask = [ + 0x000000, 0x000001, 0x000003, 0x000007, 0x00000f, 0x00001f, 0x00003f, 0x00007f, + 0x0000ff, 0x0001ff, 0x0003ff, 0x0007ff, 0x000fff, 0x001fff, 0x003fff, 0x007fff, + 0x00ffff, 0x01ffff, 0x03ffff, 0x07ffff, 0x0fffff, 0x1fffff, 0x3fffff, 0x7fffff, + 0xffffff + ] + def __init__(self, input_buffer): self.buf_ = bytearray(input_buffer) self.buf_len_ = len(input_buffer) @@ -23,28 +24,32 @@ class BrotliBitReader: self.pos_ = 0 self.bit_pos_ = 0 - def peek_bits(self, n_bits): - """Get value a n_bits unsigned integer treating input as little-endian byte stream, without advancing pointer""" + def read_bits(self, n_bits, bits_to_skip=None): + """Get n_bits unsigned integer treating input as little-endian byte stream, maybe advancing input buffer pointer + n_bits: is number of bits to read from input buffer. Set to None or 0 to seek ahead ignoring the value + bits_to_skip: number of bits to advance in input_buffer, defaults to n_bits if it is None + pass in 0 to peek at the next n_bits of value without advancing + It is ok to have n_bits and bits_to_skip be different non-zero values if that is what is wanted + Returns: the next n_bits from the buffer as a little-endian integer, 0 if n_bits is None or 0 + """ val = 0 - bytes_shift = 0 - buf_pos = self.pos_ - bit_pos_when_done = n_bits + self.bit_pos_ - while bytes_shift < bit_pos_when_done: - if buf_pos >= self.buf_len_: - break # if hit end of buffer, this simulates zero padding after end, which is correct - val |= self.buf_[buf_pos] << bytes_shift - bytes_shift += 8 - buf_pos += 1 - return (val >> self.bit_pos_) & kBitMask[n_bits] - - def skip_bits(self, n_bits): - next_in_bits = self.bit_pos_ + n_bits - self.bit_pos_ = next_in_bits & 7 - self.pos_ += next_in_bits >> 3 - - def read_bits(self, n_bits): - val = self.peek_bits(n_bits) - self.skip_bits(n_bits) + if bits_to_skip is None: + bits_to_skip = n_bits + if n_bits: + bytes_shift = 0 + buf_pos = self.pos_ + bit_pos_when_done = n_bits + self.bit_pos_ + while bytes_shift < bit_pos_when_done: + if buf_pos >= self.buf_len_: + break # if hit end of buffer, this simulates zero padding after end, which is correct + val |= self.buf_[buf_pos] << bytes_shift + bytes_shift += 8 + buf_pos += 1 + val = (val >> self.bit_pos_) & self.kBitMask[n_bits] + if bits_to_skip: + next_in_bits = self.bit_pos_ + bits_to_skip + self.bit_pos_ = next_in_bits & 7 + self.pos_ += next_in_bits >> 3 return val def copy_bytes(self, dest_buffer, dest_pos, n_bytes): diff --git a/included_dependencies/brotlidecpy/decode.py b/included_dependencies/brotlidecpy/decode.py index 133acf27..5ea2dd8a 100644 --- a/included_dependencies/brotlidecpy/decode.py +++ b/included_dependencies/brotlidecpy/decode.py @@ -104,14 +104,15 @@ def decode_meta_block_length(br): def read_symbol(table, index, br): - """Decodes the next Huffman code from bit-stream.""" - index += br.peek_bits(HUFFMAN_TABLE_BITS) + """Decodes the next Huffman code from bit-stream. table is array of nodes in a huffman tree, index points to root""" + x_bits = br.read_bits(16, 0) # The C reference version assumes 15 is the max needed and uses 16 in this function + index += (x_bits & HUFFMAN_TABLE_MASK) nbits = table[index].bits - HUFFMAN_TABLE_BITS + skip = 0 if nbits > 0: - br.skip_bits(HUFFMAN_TABLE_BITS) - index += table[index].value - index += br.peek_bits(nbits) - br.skip_bits(table[index].bits) + skip = HUFFMAN_TABLE_BITS + index += table[index].value + ((x_bits >> HUFFMAN_TABLE_BITS) & br.kBitMask[nbits]) + br.read_bits(None, skip + table[index].bits) return table[index].value @@ -128,8 +129,8 @@ def read_huffman_code_lengths(code_length_code_lengths, num_symbols, code_length while (symbol < num_symbols) and (space > 0): p = 0 - p += br.peek_bits(5) - br.skip_bits(table[p].bits) + p += br.read_bits(5, 0) + br.read_bits(None, table[p].bits) code_len = table[p].value & 0xff if code_len < kCodeLengthRepeatCode: repeat = 0 @@ -222,8 +223,8 @@ def read_huffman_code(alphabet_size, tables, table, br): break code_len_idx = kCodeLengthCodeOrder[i] p = 0 - p += br.peek_bits(4) - br.skip_bits(huff[p].bits) + p += br.read_bits(4, 0) + br.read_bits(None, huff[p].bits) v = huff[p].value code_length_code_lengths[code_len_idx] = v if v != 0: