diff options
Diffstat (limited to 'modules/brotli/enc/backward_references_hq.c')
| -rw-r--r-- | modules/brotli/enc/backward_references_hq.c | 84 |
1 files changed, 51 insertions, 33 deletions
diff --git a/modules/brotli/enc/backward_references_hq.c b/modules/brotli/enc/backward_references_hq.c index 96b0e708d..5651caeb7 100644 --- a/modules/brotli/enc/backward_references_hq.c +++ b/modules/brotli/enc/backward_references_hq.c @@ -11,6 +11,7 @@ #include <string.h> /* memcpy, memset */ #include "../common/constants.h" +#include "../common/context.h" #include "../common/platform.h" #include <brotli/types.h> #include "./command.h" @@ -26,6 +27,7 @@ extern "C" { #endif +/* BrotliCalculateDistanceCodeLimit(BROTLI_MAX_ALLOWED_DISTANCE, 3, 120). */ #define BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE 544 static const float kInfinity = 1.7e38f; /* ~= 2 ^ 127 */ @@ -86,14 +88,10 @@ typedef struct ZopfliCostModel { static void InitZopfliCostModel( MemoryManager* m, ZopfliCostModel* self, const BrotliDistanceParams* dist, size_t num_bytes) { - uint32_t distance_histogram_size = dist->alphabet_size; - if (distance_histogram_size > BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE) { - distance_histogram_size = BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE; - } self->num_bytes_ = num_bytes; self->literal_costs_ = BROTLI_ALLOC(m, float, num_bytes + 2); - self->cost_dist_ = BROTLI_ALLOC(m, float, dist->alphabet_size); - self->distance_histogram_size = distance_histogram_size; + self->cost_dist_ = BROTLI_ALLOC(m, float, dist->alphabet_size_limit); + self->distance_histogram_size = dist->alphabet_size_limit; if (BROTLI_IS_OOM(m)) return; } @@ -408,9 +406,12 @@ static size_t UpdateNodes( const int* starting_dist_cache, const size_t num_matches, const BackwardMatch* matches, const ZopfliCostModel* model, StartPosQueue* queue, ZopfliNode* nodes) { + const size_t stream_offset = params->stream_offset; const size_t cur_ix = block_start + pos; const size_t cur_ix_masked = cur_ix & ringbuffer_mask; const size_t max_distance = BROTLI_MIN(size_t, cur_ix, max_backward_limit); + const size_t dictionary_start = BROTLI_MIN(size_t, + cur_ix + stream_offset, max_backward_limit); const size_t max_len = num_bytes - pos; const size_t max_zopfli_len = MaxZopfliLen(params); const size_t max_iters = MaxZopfliCandidates(params); @@ -419,8 +420,8 @@ static size_t UpdateNodes( size_t k; size_t gap = 0; - EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, - model, queue, nodes); + EvaluateNode(block_start + stream_offset, pos, max_backward_limit, gap, + starting_dist_cache, model, queue, nodes); { const PosData* posdata = StartPosQueueAt(queue, 0); @@ -453,7 +454,7 @@ static size_t UpdateNodes( if (cur_ix_masked + best_len > ringbuffer_mask) { break; } - if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) { + if (BROTLI_PREDICT_FALSE(backward > dictionary_start + gap)) { /* Word dictionary -> ignore. */ continue; } @@ -472,6 +473,8 @@ static size_t UpdateNodes( &ringbuffer[cur_ix_masked], max_len); } else { + /* "Gray" area. It is addressable by decoder, but this encoder + instance does not have that data -> should not touch it. */ continue; } { @@ -506,7 +509,7 @@ static size_t UpdateNodes( BackwardMatch match = matches[j]; size_t dist = match.distance; BROTLI_BOOL is_dictionary_match = - TO_BROTLI_BOOL(dist > max_distance + gap); + TO_BROTLI_BOOL(dist > dictionary_start + gap); /* We already tried all possible last distance matches, so we can use normal distance code here. */ size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1; @@ -569,6 +572,7 @@ void BrotliZopfliCreateCommands(const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes, int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params, Command* commands, size_t* num_literals) { + const size_t stream_offset = params->stream_offset; const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); size_t pos = 0; uint32_t offset = nodes[0].u.next; @@ -587,9 +591,10 @@ void BrotliZopfliCreateCommands(const size_t num_bytes, { size_t distance = ZopfliNodeCopyDistance(next); size_t len_code = ZopfliNodeLengthCode(next); - size_t max_distance = - BROTLI_MIN(size_t, block_start + pos, max_backward_limit); - BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap); + size_t dictionary_start = BROTLI_MIN(size_t, + block_start + pos + stream_offset, max_backward_limit); + BROTLI_BOOL is_dictionary = + TO_BROTLI_BOOL(distance > dictionary_start + gap); size_t dist_code = ZopfliNodeDistanceCode(next); InitCommand(&commands[i], ¶ms->dist, insert_length, copy_length, (int)len_code - (int)copy_length, dist_code); @@ -613,6 +618,7 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position, const BrotliEncoderParams* params, const size_t gap, const int* dist_cache, const ZopfliCostModel* model, const uint32_t* num_matches, const BackwardMatch* matches, ZopfliNode* nodes) { + const size_t stream_offset = params->stream_offset; const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); const size_t max_zopfli_len = MaxZopfliLen(params); StartPosQueue queue; @@ -637,7 +643,7 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position, while (skip) { i++; if (i + 3 >= num_bytes) break; - EvaluateNode(position, i, max_backward_limit, gap, + EvaluateNode(position + stream_offset, i, max_backward_limit, gap, dist_cache, model, &queue, nodes); cur_match_pos += num_matches[i]; skip--; @@ -650,8 +656,9 @@ static size_t ZopfliIterate(size_t num_bytes, size_t position, /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, - const BrotliEncoderParams* params, - const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes) { + ContextLut literal_context_lut, const BrotliEncoderParams* params, + const int* dist_cache, Hasher* hasher, ZopfliNode* nodes) { + const size_t stream_offset = params->stream_offset; const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); const size_t max_zopfli_len = MaxZopfliLen(params); ZopfliCostModel model; @@ -662,6 +669,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, size_t i; size_t gap = 0; size_t lz_matches_offset = 0; + BROTLI_UNUSED(literal_context_lut); nodes[0].length = 0; nodes[0].u.cost = 0; InitZopfliCostModel(m, &model, ¶ms->dist, num_bytes); @@ -672,12 +680,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; i++) { const size_t pos = position + i; const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit); + const size_t dictionary_start = BROTLI_MIN(size_t, + pos + stream_offset, max_backward_limit); size_t skip; size_t num_matches; - num_matches = FindAllMatchesH10(hasher, + num_matches = FindAllMatchesH10(&hasher->privat._H10, ¶ms->dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance, - gap, params, &matches[lz_matches_offset]); + dictionary_start + gap, params, &matches[lz_matches_offset]); if (num_matches > 0 && BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) { matches[0] = matches[num_matches - 1]; @@ -692,13 +702,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, } if (skip > 1) { /* Add the tail of the copy to the hasher. */ - StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN( + StoreRangeH10(&hasher->privat._H10, + ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN( size_t, pos + skip, store_end)); skip--; while (skip) { i++; if (i + HashTypeLengthH10() - 1 >= num_bytes) break; - EvaluateNode(position, i, max_backward_limit, gap, + EvaluateNode(position + stream_offset, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes); skip--; } @@ -710,15 +721,14 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, - const BrotliEncoderParams* params, - HasherHandle hasher, int* dist_cache, size_t* last_insert_len, + ContextLut literal_context_lut, const BrotliEncoderParams* params, + Hasher* hasher, int* dist_cache, size_t* last_insert_len, Command* commands, size_t* num_commands, size_t* num_literals) { - ZopfliNode* nodes; - nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1); - if (BROTLI_IS_OOM(m)) return; + ZopfliNode* nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1); + if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return; BrotliInitZopfliNodes(nodes, num_bytes + 1); *num_commands += BrotliZopfliComputeShortestPath(m, num_bytes, - position, ringbuffer, ringbuffer_mask, params, + position, ringbuffer, ringbuffer_mask, literal_context_lut, params, dist_cache, hasher, nodes); if (BROTLI_IS_OOM(m)) return; BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache, @@ -728,9 +738,10 @@ void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, - const BrotliEncoderParams* params, - HasherHandle hasher, int* dist_cache, size_t* last_insert_len, + ContextLut literal_context_lut, const BrotliEncoderParams* params, + Hasher* hasher, int* dist_cache, size_t* last_insert_len, Command* commands, size_t* num_commands, size_t* num_literals) { + const size_t stream_offset = params->stream_offset; const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); uint32_t* num_matches = BROTLI_ALLOC(m, uint32_t, num_bytes); size_t matches_size = 4 * num_bytes; @@ -747,10 +758,16 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size); size_t gap = 0; size_t shadow_matches = 0; - if (BROTLI_IS_OOM(m)) return; + BROTLI_UNUSED(literal_context_lut); + if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(num_matches) || + BROTLI_IS_NULL(matches)) { + return; + } for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) { const size_t pos = position + i; size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit); + size_t dictionary_start = BROTLI_MIN(size_t, + pos + stream_offset, max_backward_limit); size_t max_length = num_bytes - i; size_t num_found_matches; size_t cur_match_end; @@ -759,10 +776,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, BROTLI_ENSURE_CAPACITY(m, BackwardMatch, matches, matches_size, cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches); if (BROTLI_IS_OOM(m)) return; - num_found_matches = FindAllMatchesH10(hasher, + num_found_matches = FindAllMatchesH10(&hasher->privat._H10, ¶ms->dictionary, ringbuffer, ringbuffer_mask, pos, max_length, - max_distance, gap, params, + max_distance, dictionary_start + gap, params, &matches[cur_match_pos + shadow_matches]); cur_match_end = cur_match_pos + num_found_matches; for (j = cur_match_pos; j + 1 < cur_match_end; ++j) { @@ -777,7 +794,8 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, matches[cur_match_pos++] = matches[cur_match_end - 1]; num_matches[i] = 1; /* Add the tail of the copy to the hasher. */ - StoreRangeH10(hasher, ringbuffer, ringbuffer_mask, pos + 1, + StoreRangeH10(&hasher->privat._H10, + ringbuffer, ringbuffer_mask, pos + 1, BROTLI_MIN(size_t, pos + match_len, store_end)); memset(&num_matches[i + 1], 0, skip * sizeof(num_matches[0])); i += skip; @@ -791,7 +809,7 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, memcpy(orig_dist_cache, dist_cache, 4 * sizeof(dist_cache[0])); orig_num_commands = *num_commands; nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1); - if (BROTLI_IS_OOM(m)) return; + if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(nodes)) return; InitZopfliCostModel(m, &model, ¶ms->dist, num_bytes); if (BROTLI_IS_OOM(m)) return; for (i = 0; i < 2; i++) { |