ZXC 0.15.0
High-performance asymmetric lossless compression library
Loading...
Searching...
No Matches
zxc_decompress.c
Go to the documentation of this file.
1// SPDX-License-Identifier: BSD-3-Clause
2/*
3 * ZXC - High-performance lossless compression
4 *
5 * Copyright (c) Bertrand Lebonnois and contributors.
6 */
7
17// Function Multi-Versioning Support
18// With ZXC_FUNCTION_SUFFIX defined (e.g. _avx2), rename the entry point AND the
19// Huffman decoder this TU consumes. The defines precede zxc_internal.h so the
20// header's prototypes get the suffix too, keeping callers and callees matched.
21#ifdef ZXC_FUNCTION_SUFFIX
22#define ZXC_CAT_IMPL(x, y) x##y
23#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y)
24#define zxc_decompress_chunk_wrapper ZXC_CAT(zxc_decompress_chunk_wrapper, ZXC_FUNCTION_SUFFIX)
25#define zxc_decompress_chunk_wrapper_dict \
26 ZXC_CAT(zxc_decompress_chunk_wrapper_dict, ZXC_FUNCTION_SUFFIX)
27#define zxc_decompress_chunk_wrapper_safe \
28 ZXC_CAT(zxc_decompress_chunk_wrapper_safe, ZXC_FUNCTION_SUFFIX)
29#define zxc_huf_decode_section ZXC_CAT(zxc_huf_decode_section, ZXC_FUNCTION_SUFFIX)
30#define zxc_huf_decode_section_dict ZXC_CAT(zxc_huf_decode_section_dict, ZXC_FUNCTION_SUFFIX)
31#endif
32
33#include "../../include/zxc_error.h"
34#include "zxc_internal.h"
35
36#if defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) || defined(ZXC_USE_AVX2) || \
37 defined(ZXC_USE_AVX512)
43static const ZXC_ALIGN(32) uint8_t zxc_overlap_masks32[32][32] = {
44 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
45 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, // off=0 (unused)
46 {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
47 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, // off=1 (RLE handled separately)
48 {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
49 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}, // off=2
50 {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0,
51 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1}, // off=3
52 {0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3,
53 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3}, // off=4
54 {0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0,
55 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1}, // off=5
56 {0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
57 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1}, // off=6
58 {0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1,
59 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3}, // off=7
60 {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7,
61 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7}, // off=8
62 {0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6,
63 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4}, // off=9
64 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5,
65 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1}, // off=10
66 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4,
67 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // off=11
68 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3,
69 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7}, // off=12
70 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2,
71 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, 3, 4, 5}, // off=13
72 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1,
73 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, 2, 3}, // off=14
74 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0,
75 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, 1}, // off=15
76 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
77 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, // off=16
78 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
79 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}, // off=17
80 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
81 16, 17, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, // off=18
82 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
83 16, 17, 18, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, // off=19
84 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
85 16, 17, 18, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, // off=20
86 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
87 16, 17, 18, 19, 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // off=21
88 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
89 16, 17, 18, 19, 20, 21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // off=22
90 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
91 16, 17, 18, 19, 20, 21, 22, 0, 1, 2, 3, 4, 5, 6, 7, 8}, // off=23
92 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
93 16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7}, // off=24
94 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
95 16, 17, 18, 19, 20, 21, 22, 23, 24, 0, 1, 2, 3, 4, 5, 6}, // off=25
96 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
97 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 1, 2, 3, 4, 5}, // off=26
98 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
99 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 0, 1, 2, 3, 4}, // off=27
100 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
101 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 0, 1, 2, 3}, // off=28
102 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
103 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 0, 1, 2}, // off=29
104 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
105 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 0, 1}, // off=30
106 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
107 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 0} // off=31
108};
109#endif
110
117static const uint8_t zxc_overlap_strides32[32] = {32, 32, 32, 30, 32, 30, 30, 28, 32, 27, 30,
118 22, 24, 26, 28, 30, 32, 17, 18, 19, 20, 21,
119 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
120
142// codeql[cpp/unused-static-function] : False positive
143static ZXC_ALWAYS_INLINE void zxc_decode_copy_overlap_run32(uint8_t* dst, const uint32_t off,
144 const uint64_t ml) {
145 const size_t stride = zxc_overlap_strides32[off];
146 size_t copied = 0;
147#if defined(ZXC_USE_NEON64)
148 uint8x16x2_t tbl;
149 tbl.val[0] = vld1q_u8(dst - off);
150 tbl.val[1] = vld1q_u8(dst - off + 16);
151 const uint8x16_t pat_lo = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off]));
152 const uint8x16_t pat_hi = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off] + 16));
153 do {
154 vst1q_u8(dst + copied, pat_lo);
155 vst1q_u8(dst + copied + 16, pat_hi);
156 copied += stride;
157 } while (copied < ml);
158
159#elif defined(ZXC_USE_NEON32)
160 // VTBL reaches 8 bytes per lookup from a 4-register (32-byte) table, so the
161 // pattern takes four lookups instead of NEON64's two.
162 uint8x8x4_t tbl;
163 tbl.val[0] = vld1_u8(dst - off);
164 tbl.val[1] = vld1_u8(dst - off + 8);
165 tbl.val[2] = vld1_u8(dst - off + 16);
166 tbl.val[3] = vld1_u8(dst - off + 24);
167 const uint8_t* const m = zxc_overlap_masks32[off];
168 const uint8x8_t p0 = vtbl4_u8(tbl, vld1_u8(m));
169 const uint8x8_t p1 = vtbl4_u8(tbl, vld1_u8(m + 8));
170 const uint8x8_t p2 = vtbl4_u8(tbl, vld1_u8(m + 16));
171 const uint8x8_t p3 = vtbl4_u8(tbl, vld1_u8(m + 24));
172 do {
173 vst1_u8(dst + copied, p0);
174 vst1_u8(dst + copied + 8, p1);
175 vst1_u8(dst + copied + 16, p2);
176 vst1_u8(dst + copied + 24, p3);
177 copied += stride;
178 } while (copied < ml);
179
180#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512)
181 // pshufb only reaches 16 bytes per lane, so the 32-byte table is emulated:
182 // shuffle both halves, then pick per byte on bit 4 of the index.
183 const __m128i t0 = _mm_loadu_si128((const __m128i*)(dst - off));
184 const __m128i t1 = _mm_loadu_si128((const __m128i*)(dst - off + 16));
185 const __m128i sixteen = _mm_set1_epi8(16);
186 const __m128i m_lo = _mm_load_si128((const __m128i*)zxc_overlap_masks32[off]);
187 const __m128i m_hi = _mm_load_si128((const __m128i*)(zxc_overlap_masks32[off] + 16));
188 const __m128i pat_lo =
189 _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_lo),
190 _mm_shuffle_epi8(t1, _mm_sub_epi8(m_lo, sixteen)), _mm_slli_epi16(m_lo, 3));
191 const __m128i pat_hi =
192 _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_hi),
193 _mm_shuffle_epi8(t1, _mm_sub_epi8(m_hi, sixteen)), _mm_slli_epi16(m_hi, 3));
194 do {
195 _mm_storeu_si128((__m128i*)(dst + copied), pat_lo);
196 _mm_storeu_si128((__m128i*)(dst + copied + 16), pat_hi);
197 copied += stride;
198 } while (copied < ml);
199
200#else
201 // SSE2 (no PSHUFB) and non-SIMD tiers: build the 32-byte pattern with a
202 // wrap counter (no per-byte modulo), then store it as two 16-byte halves.
203 const uint8_t* src = dst - off;
204 uint8_t pat[32];
205 uint32_t k = 0;
206 for (size_t i = 0; i < 32; i++) {
207 pat[i] = src[k];
208 if (++k == off) k = 0;
209 }
210 do {
211 zxc_copy16(dst + copied, pat);
212 zxc_copy16(dst + copied + 16, pat + 16);
213 copied += stride;
214 } while (copied < ml);
215#endif
216}
217
237// codeql[cpp/unused-static-function] : False positive
238static ZXC_ALWAYS_INLINE void zxc_decode_copy_overlap_short(uint8_t* dst, const uint32_t off,
239 const uint64_t ml) {
240#if defined(ZXC_USE_NEON64)
241 const uint8x16_t s0 = vld1q_u8(dst - off);
242 vst1q_u8(dst, vqtbl1q_u8(s0, vld1q_u8(zxc_overlap_masks32[off])));
243 if (UNLIKELY(ml > 16)) {
244 uint8x16x2_t tbl;
245 tbl.val[0] = s0;
246 tbl.val[1] = vld1q_u8(dst - off + 16);
247 vst1q_u8(dst + 16, vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off] + 16)));
248 }
249
250#elif defined(ZXC_USE_NEON32)
251 uint8x8x2_t lo_tbl;
252 lo_tbl.val[0] = vld1_u8(dst - off);
253 lo_tbl.val[1] = vld1_u8(dst - off + 8);
254 const uint8_t* const m = zxc_overlap_masks32[off];
255 vst1_u8(dst, vtbl2_u8(lo_tbl, vld1_u8(m)));
256 vst1_u8(dst + 8, vtbl2_u8(lo_tbl, vld1_u8(m + 8)));
257 if (UNLIKELY(ml > 16)) {
258 uint8x8x4_t tbl;
259 tbl.val[0] = lo_tbl.val[0];
260 tbl.val[1] = lo_tbl.val[1];
261 tbl.val[2] = vld1_u8(dst - off + 16);
262 tbl.val[3] = vld1_u8(dst - off + 24);
263 vst1_u8(dst + 16, vtbl4_u8(tbl, vld1_u8(m + 16)));
264 vst1_u8(dst + 24, vtbl4_u8(tbl, vld1_u8(m + 24)));
265 }
266
267#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512)
268 // pshufb reaches 16 bytes, so only the upper half needs two tables: shuffle
269 // both and pick per byte on bit 4 of the index.
270 const __m128i t0 = _mm_loadu_si128((const __m128i*)(dst - off));
271 _mm_storeu_si128(
272 (__m128i*)dst,
273 _mm_shuffle_epi8(t0, _mm_load_si128((const __m128i*)zxc_overlap_masks32[off])));
274 if (UNLIKELY(ml > 16)) {
275 const __m128i t1 = _mm_loadu_si128((const __m128i*)(dst - off + 16));
276 const __m128i m_hi = _mm_load_si128((const __m128i*)(zxc_overlap_masks32[off] + 16));
277 _mm_storeu_si128(
278 (__m128i*)(dst + 16),
279 _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_hi),
280 _mm_shuffle_epi8(t1, _mm_sub_epi8(m_hi, _mm_set1_epi8(16))),
281 _mm_slli_epi16(m_hi, 3)));
282 }
283
284#else
285 const uint8_t* src = dst - off;
286 const size_t n = (ml > 16) ? 32 : 16;
287 uint32_t k = 0;
288 for (size_t i = 0; i < n; i++) {
289 dst[i] = src[k];
290 if (++k == off) k = 0;
291 }
292#endif
293}
294
308// codeql[cpp/unused-static-function] : False positive, used in DECODE_MATCH_SAFE/FAST macros
309static ZXC_ALWAYS_INLINE void zxc_decode_fill_run(uint8_t* dst, const uint8_t byte,
310 const uint64_t ml) {
311#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512)
312 const __m256i v = _mm256_set1_epi8((char)byte);
313 _mm256_storeu_si256((__m256i*)dst, v);
314 if (UNLIKELY(ml > 32)) {
315 uint8_t* p = dst;
316 const uint8_t* const last = dst + ml - 32;
317 do {
318 p += 32;
319 _mm256_storeu_si256((__m256i*)p, v);
320 } while (p < last);
321 }
322#elif defined(ZXC_USE_SSE2)
323 const __m128i v = _mm_set1_epi8((char)byte);
324 _mm_storeu_si128((__m128i*)dst, v);
325 _mm_storeu_si128((__m128i*)(dst + 16), v);
326 if (UNLIKELY(ml > 32)) {
327 uint8_t* p = dst;
328 const uint8_t* const last = dst + ml - 32;
329 do {
330 p += 32;
331 _mm_storeu_si128((__m128i*)p, v);
332 _mm_storeu_si128((__m128i*)(p + 16), v);
333 } while (p < last);
334 }
335#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32)
336 const uint8x16_t v = vdupq_n_u8(byte);
337 vst1q_u8(dst, v);
338 vst1q_u8(dst + 16, v);
339 if (UNLIKELY(ml > 32)) {
340 uint8_t* p = dst;
341 const uint8_t* const last = dst + ml - 32;
342 do {
343 p += 32;
344 vst1q_u8(p, v);
345 vst1q_u8(p + 16, v);
346 } while (p < last);
347 }
348#else
349 ZXC_MEMSET(dst, byte, ml);
350#endif
351}
352
353// ==========================================================================
354// Shared decode macros for the GLO and GHI decoders (fast + safe variants).
355// Defined at file scope to avoid four identical copies inside each function.
356// They reference the local names l_ptr, d_ptr, d_floor that every call site
357// has in scope. #undef-ed at the end of the last consumer.
358// ==========================================================================
359
372static ZXC_ALWAYS_INLINE void zxc_wild_copy32(uint8_t* dst, const uint8_t* src, const size_t len) {
373 zxc_copy32(dst, src);
374 if (UNLIKELY(len > 32)) {
375 size_t rem = len - 32;
376 dst += 32;
377 src += 32;
378 while (rem > 32) {
379 zxc_copy32(dst, src);
380 dst += 32;
381 src += 32;
382 rem -= 32;
383 }
384 zxc_copy32(dst, src);
385 }
386}
387
403static ZXC_ALWAYS_INLINE void zxc_decode_copy_literals(uint8_t* RESTRICT dst,
404 const uint8_t* RESTRICT src,
405 const uint64_t ll) {
406 zxc_wild_copy32(dst, src, ll);
407}
408
432static ZXC_ALWAYS_INLINE void zxc_decode_copy_match(uint8_t* RESTRICT d_ptr, const uint32_t off,
433 const uint64_t ml) {
434 const uint8_t* match_src = d_ptr - off;
435 if (LIKELY(off >= 32)) {
436 zxc_wild_copy32(d_ptr, match_src, ml);
437 } else if (off == 1) {
438 zxc_decode_fill_run(d_ptr, match_src[0], ml);
439 } else {
440 zxc_decode_copy_overlap_run32(d_ptr, off, ml);
441 }
442}
443
459static ZXC_ALWAYS_INLINE void zxc_decode_copy_match_short(uint8_t* RESTRICT d_ptr,
460 const uint32_t off, const uint64_t ml) {
461 const uint8_t* match_src = d_ptr - off;
462 if (LIKELY(off >= 32)) {
463 zxc_copy32(d_ptr, match_src);
464 } else if (off == 1) {
465 zxc_decode_fill_run(d_ptr, match_src[0], ml);
466 } else {
467 zxc_decode_copy_overlap_short(d_ptr, off, ml);
468 }
469}
470
478static ZXC_ALWAYS_INLINE void zxc_decode_copy_match_glo(uint8_t* RESTRICT d_ptr, const uint32_t off,
479 const uint64_t ml) {
480 if (LIKELY(ml <= ZXC_GLO_MAX_INLINE_ML)) {
481 zxc_decode_copy_match_short(d_ptr, off, ml);
482 } else {
483 zxc_decode_copy_match(d_ptr, off, ml);
484 }
485}
486
503static ZXC_NOINLINE void zxc_decode_copy_match_exact(uint8_t* d_ptr, const uint8_t* match_src,
504 const size_t off, const size_t ml) {
505 if (off >= ml) {
506 ZXC_MEMCPY(d_ptr, match_src, ml);
507 } else if (ml < 16) {
508 for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i];
509 } else {
510 size_t n = off;
511 ZXC_MEMCPY(d_ptr, match_src, n);
512 while (n <= ml - n) {
513 ZXC_MEMCPY(d_ptr + n, d_ptr, n);
514 n <<= 1;
515 }
516 ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n);
517 }
518}
519
520// Match emission only. The literal copy lives in the per-format sequence macros
521// because the two formats want different strategies: GLO's inline ll is at most
522// 14, so one zxc_copy16 covers it, while GHI's reaches 254 and needs the
523// 32-byte ladder.
524
525// SAFE version: rejects a match reaching below d_floor. COPY is the format's
526// match-copy helper, so GLO can pass the length-bounded one.
527#define DECODE_MATCH_SAFE(ml, off, COPY) \
528 do { \
529 if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \
530 COPY(d_ptr, off, ml); \
531 d_ptr += ml; \
532 } while (0)
533
534// FAST version: no offset check. Only reached past d_bounds, where no encodable
535// offset can reach below d_floor, so the check above would always pass.
536#define DECODE_MATCH_FAST(ml, off, COPY) \
537 do { \
538 COPY(d_ptr, off, ml); \
539 d_ptr += ml; \
540 } while (0)
541
553#define GLO_OFF8 (gh.enc_off == 1)
554
568#define DECODE_GLO_SEQ(LL, ML, OFF, RESERVE, N_REM, DECODE, ON_FAIL) \
569 do { \
570 uint64_t ll = (LL); \
571 uint64_t ml = (ML); \
572 if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \
573 ll += zxc_read_varint(&e_ptr, e_end); \
574 const uint64_t reserve = (RESERVE); \
575 /* An extended ll eats the batch's destination budget, so reserve \
576 * this sequence's match and the remaining ones too. ml is still \
577 * the raw nibble here; an escaped one re-checks itself below. */ \
578 if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \
579 ll + ml + ZXC_LZ_MIN_MATCH_LEN + \
580 (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \
581 (size_t)(d_end - d_ptr))) \
582 ON_FAIL; \
583 zxc_decode_copy_literals(d_ptr, l_ptr, ll); \
584 } else { \
585 /* ll <= 14 here, so one 16-byte store covers it. */ \
586 zxc_copy16(d_ptr, l_ptr); \
587 } \
588 l_ptr += ll; \
589 d_ptr += ll; \
590 if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \
591 ml += zxc_read_varint(&e_ptr, e_end); \
592 /* d_ptr already carries the literals, hence no `ll +` term here. */ \
593 if (UNLIKELY(ml + ZXC_LZ_MIN_MATCH_LEN + (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + \
594 ZXC_PAD_SIZE > \
595 (size_t)(d_end - d_ptr))) \
596 ON_FAIL; \
597 } \
598 ml += ZXC_LZ_MIN_MATCH_LEN; \
599 DECODE(ml, OFF, zxc_decode_copy_match_glo); \
600 } while (0)
601
610#define DECODE_GLO_BATCH_4X(DECODE, ON_FAIL) \
611 do { \
612 uint32_t tokens = zxc_le32(t_ptr); \
613 t_ptr += sizeof(uint32_t); \
614 uint64_t offs; \
615 if (GLO_OFF8) { \
616 uint64_t x = zxc_le32(o_ptr); \
617 o_ptr += sizeof(uint32_t); \
618 x = (x | (x << 16)) & 0x0000FFFF0000FFFFULL; \
619 offs = (x | (x << 8)) & 0x00FF00FF00FF00FFULL; \
620 } else { \
621 offs = zxc_le64(o_ptr); \
622 o_ptr += sizeof(uint64_t); \
623 } \
624 DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), \
625 ZXC_LZ_OFFSET_BIAS + (uint32_t)(offs & 0xFFFF), \
626 ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \
627 DECODE, ON_FAIL); \
628 DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, \
629 ZXC_LZ_OFFSET_BIAS + (uint32_t)((offs >> 16) & 0xFFFF), \
630 ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \
631 DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, \
632 ZXC_LZ_OFFSET_BIAS + (uint32_t)((offs >> 32) & 0xFFFF), (tokens >> 28), 1U, \
633 DECODE, ON_FAIL); \
634 DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, \
635 ZXC_LZ_OFFSET_BIAS + (uint32_t)(offs >> 48), 0, 0U, DECODE, ON_FAIL); \
636 n_seq -= 4; \
637 } while (0)
638
644#define DECODE_GHI_SEQ(S, RESERVE, N_REM, DECODE, ON_FAIL) \
645 do { \
646 uint64_t ll = (S) >> 24; \
647 const uint32_t mb = ((S) >> 16) & 0xFF; \
648 uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \
649 if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \
650 ll += zxc_read_varint(&extras_ptr, extras_end); \
651 const uint64_t reserve = (RESERVE); \
652 /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \
653 if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \
654 ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \
655 (size_t)(d_end - d_ptr))) \
656 ON_FAIL; \
657 } \
658 if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \
659 ml += zxc_read_varint(&extras_ptr, extras_end); \
660 if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \
661 (size_t)(d_end - d_ptr))) \
662 ON_FAIL; \
663 } \
664 const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \
665 /* GHI keeps the ladder: its inline ll reaches 254. */ \
666 zxc_decode_copy_literals(d_ptr, l_ptr, ll); \
667 l_ptr += ll; \
668 d_ptr += ll; \
669 DECODE(ml, off, zxc_decode_copy_match); \
670 } while (0)
671
677#define DECODE_GHI_BATCH_4X(DECODE, ON_FAIL, PREFETCH) \
678 do { \
679 uint32_t s1 = zxc_le32(seq_ptr); \
680 uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \
681 uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \
682 uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \
683 seq_ptr += 4 * sizeof(uint32_t); \
684 PREFETCH; \
685 DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \
686 DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \
687 DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \
688 DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \
689 n_seq -= 4; \
690 } while (0)
691
706static ZXC_NOINLINE ZXC_COLD int zxc_ensure_entropy_scratch(const zxc_cctx_t* RESTRICT ctx) {
707 if (LIKELY(ctx->pivco_scratch)) return ZXC_OK;
708 return zxc_cctx_alloc_entropy_scratch((zxc_cctx_t*)(uintptr_t)ctx);
709}
710
711static ZXC_NOINLINE ZXC_COLD int zxc_decode_lit_pivco(const zxc_cctx_t* RESTRICT ctx,
712 const uint8_t* RESTRICT payload,
713 const size_t psize,
714 const size_t required_size) {
715 const int arc = zxc_ensure_entropy_scratch(ctx);
716 if (UNLIKELY(arc != ZXC_OK)) return arc;
717 if (UNLIKELY(ctx->lit_buffer_cap < required_size + ZXC_PAD_SIZE ||
718 ctx->pivco_scratch_cap < required_size + ZXC_PIVCO_SCRATCH_PAD))
719 return ZXC_ERROR_DST_TOO_SMALL;
720 return zxc_huf_decode_section(payload, psize, ctx->lit_buffer, required_size,
721 ctx->pivco_scratch);
722}
723
724static ZXC_NOINLINE ZXC_COLD int zxc_decode_lit_pivco_dict(const zxc_cctx_t* RESTRICT ctx,
725 const uint8_t* RESTRICT payload,
726 const size_t psize,
727 const size_t required_size) {
728 if (UNLIKELY(!ctx->dict_huf_tree_ok)) return ZXC_ERROR_DICT_REQUIRED;
729 const int arc = zxc_ensure_entropy_scratch(ctx);
730 if (UNLIKELY(arc != ZXC_OK)) return arc;
731 if (UNLIKELY(ctx->lit_buffer_cap < required_size + ZXC_PAD_SIZE ||
732 ctx->pivco_scratch_cap < required_size + ZXC_PIVCO_SCRATCH_PAD))
733 return ZXC_ERROR_DST_TOO_SMALL;
734 return zxc_huf_decode_section_dict(payload, psize, ctx->lit_buffer, required_size,
735 &ctx->dict_huf->tree, ctx->pivco_scratch);
736}
737
738static ZXC_NOINLINE ZXC_COLD int zxc_decode_tok_pivco(const zxc_cctx_t* RESTRICT ctx,
739 const uint8_t* RESTRICT payload,
740 const size_t psize, const size_t n_tok) {
741 const int arc = zxc_ensure_entropy_scratch(ctx);
742 if (UNLIKELY(arc != ZXC_OK)) return arc;
743 if (UNLIKELY(n_tok + ZXC_PAD_SIZE > ctx->tok_buffer_cap ||
744 n_tok + ZXC_PIVCO_SCRATCH_PAD > ctx->pivco_scratch_cap))
745 return ZXC_ERROR_DST_TOO_SMALL;
746 return zxc_huf_decode_section(payload, psize, ctx->tok_buffer, n_tok, ctx->pivco_scratch);
747}
748
749static ZXC_NOINLINE int zxc_decode_block_glo_entropy(const zxc_cctx_t* RESTRICT ctx,
750 const uint8_t* RESTRICT src, size_t src_size,
751 uint8_t* RESTRICT dst, size_t dst_capacity);
752static ZXC_NOINLINE int zxc_decode_block_glo_entropy_dict(const zxc_cctx_t* RESTRICT ctx,
753 const uint8_t* RESTRICT src,
754 size_t src_size, uint8_t* RESTRICT dst,
755 size_t dst_capacity);
756static ZXC_NOINLINE int zxc_decode_block_glo_entropy_safe(const zxc_cctx_t* RESTRICT ctx,
757 const uint8_t* RESTRICT src,
758 size_t src_size, uint8_t* RESTRICT dst,
759 size_t dst_capacity);
760
789static ZXC_ALWAYS_INLINE int zxc_decode_block_glo_impl(const zxc_cctx_t* RESTRICT ctx,
790 const uint8_t* RESTRICT src,
791 const size_t src_size, uint8_t* RESTRICT dst,
792 const size_t dst_capacity, const int safe,
793 const int has_dict, const int tok_entropy) {
794 zxc_gnr_header_t gh;
795
796 // Constant 0 when !has_dict, so `d_floor` folds to `dst`.
797 const size_t dict_size = has_dict ? ctx->dict_size : 0;
798 uint32_t lit_comp;
799 uint32_t tok_comp;
800
801 const int hdr_sz = zxc_read_glo_header_and_desc(src, src_size, &gh, &lit_comp, &tok_comp);
802 if (UNLIKELY(hdr_sz < 0)) return ZXC_ERROR_BAD_HEADER;
803
804 // Only 0 and 1 are defined widths.
805 if (UNLIKELY(gh.enc_off > 1)) return ZXC_ERROR_CORRUPT_DATA;
806
807 // Entropy-coded tokens (level 7): tail-call the dedicated instantiation
808 // before any section work - it restarts from the header, so only the parse
809 // above is duplicated. Flags are constant per instantiation, so exactly one
810 // call survives and t_ptr keeps a single provenance below.
811 if (!tok_entropy && UNLIKELY(gh.enc_tok == ZXC_SECTION_ENCODING_HUFFMAN)) {
812 if (safe) return zxc_decode_block_glo_entropy_safe(ctx, src, src_size, dst, dst_capacity);
813 if (has_dict)
814 return zxc_decode_block_glo_entropy_dict(ctx, src, src_size, dst, dst_capacity);
815 return zxc_decode_block_glo_entropy(ctx, src, src_size, dst, dst_capacity);
816 }
817
818 const uint8_t* p_data = src + (size_t)hdr_sz;
819 const uint8_t* p_curr = p_data;
820
821 // --- Literal Stream Setup ---
822 const uint8_t* l_ptr;
823 const uint8_t* l_end;
824 uint8_t* rle_buf = NULL;
825
826 size_t lit_stream_size = lit_comp;
827 // Decoded size of an encoded literal section (RAW ignores it).
828 const size_t required_size = gh.n_literals;
829
830 if (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN ||
831 gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN_DICT) {
832 if (UNLIKELY(lit_stream_size > (size_t)(src + src_size - p_curr)))
833 return ZXC_ERROR_CORRUPT_DATA;
834 if (required_size == 0) {
835 l_ptr = p_curr;
836 l_end = p_curr;
837 } else {
838 if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE))
839 return ZXC_ERROR_DST_TOO_SMALL;
840 const int rc =
841 (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN)
842 ? zxc_decode_lit_pivco(ctx, p_curr, lit_stream_size, required_size)
843 : zxc_decode_lit_pivco_dict(ctx, p_curr, lit_stream_size, required_size);
844 if (UNLIKELY(rc != ZXC_OK)) return rc;
845 l_ptr = ctx->lit_buffer;
846 l_end = ctx->lit_buffer + required_size;
847 }
848 } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RLE) {
849 if (required_size > 0) {
850 if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE))
851 return ZXC_ERROR_DST_TOO_SMALL;
852 const size_t alloc_size = required_size + ZXC_PAD_SIZE;
853
854 // lit_buffer is pre-allocated to chunk_size + ZXC_PAD_SIZE by
855 // zxc_cctx_init (mode == 0).
856 if (UNLIKELY(ctx->lit_buffer_cap < alloc_size)) return ZXC_ERROR_DST_TOO_SMALL;
857
858 rle_buf = ctx->lit_buffer;
859 if (UNLIKELY(!rle_buf || lit_stream_size > (size_t)(src + src_size - p_curr)))
860 return ZXC_ERROR_CORRUPT_DATA;
861
862 const uint8_t* r_ptr = p_curr;
863 const uint8_t* r_end = r_ptr + lit_stream_size;
864 uint8_t* w_ptr = rle_buf;
865 const uint8_t* const w_end = rle_buf + required_size;
866
867 while (r_ptr < r_end && w_ptr < w_end) {
868 uint8_t token = *r_ptr++;
869 if (LIKELY(!(token & ZXC_LIT_RLE_FLAG))) {
870 // Raw copy (most common path): 32-byte wild copies (zxc_copy32 width)
871 // token is 7-bit (0-127), so len is 1-128 bytes
872 const uint32_t len = (uint32_t)token + 1;
873 if (UNLIKELY(w_ptr + len > w_end || r_ptr + len > r_end))
874 return ZXC_ERROR_CORRUPT_DATA;
875
876 // Destination has ZXC_PAD_SIZE bytes of safe overrun space.
877 // Source may not - check before wild copy.
878 // Fast path: source holds a full zxc_copy32 read (most common)
879 if (LIKELY(r_ptr + 32 <= r_end)) {
880 // Single copy covers len <= 32 (most tokens)
881 zxc_copy32(w_ptr, r_ptr);
882
883 if (UNLIKELY(len > 32)) {
884 // Unroll: max len=128, so max 4 copies total. The last
885 // copy is placed to end exactly on len, overlapping the
886 // previous one - unconditional stores beat branches here.
887 if (len <= 64) {
888 zxc_copy32(w_ptr + len - 32, r_ptr + len - 32);
889 } else if (len <= 96) {
890 zxc_copy32(w_ptr + 32, r_ptr + 32);
891 zxc_copy32(w_ptr + len - 32, r_ptr + len - 32);
892 } else {
893 zxc_copy32(w_ptr + 32, r_ptr + 32);
894 zxc_copy32(w_ptr + 64, r_ptr + 64);
895 zxc_copy32(w_ptr + len - 32, r_ptr + len - 32);
896 }
897 }
898 } else {
899 // Near end of source: safe copy (rare cold path)
900 ZXC_MEMCPY(w_ptr, r_ptr, len);
901 }
902
903 w_ptr += len;
904 r_ptr += len;
905 } else {
906 // RLE run: fill with single byte
907 const uint32_t len = (token & ZXC_LIT_LEN_MASK) + 4;
908 if (UNLIKELY(w_ptr + len > w_end || r_ptr >= r_end))
909 return ZXC_ERROR_CORRUPT_DATA;
910 ZXC_MEMSET(w_ptr, *r_ptr++, len);
911 w_ptr += len;
912 }
913 }
914 if (UNLIKELY(w_ptr != w_end || r_ptr != r_end)) return ZXC_ERROR_CORRUPT_DATA;
915 l_ptr = rle_buf;
916 l_end = rle_buf + required_size;
917 } else {
918 if (UNLIKELY(lit_stream_size != 0)) return ZXC_ERROR_CORRUPT_DATA;
919 l_ptr = p_curr;
920 l_end = p_curr;
921 }
922 } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RAW) {
923 l_ptr = p_curr;
924 l_end = p_curr + lit_stream_size;
925 } else {
926 return ZXC_ERROR_CORRUPT_DATA;
927 }
928
929 p_curr += lit_stream_size;
930
931 // --- Stream Pointers & Validation ---
932 // Only the literal and token sizes are on the wire; offsets follow from the
933 // sequence count and width, and extras take the payload residue.
934 const size_t sz_tokens = tok_comp;
935 const uint64_t sz_offsets = GLO_OFF8 ? (uint64_t)gh.n_sequences : (uint64_t)gh.n_sequences * 2;
936
937 const size_t payload_avail = (size_t)(src + src_size - p_data);
938 const uint64_t consumed = (uint64_t)lit_stream_size + (uint64_t)sz_tokens + sz_offsets;
939 if (UNLIKELY(consumed > (uint64_t)payload_avail)) return ZXC_ERROR_CORRUPT_DATA;
940 const size_t sz_extras = payload_avail - (size_t)consumed; /* slack padding included */
941
942 // RAW literals point into the caller's buffer and the wild copy overshoots,
943 // so it needs ZXC_BLOCK_LIT_SLACK readable bytes behind it. v7 staged short
944 // streams into a padded scratch here; v8 rejects instead. No underflow:
945 // lit_stream_size <= consumed <= payload_avail.
946 if (UNLIKELY(payload_avail - lit_stream_size < ZXC_BLOCK_LIT_SLACK))
947 return ZXC_ERROR_CORRUPT_DATA;
948
949 // Offsets/extras follow the on-disk token SECTION; sz_tokens is its size
950 // (== n_sequences when RAW, the Huffman payload size when enc_tok set).
951 const uint8_t* o_ptr = p_curr + sz_tokens;
952 const uint8_t* e_ptr = o_ptr + (size_t)sz_offsets;
953 const uint8_t* const e_end = e_ptr + sz_extras;
954
955 const uint8_t* RESTRICT t_ptr;
956 if (!tok_entropy) {
957 // enc_tok == 2 was re-routed right after the header parse; any other
958 // value would have mis-sized the section descriptors above.
959 if (UNLIKELY(gh.enc_tok != 0)) return ZXC_ERROR_CORRUPT_DATA;
960 t_ptr = p_curr;
961 } else {
962 if (UNLIKELY(gh.enc_tok != ZXC_SECTION_ENCODING_HUFFMAN)) return ZXC_ERROR_CORRUPT_DATA;
963 const int rc = zxc_decode_tok_pivco(ctx, p_curr, sz_tokens, gh.n_sequences);
964 if (UNLIKELY(rc != ZXC_OK)) return rc;
965 t_ptr = ctx->tok_buffer;
966 }
967
968 uint8_t* d_ptr = dst;
969 const uint8_t* const d_end = dst + dst_capacity;
970 // Lowest address a match may reach: the dictionary prefix if any, else dst.
971 const uint8_t* const d_floor = dst - dict_size;
972 // Destination safe margin for 4x loop: max output without varint extension.
973 // ll_max = 14, ml_max = 14 + 5 = 19, per-seq = 33, 4x = 132.
974 // Plus the overshoot the wild copies are allowed (ZXC_PAD_SIZE) + 4 safety = 168.
975 const uint8_t* const d_end_safe =
976 (dst_capacity > 132 + ZXC_PAD_SIZE + 4) ? d_end - (132 + ZXC_PAD_SIZE + 4) : dst;
977
978 // Literal margin for the 4x loops: without a varint, ll <= 14 per sequence,
979 // so 4 * 14 = 56. Past that margin only the cold varint path checks l_ptr.
980 const size_t glo_sz_lit = (size_t)(l_end - l_ptr);
981 const size_t glo_margin_4x = 4 * (ZXC_TOKEN_LL_MASK - 1); // 56
982 const size_t glo_margin_1x = ZXC_TOKEN_LL_MASK - 1; // 14
983 const uint8_t* const l_end_safe_4x =
984 (glo_sz_lit > glo_margin_4x) ? l_end - glo_margin_4x : l_ptr;
985 const uint8_t* const l_end_safe_1x =
986 (glo_sz_lit > glo_margin_1x) ? l_end - glo_margin_1x : l_ptr;
987
988 uint32_t n_seq = gh.n_sequences;
989
990 // Offsets only need checking until the output is wider than the widest offset
991 // the format can encode (256 for 1-byte offsets, 65536 for 2-byte); past that
992 // none can reach below d_floor. A dictionary counts as already written, so a
993 // large one puts d_bounds at dst and skips the SAFE loop.
994 const size_t off_span = GLO_OFF8 ? (1U << 8) : (1U << 16);
995 const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size;
996 const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span;
997
998 // --- SAFE Loop: offset validation until d_bounds (4x unroll) ---
999 if (safe) {
1000 // SAFE variant: save per-batch state so overflow can rollback.
1001 while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) {
1002 const uint8_t* const t_save = t_ptr;
1003 const uint8_t* const o_save = o_ptr;
1004 const uint8_t* const e_save = e_ptr;
1005 uint8_t* const d_save = d_ptr;
1006 const uint8_t* const l_save = l_ptr;
1007 DECODE_GLO_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x);
1008 continue;
1009
1010 rollback_safe_4x:
1011 t_ptr = t_save;
1012 o_ptr = o_save;
1013 e_ptr = e_save;
1014 d_ptr = d_save;
1015 l_ptr = l_save;
1016 break;
1017 }
1018 } else {
1019 while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) {
1020 DECODE_GLO_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW);
1021 }
1022 }
1023
1024 // --- FAST Loop: After threshold, no offset validation needed (4x unroll) ---
1025 if (safe) {
1026 while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) {
1027 const uint8_t* const t_save = t_ptr;
1028 const uint8_t* const o_save = o_ptr;
1029 const uint8_t* const e_save = e_ptr;
1030 uint8_t* const d_save = d_ptr;
1031 const uint8_t* const l_save = l_ptr;
1032 DECODE_GLO_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x);
1033 continue;
1034
1035 rollback_fast_4x:
1036 t_ptr = t_save;
1037 o_ptr = o_save;
1038 e_ptr = e_save;
1039 d_ptr = d_save;
1040 l_ptr = l_save;
1041 break;
1042 }
1043 } else {
1044 while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) {
1045 DECODE_GLO_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW);
1046 }
1047 }
1048
1049 // Validate vbyte reads didn't overflow
1050 if (UNLIKELY(e_ptr > e_end)) return ZXC_ERROR_CORRUPT_DATA;
1051
1052 // --- Remaining 1 sequence (Fast Path) ---
1053 while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) {
1054 // Save pointers before reading (in case we need to fall back to Safe Path)
1055 const uint8_t* t_save = t_ptr;
1056 const uint8_t* o_save = o_ptr;
1057 const uint8_t* e_save = e_ptr;
1058
1059 uint8_t token = *t_ptr++;
1060 uint64_t ll = token >> ZXC_TOKEN_LIT_BITS;
1061 uint64_t ml = token & ZXC_TOKEN_ML_MASK;
1062 uint32_t offset = ZXC_LZ_OFFSET_BIAS;
1063 if (GLO_OFF8) {
1064 offset += *o_ptr++; // 1-byte offset (biased)
1065 } else {
1066 offset += zxc_le16(o_ptr); // 2-byte offset (biased)
1067 o_ptr += sizeof(uint16_t);
1068 }
1069
1070 if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) {
1071 ll += zxc_read_varint(&e_ptr, e_end);
1072 if (UNLIKELY(l_ptr + ll > l_end)) {
1073 t_ptr = t_save;
1074 o_ptr = o_save;
1075 e_ptr = e_save;
1076 break;
1077 }
1078 }
1079 if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end);
1080 ml += ZXC_LZ_MIN_MATCH_LEN;
1081
1082 // Check bounds before wild copies - if too close to end, fall back to Safe Path
1083 if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) {
1084 t_ptr = t_save;
1085 o_ptr = o_save;
1086 e_ptr = e_save;
1087 break;
1088 }
1089
1090 // Unlike the 4x batch the copy cannot sit in the varint branch's else -
1091 // the rollback above must run first - so re-test ll instead. Exact:
1092 // the escape leaves ll >= ZXC_TOKEN_LL_MASK.
1093 if (LIKELY(ll < ZXC_TOKEN_LL_MASK)) {
1094 zxc_copy16(d_ptr, l_ptr);
1095 } else {
1096 zxc_decode_copy_literals(d_ptr, l_ptr, ll);
1097 }
1098 l_ptr += ll;
1099 d_ptr += ll;
1100
1101 if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset))
1102 return ZXC_ERROR_BAD_OFFSET;
1103
1104 // The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of
1105 // headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe.
1106 zxc_decode_copy_match_glo(d_ptr, offset, ml);
1107 d_ptr += ml;
1108 n_seq--;
1109 }
1110
1111 // --- Safe Path for Remaining Sequences ---
1112 while (n_seq > 0) {
1113 uint8_t token = *t_ptr++;
1114 uint64_t ll = token >> ZXC_TOKEN_LIT_BITS;
1115 uint64_t ml = token & ZXC_TOKEN_ML_MASK;
1116 uint32_t offset = ZXC_LZ_OFFSET_BIAS;
1117 if (GLO_OFF8) {
1118 offset += *o_ptr++; // 1-byte offset (biased)
1119 } else {
1120 offset += zxc_le16(o_ptr); // 2-byte offset (biased)
1121 o_ptr += sizeof(uint16_t);
1122 }
1123
1124 if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) ll += zxc_read_varint(&e_ptr, e_end);
1125 if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end);
1126 ml += ZXC_LZ_MIN_MATCH_LEN;
1127
1128 if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end))
1129 return ZXC_ERROR_OVERFLOW;
1130 ZXC_MEMCPY(d_ptr, l_ptr, ll);
1131 l_ptr += ll;
1132 d_ptr += ll;
1133
1134 if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET;
1135 const uint8_t* match_src = d_ptr - offset;
1136
1137 zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml);
1138 d_ptr += ml;
1139 n_seq--;
1140 }
1141
1142 // --- Trailing Literals ---
1143 // Copy remaining literals from source stream (literal exhaustion)
1144 if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA;
1145 if (UNLIKELY(d_ptr > d_end)) return ZXC_ERROR_OVERFLOW;
1146
1147 const size_t remaining_literals = (size_t)(l_end - l_ptr);
1148 if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW;
1149 ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals);
1150 d_ptr += remaining_literals;
1151
1152 return (int)(d_ptr - dst);
1153}
1154
1175static ZXC_ALWAYS_INLINE int zxc_decode_block_ghi_impl(const zxc_cctx_t* RESTRICT ctx,
1176 const uint8_t* RESTRICT src,
1177 const size_t src_size, uint8_t* RESTRICT dst,
1178 const size_t dst_capacity, const int safe,
1179 const int has_dict) {
1180 zxc_gnr_header_t gh;
1181
1182 // 0 when !has_dict (safe path) -> folds `d_floor` to `dst`.
1183 const size_t dict_size = has_dict ? ctx->dict_size : 0;
1184
1185 if (UNLIKELY(zxc_read_ghi_header(src, src_size, &gh) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER;
1186 if (UNLIKELY(gh.enc_lit != ZXC_SECTION_ENCODING_RAW || gh.enc_tok != 0))
1187 return ZXC_ERROR_CORRUPT_DATA;
1188
1189 const uint8_t* const p_data = src + ZXC_GHI_HEADER_BINARY_SIZE;
1190 const uint8_t* p_curr = p_data;
1191
1192 // --- Stream Pointers & Validation ---
1193 // GHI carries no section descriptors: literals are always RAW, the sequence
1194 // stream is four bytes per sequence, and extras run to the payload end.
1195 const size_t sz_lit = gh.n_literals;
1196 const uint64_t sz_seqs = (uint64_t)gh.n_sequences * sizeof(uint32_t);
1197
1198 const size_t payload_avail = (size_t)(src + src_size - p_data);
1199 const uint64_t consumed = (uint64_t)sz_lit + sz_seqs;
1200 if (UNLIKELY(consumed > (uint64_t)payload_avail)) return ZXC_ERROR_CORRUPT_DATA;
1201 const size_t sz_exts = payload_avail - (size_t)consumed; /* slack padding included */
1202
1203 // GHI literals are always RAW, so the slack is always load-bearing here.
1204 // No underflow: sz_lit <= consumed <= payload_avail.
1205 if (UNLIKELY(payload_avail - sz_lit < ZXC_BLOCK_LIT_SLACK)) return ZXC_ERROR_CORRUPT_DATA;
1206
1207 const uint8_t* l_ptr = p_curr;
1208 const uint8_t* l_end = l_ptr + sz_lit;
1209 p_curr += sz_lit;
1210
1211 const uint8_t* seq_ptr = p_curr;
1212 const uint8_t* extras_ptr = p_curr + (size_t)sz_seqs;
1213 const uint8_t* const extras_end = extras_ptr + sz_exts;
1214
1215 uint8_t* d_ptr = dst;
1216 const uint8_t* const d_end = dst + dst_capacity;
1217 // Lowest address a match may reach: the dictionary prefix if any, else dst.
1218 const uint8_t* const d_floor = dst - dict_size;
1219 const uint8_t* const d_end_safe =
1220 (dst_capacity > ZXC_PAD_SIZE * 4) ? d_end - (ZXC_PAD_SIZE * 4) : dst; // 128
1221 // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL +
1222 // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092
1223 const uint8_t* const d_end_fast =
1224 (dst_capacity > ZXC_DECOMPRESS_TAIL_PAD) ? d_end - ZXC_DECOMPRESS_TAIL_PAD : dst; // 2112
1225
1226 // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence,
1227 // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr.
1228 const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016
1229 const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254
1230 const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr;
1231 const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr;
1232
1233 uint32_t n_seq = gh.n_sequences;
1234
1235 // Same handover as GLO, but the GHI offset is inline in the sequence word and
1236 // always 16 bits -- enc_off is not a width here, so the span is fixed.
1237 const size_t off_span = 1U << 16;
1238 const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size;
1239 const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span;
1240
1241 // --- SAFE loop: validate offsets until d_bounds (4x unroll) ---
1242 if (safe) {
1243 // SAFE variant: save per-batch state so an OVERFLOW can rollback and
1244 // hand over to the 1x loop / Safe Path. Wild writes already committed
1245 // are deterministically overwritten when the 1x loop replays.
1246 while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) {
1247 const uint8_t* const t_save = seq_ptr;
1248 const uint8_t* const e_save = extras_ptr;
1249 uint8_t* const d_save = d_ptr;
1250 const uint8_t* const l_save = l_ptr;
1251 DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0);
1252 continue;
1253
1254 rollback_safe_4x:
1255 seq_ptr = t_save;
1256 extras_ptr = e_save;
1257 d_ptr = d_save;
1258 l_ptr = l_save;
1259 break;
1260 }
1261 } else {
1262 while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) {
1263 DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0);
1264 }
1265 }
1266
1267 // --- SAFE Loop tail: remaining sequences with offset validation (1x) ---
1268 while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) {
1269 uint32_t seq = zxc_le32(seq_ptr);
1270 seq_ptr += sizeof(uint32_t);
1271
1272 uint64_t ll = seq >> 24;
1273 if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end);
1274
1275 uint32_t m_bits = (seq >> 16) & 0xFF;
1276 uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN;
1277 if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end);
1278
1279 uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS;
1280
1281 // Strict bounds check: sequence must fit, AND wild copies must not overshoot
1282 // Check both destination (d_ptr) and source literal stream (l_ptr)
1283 if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) ||
1284 ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) {
1285 // Fallback to exact copy (slow but safe)
1286 if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW;
1287 ZXC_MEMCPY(d_ptr, l_ptr, ll);
1288 l_ptr += ll;
1289 d_ptr += ll;
1290
1291 if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW;
1292 if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET;
1293 const uint8_t* match_src = d_ptr - offset;
1294
1295 zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml);
1296 d_ptr += ml;
1297 } else {
1298 zxc_decode_copy_literals(d_ptr, l_ptr, ll);
1299 l_ptr += ll;
1300 d_ptr += ll;
1301 DECODE_MATCH_SAFE(ml, offset, zxc_decode_copy_match);
1302 }
1303 n_seq--;
1304 }
1305
1306 // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks ---
1307 if (safe) {
1308 while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) {
1309 const uint8_t* const t_save = seq_ptr;
1310 const uint8_t* const e_save = extras_ptr;
1311 uint8_t* const d_save = d_ptr;
1312 const uint8_t* const l_save = l_ptr;
1313 DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x,
1314 ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE));
1315 continue;
1316
1317 rollback_fast_4x:
1318 seq_ptr = t_save;
1319 extras_ptr = e_save;
1320 d_ptr = d_save;
1321 l_ptr = l_save;
1322 break;
1323 }
1324 } else {
1325 while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) {
1326 DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW,
1327 ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE));
1328 }
1329 }
1330
1331 // --- Remaining 1 sequence (Fast Path) ---
1332 while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) {
1333 // Save state for fallback
1334 const uint8_t* seq_save = seq_ptr;
1335 const uint8_t* ext_save = extras_ptr;
1336
1337 const uint32_t seq = zxc_le32(seq_ptr);
1338 seq_ptr += sizeof(uint32_t);
1339
1340 uint64_t ll = seq >> 24;
1341 if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) {
1342 ll += zxc_read_varint(&extras_ptr, extras_end);
1343 if (UNLIKELY(l_ptr + ll > l_end)) {
1344 seq_ptr = seq_save;
1345 extras_ptr = ext_save;
1346 break;
1347 }
1348 }
1349
1350 uint32_t m_bits = (seq >> 16) & 0xFF;
1351 uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN;
1352 if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end);
1353
1354 // Strict bounds checks (including wild copy overrun safety)
1355 if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) ||
1356 ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) {
1357 seq_ptr = seq_save;
1358 extras_ptr = ext_save;
1359 break;
1360 }
1361 uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS;
1362
1363 zxc_decode_copy_literals(d_ptr, l_ptr, ll);
1364 l_ptr += ll;
1365 d_ptr += ll;
1366
1367 if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset))
1368 return ZXC_ERROR_BAD_OFFSET;
1369
1370 // The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of
1371 // headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe.
1372 zxc_decode_copy_match(d_ptr, offset, ml);
1373 d_ptr += ml;
1374 n_seq--;
1375 }
1376
1377 // --- Safe Path for Remaining Sequences ---
1378 while (n_seq > 0) {
1379 uint32_t seq = zxc_le32(seq_ptr);
1380 seq_ptr += sizeof(uint32_t);
1381
1382 uint64_t ll = seq >> 24;
1383 if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end);
1384
1385 uint32_t m_bits = (seq >> 16) & 0xFF;
1386 uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN;
1387 if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end);
1388 uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS;
1389
1390 if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end))
1391 return ZXC_ERROR_OVERFLOW;
1392 ZXC_MEMCPY(d_ptr, l_ptr, ll);
1393 l_ptr += ll;
1394 d_ptr += ll;
1395
1396 if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET;
1397 const uint8_t* match_src = d_ptr - offset;
1398
1399 zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml);
1400 d_ptr += ml;
1401 n_seq--;
1402 }
1403
1404 // --- Trailing Literals ---
1405 // Copy remaining literals from source stream (literal exhaustion)
1406 if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA;
1407 if (UNLIKELY(d_ptr > d_end)) return ZXC_ERROR_OVERFLOW;
1408
1409 const size_t remaining_literals = (size_t)(l_end - l_ptr);
1410 if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW;
1411 ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals);
1412 d_ptr += remaining_literals;
1413
1414 return (int)(d_ptr - dst);
1415}
1416
1422static ZXC_NOINLINE int zxc_decode_block_glo_entropy(const zxc_cctx_t* RESTRICT ctx,
1423 const uint8_t* RESTRICT src,
1424 const size_t src_size, uint8_t* RESTRICT dst,
1425 const size_t dst_capacity) {
1426 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 0, 1);
1427}
1428
1429static ZXC_NOINLINE int zxc_decode_block_glo_entropy_dict(const zxc_cctx_t* RESTRICT ctx,
1430 const uint8_t* RESTRICT src,
1431 const size_t src_size,
1432 uint8_t* RESTRICT dst,
1433 const size_t dst_capacity) {
1434 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 1, 1);
1435}
1436
1437static ZXC_NOINLINE int zxc_decode_block_glo_entropy_safe(const zxc_cctx_t* RESTRICT ctx,
1438 const uint8_t* RESTRICT src,
1439 const size_t src_size,
1440 uint8_t* RESTRICT dst,
1441 const size_t dst_capacity) {
1442 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 1, 0, 1);
1443}
1444
1458static int zxc_decode_block_glo(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
1459 const size_t src_size, uint8_t* RESTRICT dst,
1460 const size_t dst_capacity) {
1461 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 0, 0);
1462}
1463
1476static int zxc_decode_block_ghi(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
1477 const size_t src_size, uint8_t* RESTRICT dst,
1478 const size_t dst_capacity) {
1479 return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0);
1480}
1481
1496static ZXC_NOINLINE int zxc_decode_block_glo_dict(const zxc_cctx_t* RESTRICT ctx,
1497 const uint8_t* RESTRICT src,
1498 const size_t src_size, uint8_t* RESTRICT dst,
1499 const size_t dst_capacity) {
1500 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 1, 0);
1501}
1502
1516static ZXC_NOINLINE int zxc_decode_block_ghi_dict(const zxc_cctx_t* RESTRICT ctx,
1517 const uint8_t* RESTRICT src,
1518 const size_t src_size, uint8_t* RESTRICT dst,
1519 const size_t dst_capacity) {
1520 return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 1);
1521}
1522
1537static ZXC_NOINLINE int zxc_decode_block_glo_safe(const zxc_cctx_t* RESTRICT ctx,
1538 const uint8_t* RESTRICT src,
1539 const size_t src_size, uint8_t* RESTRICT dst,
1540 const size_t dst_capacity) {
1541 return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 1, 0, 0);
1542}
1543
1558static ZXC_NOINLINE int zxc_decode_block_ghi_safe(const zxc_cctx_t* RESTRICT ctx,
1559 const uint8_t* RESTRICT src,
1560 const size_t src_size, uint8_t* RESTRICT dst,
1561 const size_t dst_capacity) {
1562 return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 1, 0);
1563}
1564
1565#undef DECODE_GHI_BATCH_4X
1566#undef DECODE_GHI_SEQ
1567#undef DECODE_GLO_BATCH_4X
1568#undef DECODE_GLO_SEQ
1569#undef DECODE_MATCH_FAST
1570#undef GLO_OFF8
1571#undef DECODE_MATCH_SAFE
1572
1593static ZXC_ALWAYS_INLINE int zxc_decompress_chunk_wrapper_body(
1594 const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src, const size_t src_sz,
1595 uint8_t* RESTRICT dst, const size_t dst_cap, const uint64_t block_index, const int has_dict,
1596 const int safe) {
1597 if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL;
1598
1599 const uint8_t type = src[0];
1600 const uint32_t comp_sz = zxc_le32(src + 3);
1601 const int has_checksum = ctx->checksum_enabled;
1602
1603 // A block never compresses past its own content size
1604 if (UNLIKELY((uint64_t)comp_sz > (uint64_t)ctx->chunk_size)) return ZXC_ERROR_BAD_BLOCK_SIZE;
1605
1606 // Check bounds: Header + Body + Checksum(if any)
1607 const size_t expected_sz =
1608 (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_checksum ? ZXC_BLOCK_CHECKSUM_SIZE : 0);
1609 if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL;
1610
1611 const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE;
1612
1613 int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE;
1614
1615 switch (type) {
1616 case ZXC_BLOCK_GLO:
1617 decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap)
1618 : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap)
1619 : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap);
1620 break;
1621 case ZXC_BLOCK_GHI:
1622 decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap)
1623 : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap)
1624 : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap);
1625 break;
1626 case ZXC_BLOCK_RAW:
1627 // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is)
1628 if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL;
1629 ZXC_MEMCPY(dst, data, comp_sz);
1630 decoded_sz = (int)comp_sz;
1631 break;
1632 case ZXC_BLOCK_EOF:
1633 // EOF should be handled by the dispatcher, not here
1634 return ZXC_ERROR_CORRUPT_DATA;
1635 default:
1636 return ZXC_ERROR_BAD_BLOCK_TYPE;
1637 }
1638
1639 if (has_checksum && LIKELY(decoded_sz >= 0)) {
1640 const uint32_t stored = zxc_le32(data + comp_sz);
1641 if (UNLIKELY(stored !=
1642 zxc_checksum(dst, (size_t)decoded_sz, block_index, ZXC_CHECKSUM_RAPIDHASH)))
1643 return ZXC_ERROR_BAD_CHECKSUM;
1644 }
1645
1646 return decoded_sz;
1647}
1648
1655// cppcheck-suppress unusedFunction
1656int zxc_decompress_chunk_wrapper(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
1657 const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap,
1658 const uint64_t block_index) {
1659 return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, block_index, 0, 0);
1660}
1661
1677// cppcheck-suppress unusedFunction
1678int zxc_decompress_chunk_wrapper_dict(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
1679 const size_t src_sz, uint8_t* RESTRICT dst,
1680 const size_t dst_cap, const uint64_t block_index) {
1681 return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, block_index, 1, 0);
1682}
1683
1700// cppcheck-suppress unusedFunction
1701int zxc_decompress_chunk_wrapper_safe(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
1702 const size_t src_sz, uint8_t* RESTRICT dst,
1703 const size_t dst_cap, const uint64_t block_index) {
1704 return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, block_index, 0, 1);
1705}
ZXC_ERROR_BAD_BLOCK_TYPE
@ ZXC_ERROR_BAD_BLOCK_TYPE
Unknown or unexpected block type.
Definition zxc_error.h:65
ZXC_ERROR_OVERFLOW
@ ZXC_ERROR_OVERFLOW
Buffer overflow detected during processing.
Definition zxc_error.h:58
ZXC_ERROR_BAD_OFFSET
@ ZXC_ERROR_BAD_OFFSET
Invalid match offset during decompression.
Definition zxc_error.h:57
ZXC_ERROR_CORRUPT_DATA
@ ZXC_ERROR_CORRUPT_DATA
Corrupted compressed data.
Definition zxc_error.h:56
ZXC_ERROR_BAD_HEADER
@ ZXC_ERROR_BAD_HEADER
Corrupted or invalid header (checksum mismatch).
Definition zxc_error.h:50
ZXC_ERROR_DST_TOO_SMALL
@ ZXC_ERROR_DST_TOO_SMALL
Destination buffer too small.
Definition zxc_error.h:44
ZXC_ERROR_SRC_TOO_SMALL
@ ZXC_ERROR_SRC_TOO_SMALL
Source buffer too small or truncated input.
Definition zxc_error.h:45
ZXC_ERROR_DICT_REQUIRED
@ ZXC_ERROR_DICT_REQUIRED
File requires a dictionary but none was provided.
Definition zxc_error.h:69
ZXC_ERROR_BAD_CHECKSUM
@ ZXC_ERROR_BAD_CHECKSUM
A block's decoded bytes fail its checksum, or the archive digest mismatches.
Definition zxc_error.h:51
ZXC_OK
@ ZXC_OK
Success (no error).
Definition zxc_error.h:38
ZXC_ERROR_BAD_BLOCK_SIZE
@ ZXC_ERROR_BAD_BLOCK_SIZE
Invalid block size.
Definition zxc_error.h:66
ZXC_LIT_LEN_MASK
#define ZXC_LIT_LEN_MASK
Mask to extract the run/literal length (lower 7 bits).
Definition zxc_internal.h:528
ZXC_ALWAYS_INLINE
#define ZXC_ALWAYS_INLINE
Forces a function to be inlined (fallback for non-GCC/Clang/MSVC compilers).
Definition zxc_internal.h:245
ZXC_SEQ_ML_MASK
#define ZXC_SEQ_ML_MASK
Mask to extract Match Length from a GHI sequence.
Definition zxc_internal.h:516
ZXC_CHECKSUM_RAPIDHASH
#define ZXC_CHECKSUM_RAPIDHASH
Checksum algorithm id for RapidHash (default, sole implementation).
Definition zxc_internal.h:438
ZXC_TOKEN_ML_MASK
#define ZXC_TOKEN_ML_MASK
Mask to extract Match Length from a GLO token.
Definition zxc_internal.h:499
ZXC_TOKEN_LL_MASK
#define ZXC_TOKEN_LL_MASK
Mask to extract Literal Length from a GLO token.
Definition zxc_internal.h:497
ZXC_LZ_MIN_MATCH_LEN
#define ZXC_LZ_MIN_MATCH_LEN
Minimum match length for an LZ77 match.
Definition zxc_internal.h:551
ZXC_PAD_SIZE
#define ZXC_PAD_SIZE
Safety padding appended to buffers to tolerate overruns.
Definition zxc_internal.h:331
ZXC_LZ_OFFSET_BIAS
#define ZXC_LZ_OFFSET_BIAS
Base bias added to encoded offsets (stored = actual - bias).
Definition zxc_internal.h:643
ZXC_GHI_HEADER_BINARY_SIZE
#define ZXC_GHI_HEADER_BINARY_SIZE
Binary size of a GHI block sub-header.
Definition zxc_internal.h:420
ZXC_BLOCK_HEADER_SIZE
#define ZXC_BLOCK_HEADER_SIZE
Block header size: Type(1)+Flags(1)+Reserved(1)+Checksum(1)+CompSize(4).
Definition zxc_internal.h:414
ZXC_BLOCK_LIT_SLACK
#define ZXC_BLOCK_LIT_SLACK
Readable bytes a GLO/GHI payload guarantees after its literal section.
Definition zxc_internal.h:345
ZXC_GLO_MAX_INLINE_ML
#define ZXC_GLO_MAX_INLINE_ML
Longest match a GLO sequence carries without a varint extension.
Definition zxc_internal.h:639
ZXC_NOINLINE
#define ZXC_NOINLINE
Prevents inlining (best-effort no-op fallback for unknown compilers).
Definition zxc_internal.h:250
ZXC_ALIGN
#define ZXC_ALIGN(x)
Specifies memory alignment using C11 _Alignas.
Definition zxc_internal.h:257
zxc_read_varint
static uint32_t zxc_read_varint(const uint8_t **ptr, const uint8_t *end)
Reads a Prefix Varint encoded integer.
Definition zxc_internal.h:586
ZXC_LIT_RLE_FLAG
#define ZXC_LIT_RLE_FLAG
Flag bit indicating an RLE run in the literal stream (0x80).
Definition zxc_internal.h:526
ZXC_BLOCK_CHECKSUM_SIZE
#define ZXC_BLOCK_CHECKSUM_SIZE
Size of the per-block checksum field in bytes.
Definition zxc_internal.h:416
ZXC_CACHE_LINE_SIZE
#define ZXC_CACHE_LINE_SIZE
Assumed CPU cache line size for alignment.
Definition zxc_internal.h:359
ZXC_TOKEN_LIT_BITS
#define ZXC_TOKEN_LIT_BITS
Bits for Literal Length in a GLO token.
Definition zxc_internal.h:493
ZXC_DECOMPRESS_TAIL_PAD
#define ZXC_DECOMPRESS_TAIL_PAD
Tail padding required on the decompression destination buffer.
Definition zxc_internal.h:357
ZXC_SEQ_LL_MASK
#define ZXC_SEQ_LL_MASK
Mask to extract Literal Length from a GHI sequence.
Definition zxc_internal.h:514
zxc_cctx_t
Compression / decompression context.
Definition zxc_internal.h:1884
zxc_gnr_header_t
Header specific to General (LZ-based) compression blocks.
Definition zxc_internal.h:1254
zxc_gnr_header_t::n_sequences
uint32_t n_sequences
The total count of LZ sequences in the block.
Definition zxc_internal.h:1255
zxc_gnr_header_t::enc_off
uint8_t enc_off
GLO only: width of the offset stream (1 = 1-byte, 0 = 2-byte).
Definition zxc_internal.h:1260
zxc_gnr_header_t::n_literals
uint32_t n_literals
The total count of literal bytes.
Definition zxc_internal.h:1256
zxc_gnr_header_t::enc_tok
uint8_t enc_tok
GLO only: encoding of the token section, whose bytes each pack a literal length and a match length ni...
Definition zxc_internal.h:1258
zxc_gnr_header_t::enc_lit
uint8_t enc_lit
Encoding method used for the literal stream.
Definition zxc_internal.h:1257
zxc_cctx_alloc_entropy_scratch
int zxc_cctx_alloc_entropy_scratch(zxc_cctx_t *ctx)
Lazily allocates the deferred decode-side entropy scratch.
Definition zxc_common.c:373
zxc_read_ghi_header
int zxc_read_ghi_header(const uint8_t *src, const size_t len, zxc_gnr_header_t *gh)
Parses a GHI block header from src.
Definition zxc_common.c:1008
zxc_read_glo_header_and_desc
int zxc_read_glo_header_and_desc(const uint8_t *src, const size_t len, zxc_gnr_header_t *gh, uint32_t *lit_comp, uint32_t *tok_comp)
Parses a GLO block header and its section descriptors from src.
Definition zxc_common.c:965
zxc_decode_copy_match_glo
static void zxc_decode_copy_match_glo(uint8_t *d_ptr, const uint32_t off, const uint64_t ml)
GLO match copy: bounded form when the ml nibble did not saturate.
Definition zxc_decompress.c:478
zxc_wild_copy32
static void zxc_wild_copy32(uint8_t *dst, const uint8_t *src, const size_t len)
Copies len bytes in 32-byte wild chunks (the zxc_copy32 width).
Definition zxc_decompress.c:372
zxc_decode_copy_match_short
static void zxc_decode_copy_match_short(uint8_t *d_ptr, const uint32_t off, const uint64_t ml)
Match copy for a length that cannot exceed 32 bytes.
Definition zxc_decompress.c:459
zxc_ensure_entropy_scratch
static int zxc_ensure_entropy_scratch(const zxc_cctx_t *ctx)
Ensures the entropy decode scratch (tok_buffer + pivco_scratch) is available before decoding an entro...
Definition zxc_decompress.c:704
zxc_decode_copy_match_exact
static void zxc_decode_copy_match_exact(uint8_t *d_ptr, const uint8_t *match_src, const size_t off, const size_t ml)
Exact-size match copy for the tail loops (no overshoot headroom).
Definition zxc_decompress.c:503
zxc_decode_copy_overlap_short
static void zxc_decode_copy_overlap_short(uint8_t *dst, const uint32_t off, const uint64_t ml)
Overlap copy for a run of period off (2..31) bounded by ml <= 32.
Definition zxc_decompress.c:238
zxc_decode_copy_match
static void zxc_decode_copy_match(uint8_t *d_ptr, const uint32_t off, const uint64_t ml)
Copies an ml-byte LZ match from d_ptr-off to d_ptr, handling overlap.
Definition zxc_decompress.c:432
zxc_decode_block_ghi_impl
static int zxc_decode_block_ghi_impl(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity, const int safe, const int has_dict)
Unified GHI (General High) block decoder body, shared by the fast, safe and dictionary variants.
Definition zxc_decompress.c:1173
DECODE_GLO_BATCH_4X
#define DECODE_GLO_BATCH_4X(DECODE, ON_FAIL)
One full GLO 4x batch: token word, four offsets (1- or 2-byte form), four DECODE_GLO_SEQ emissions,...
Definition zxc_decompress.c:608
zxc_decode_block_glo_entropy
static int zxc_decode_block_glo_entropy(const zxc_cctx_t *ctx, const uint8_t *src, size_t src_size, uint8_t *dst, size_t dst_capacity)
Cold specializations for entropy-coded GLO tokens (level 7).
Definition zxc_decompress.c:1420
zxc_decode_copy_overlap_run32
static void zxc_decode_copy_overlap_run32(uint8_t *dst, const uint32_t off, const uint64_t ml)
Copies an ml-byte run of period off (2..31) with 32-byte stores.
Definition zxc_decompress.c:143
zxc_decompress_chunk_wrapper_dict
int zxc_decompress_chunk_wrapper_dict(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_sz, uint8_t *dst, const size_t dst_cap, const uint64_t block_index)
Public dictionary chunk decoder.
Definition zxc_decompress.c:1676
zxc_decode_block_glo_safe
static int zxc_decode_block_glo_safe(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a GLO block with the strict-tail safe loop (no wild copies).
Definition zxc_decompress.c:1535
zxc_decode_copy_literals
static void zxc_decode_copy_literals(uint8_t *dst, const uint8_t *src, const uint64_t ll)
Copies ll literal bytes from src to dst using 32-byte wild copies.
Definition zxc_decompress.c:403
zxc_decode_block_ghi_safe
static int zxc_decode_block_ghi_safe(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a GHI block with the strict-tail safe loop (no wild copies).
Definition zxc_decompress.c:1556
zxc_decode_block_ghi
static int zxc_decode_block_ghi(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a no-dict GHI block (plain, inlinable path).
Definition zxc_decompress.c:1474
zxc_decode_block_ghi_dict
static int zxc_decode_block_ghi_dict(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a GHI block against a dictionary prefix (cold path).
Definition zxc_decompress.c:1514
zxc_decompress_chunk_wrapper
int zxc_decompress_chunk_wrapper(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_sz, uint8_t *dst, const size_t dst_cap, const uint64_t block_index)
Public no-dict chunk decoder (decompression hot path).
Definition zxc_decompress.c:1654
zxc_overlap_strides32
static const uint8_t zxc_overlap_strides32[32]
Store stride for the 32-byte pattern: (32 / off) * off.
Definition zxc_decompress.c:117
DECODE_GHI_BATCH_4X
#define DECODE_GHI_BATCH_4X(DECODE, ON_FAIL, PREFETCH)
One full GHI 4x batch.
Definition zxc_decompress.c:675
zxc_decode_block_glo_dict
static int zxc_decode_block_glo_dict(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a GLO block against a dictionary prefix (cold path).
Definition zxc_decompress.c:1494
zxc_decode_block_glo_impl
static int zxc_decode_block_glo_impl(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity, const int safe, const int has_dict, const int tok_entropy)
Unified GLO (General Low) block decoder body, shared by the fast, safe, dictionary and entropy-token ...
Definition zxc_decompress.c:787
zxc_decode_fill_run
static void zxc_decode_fill_run(uint8_t *dst, const uint8_t byte, const uint64_t ml)
Fills an ml-byte single-byte run (LZ offset == 1) with wild stores.
Definition zxc_decompress.c:309
zxc_decompress_chunk_wrapper_safe
int zxc_decompress_chunk_wrapper_safe(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_sz, uint8_t *dst, const size_t dst_cap, const uint64_t block_index)
Public strict-tail safe chunk decoder (dst_cap == exact decoded size).
Definition zxc_decompress.c:1699
GLO_OFF8
#define GLO_OFF8
True when the GLO offset stream stores 1-byte offsets, 2-byte otherwise.
Definition zxc_decompress.c:553
zxc_decompress_chunk_wrapper_body
static int zxc_decompress_chunk_wrapper_body(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_sz, uint8_t *dst, const size_t dst_cap, const uint64_t block_index, const int has_dict, const int safe)
Shared chunk-decode body: validates the header, decodes, then verifies the optional checksum.
Definition zxc_decompress.c:1591
zxc_decode_block_glo
static int zxc_decode_block_glo(const zxc_cctx_t *ctx, const uint8_t *src, const size_t src_size, uint8_t *dst, const size_t dst_capacity)
Decode a no-dict GLO block (plain, inlinable path).
Definition zxc_decompress.c:1456
zxc_huf_decode_section_dict
int zxc_huf_decode_section_dict(const uint8_t *payload, const size_t payload_size, uint8_t *dst, const size_t n, const zxc_pivco_tree_t *tree, uint8_t *scratch)
Decode a PivCo dict section against a prebuilt dict tree.
Definition zxc_dispatch.c:521
zxc_huf_decode_section
int zxc_huf_decode_section(const uint8_t *payload, const size_t payload_size, uint8_t *dst, const size_t n, uint8_t *scratch)
Decode a PivCo literal section into dst (exactly n bytes).
Definition zxc_dispatch.c:507
zxc_internal.h
Internal definitions, constants, SIMD helpers, and utility functions.
ZXC_PIVCO_SCRATCH_PAD
#define ZXC_PIVCO_SCRATCH_PAD
Extra scratch slack required past n by the PivCo decoder.
Definition zxc_internal.h:1821
zxc_le32
static uint32_t zxc_le32(const void *p)
Reads a little-endian 32-bit value from a possibly unaligned address.
Definition zxc_internal.h:1298
zxc_copy32
static void zxc_copy32(void *dst, const void *src)
Copies 32 bytes from source to destination using SIMD when available.
Definition zxc_internal.h:1500
zxc_le16
static uint16_t zxc_le16(const void *p)
Reads a little-endian 16-bit value from a possibly unaligned address.
Definition zxc_internal.h:1279
zxc_copy16
static void zxc_copy16(void *dst, const void *src)
Copies exactly 16 bytes as one vector move where the ISA has one.
Definition zxc_internal.h:1481
zxc_checksum
static uint32_t zxc_checksum(const void *input, const size_t len, const uint64_t seed, const uint8_t hash_method)
Calculates a 32-bit hash for a given input buffer.
Definition zxc_internal.h:1613