zxc_internal.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause */
/*
 * ZXC - High-performance lossless compression
 *
 * Copyright (c) Bertrand Lebonnois and contributors.
 */
 
#ifndef ZXC_INTERNAL_H
#define ZXC_INTERNAL_H
 
#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112L)
#error "libzxc requires C11 or later (MSVC: /std:c11)"
#endif
 
#include "zxc_deps.h" /* libc deps: <assert.h>, <limits.h>, <stdint.h>, <stdlib.h>, <string.h>,
                        and the ZXC_MALLOC / ZXC_ALIGNED_MALLOC macros.
                        Vendor this file to retarget non-libc environments. */
 
#include "../../include/zxc_buffer.h"
#include "../../include/zxc_constants.h"
#include "../../include/zxc_error.h"
#include "../../include/zxc_seekable.h"
#include "rapidhash.h"
 
#ifdef __cplusplus
extern "C" {
#endif
 
#if !defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__)
#include <stdatomic.h>
#define ZXC_ATOMIC _Atomic
#define ZXC_USE_C11_ATOMICS 1
#else
#define ZXC_ATOMIC volatile
#define ZXC_USE_C11_ATOMICS 0
#endif
 
#ifndef ZXC_DISABLE_SIMD
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
#include <immintrin.h>
#include <nmmintrin.h>
#if defined(__AVX512F__) && defined(__AVX512BW__)
#ifndef ZXC_USE_AVX512
#define ZXC_USE_AVX512
#endif
#endif
#if defined(__AVX2__)
#ifndef ZXC_USE_AVX2
#define ZXC_USE_AVX2
#endif
#endif
#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
#ifndef ZXC_USE_SSE2
#define ZXC_USE_SSE2
#endif
#endif
#elif (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64) || \
       defined(ZXC_USE_NEON32) || defined(ZXC_USE_NEON64))
#if !defined(_MSC_VER)
#include <arm_acle.h>
#endif
#include <arm_neon.h>
#if defined(__aarch64__) || defined(_M_ARM64)
#ifndef ZXC_USE_NEON64
#define ZXC_USE_NEON64
#endif
#else
#ifndef ZXC_USE_NEON32
#define ZXC_USE_NEON32
#endif
#endif
#endif
#endif    /* ZXC_DISABLE_SIMD */
 
#if defined(__GNUC__) || defined(__clang__)
#define LIKELY(x) (__builtin_expect(!!(x), 1))
 
#define UNLIKELY(x) (__builtin_expect(!!(x), 0))
 
#define RESTRICT __restrict__
 
#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3)
 
#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n)
 
#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n)
 
#define ZXC_ALIGN(x) __attribute__((aligned(x)))
 
#define ZXC_ALWAYS_INLINE inline __attribute__((always_inline))
 
#define ZXC_NOINLINE __attribute__((__noinline__))
 
#define ZXC_COLD __attribute__((cold))
 
#elif defined(_MSC_VER)
#include <intrin.h>
#if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64)
#include <xmmintrin.h>
#define ZXC_PREFETCH_READ(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
#else
#define ZXC_PREFETCH_READ(ptr) __prefetch((const void*)(ptr))
#endif
#define LIKELY(x) (x)
#define UNLIKELY(x) (x)
#define RESTRICT __restrict
#pragma intrinsic(memcpy, memset)
#define ZXC_MEMCPY(dst, src, n) memcpy(dst, src, n)
#define ZXC_MEMSET(dst, val, n) memset(dst, val, n)
 
#define ZXC_ALIGN(x) __declspec(align(x))
 
#define ZXC_ALWAYS_INLINE __forceinline
 
#define ZXC_NOINLINE __declspec(noinline)
#define ZXC_COLD
#pragma intrinsic(_BitScanReverse)
#else
#define LIKELY(x) (x)
#define UNLIKELY(x) (x)
#define RESTRICT
#define ZXC_PREFETCH_READ(ptr)
#define ZXC_MEMCPY(dst, src, n) memcpy(dst, src, n)
#define ZXC_MEMSET(dst, val, n) memset(dst, val, n)
 
#define ZXC_ALWAYS_INLINE inline
 
#define ZXC_NOINLINE
#define ZXC_COLD
 
#define ZXC_ALIGN(x) _Alignas(x)
#endif
 
// Heap allocator and cache-line-aligned allocator macros are now defined
// in @c zxc_deps.h (included at the top of this header), so non-libc
// targets can override them by vendoring that single file.
 
#ifndef ZXC_LITTLE_ENDIAN
#if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || \
    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define ZXC_LITTLE_ENDIAN
#elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#define ZXC_BIG_ENDIAN
#else
#warning "Endianness not detected, defaulting to little-endian"
#define ZXC_LITTLE_ENDIAN
#endif
#endif
 
#ifdef ZXC_BIG_ENDIAN
#if defined(__GNUC__) || defined(__clang__)
#define ZXC_BSWAP16(x) __builtin_bswap16(x)
#define ZXC_BSWAP32(x) __builtin_bswap32(x)
#define ZXC_BSWAP64(x) __builtin_bswap64(x)
#elif defined(_MSC_VER)
#define ZXC_BSWAP16(x) _byteswap_ushort(x)
#define ZXC_BSWAP32(x) _byteswap_ulong(x)
#define ZXC_BSWAP64(x) _byteswap_uint64(x)
#else
#define ZXC_BSWAP16(x) ((uint16_t)(((x) >> 8) | ((x) << 8)))
#define ZXC_BSWAP32(x) \
    ((uint32_t)(((x) >> 24) | (((x) >> 8) & 0xFF00) | (((x) << 8) & 0xFF0000) | ((x) << 24)))
#define ZXC_BSWAP64(x) \
    ((uint64_t)(((uint64_t)ZXC_BSWAP32((uint32_t)(x)) << 32) | ZXC_BSWAP32((uint32_t)((x) >> 32))))
#endif
#endif
 
static inline int zxc_magic_prefix_ok(const uint8_t* p, const size_t n) {
    static const uint8_t magic[4] = {(uint8_t)ZXC_MAGIC_WORD, (uint8_t)(ZXC_MAGIC_WORD >> 8),
                                     (uint8_t)(ZXC_MAGIC_WORD >> 16),
                                     (uint8_t)(ZXC_MAGIC_WORD >> 24)};
    for (size_t i = 0; i < n && i < sizeof(magic); i++)
        if (p[i] != magic[i]) return 0;
    return 1;
}
#define ZXC_FILE_FORMAT_VERSION 9
 
#define ZXC_PAD_SIZE 32
#define ZXC_BLOCK_LIT_SLACK 32
#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66)
#define ZXC_CACHE_LINE_SIZE 64
#define ZXC_ALIGNMENT_MASK ((size_t)ZXC_CACHE_LINE_SIZE - 1)
#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK)
static_assert((ZXC_CACHE_LINE_SIZE & ZXC_ALIGNMENT_MASK) == 0,
              "ZXC_CACHE_LINE_SIZE must be a power of two");
 
#define ZXC_BITMAP_WORDS(n_bits) (((n_bits) + 63) / 64)
 
#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U
#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U
#define ZXC_FILE_FLAG_HAS_SEEK_TABLE 0x20U
#define ZXC_FILE_CHECKSUM_ALGO_MASK 0x0FU
static_assert(((ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_FILE_FLAG_HAS_DICTIONARY |
                ZXC_FILE_FLAG_HAS_SEEK_TABLE) &
               ZXC_FILE_CHECKSUM_ALGO_MASK) == 0,
              "a file flag overlaps the checksum algorithm id");
 
#define ZXC_DICT_VERSION 2
#define ZXC_DICT_KGRAM_LEN ZXC_LZ_MIN_MATCH_LEN
#define ZXC_DICT_HASH_BITS 16
#define ZXC_DICT_MAX_SEGMENTS (1U << 16)
#define ZXC_DICT_SAMPLE_TARGET (1U << 19)
#define ZXC_DICT_HASH_SIZE (1U << ZXC_DICT_HASH_BITS)
#define ZXC_DICT_HUF_TRAIN_BLOCK 4096U
#define ZXC_DICT_HUF_SAMPLE_BUDGET (8U << 20)
 
#define ZXC_BLOCK_HEADER_SIZE 8
#define ZXC_BLOCK_CHECKSUM_SIZE 4
#define ZXC_GLO_HEADER_BINARY_SIZE 12
#define ZXC_GHI_HEADER_BINARY_SIZE 12
#define ZXC_GLO_DESC_MAX_SIZE (2 * sizeof(uint32_t))
 
#define ZXC_BLOCK_FORMAT_OVERHEAD 68
static_assert(ZXC_BLOCK_FORMAT_OVERHEAD >=
                  ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_DESC_MAX_SIZE + ZXC_BLOCK_LIT_SLACK,
              "ZXC_BLOCK_FORMAT_OVERHEAD must cover sub-header, descriptors and slack");
 
#define ZXC_CHECKSUM_RAPIDHASH 0
 
#define ZXC_SEEK_GROUP 64U
#define ZXC_SEEK_ANCHOR_SIZE 8U
#define ZXC_SEEK_SIZE_ENTRY 4U
#define ZXC_SEEK_GROUP_BYTES (ZXC_SEEK_ANCHOR_SIZE + ZXC_SEEK_GROUP * ZXC_SEEK_SIZE_ENTRY)
 
static ZXC_ALWAYS_INLINE uint64_t zxc_seek_group_count(const uint64_t nblocks) {
    return (nblocks + ZXC_SEEK_GROUP - 1) / ZXC_SEEK_GROUP;
}
 
static ZXC_ALWAYS_INLINE uint32_t zxc_seek_group_len(const uint64_t nblocks, const uint64_t g) {
    const uint64_t left = nblocks - g * ZXC_SEEK_GROUP;
    return left < ZXC_SEEK_GROUP ? (uint32_t)left : ZXC_SEEK_GROUP;
}
 
static ZXC_ALWAYS_INLINE uint64_t zxc_seek_table_bytes(const uint64_t nblocks) {
    return zxc_seek_group_count(nblocks) * ZXC_SEEK_ANCHOR_SIZE + nblocks * ZXC_SEEK_SIZE_ENTRY;
}
 
static ZXC_ALWAYS_INLINE uint32_t zxc_seek_size_field(const uint64_t table_bytes) {
#if defined(_MSC_VER) && !defined(__clang__) && (defined(_M_ARM64) || defined(_M_ARM64EC))
    // MSVC 19.51 ARM64 /O2 sees the table size as 4u and compiles this fold into
    // `ror w, #30` on the low half of u, dropping the high word: tables of 4 GiB
    // and more get a wrong field. A volatile copy hides that shape. Once per table.
    const volatile uint64_t t = table_bytes;
    return (uint32_t)(t ^ (t >> 32));
#else
    return (uint32_t)(table_bytes ^ (table_bytes >> 32));
#endif
}
 
 
#define ZXC_TOKEN_LIT_BITS 4
#define ZXC_TOKEN_ML_BITS 4
#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1)
#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1)
static_assert(ZXC_TOKEN_LIT_BITS == 4 && ZXC_TOKEN_ML_BITS == 4,
              "the GLO decoder reads its tokens as two nibbles");
#define ZXC_SEQ_LL_BITS 8
#define ZXC_SEQ_ML_BITS 8
#define ZXC_SEQ_OFF_BITS 16
#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1)
#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1)
#define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1)
static_assert(ZXC_SEQ_LL_BITS + ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS == 32,
              "a GHI sequence is one 32-bit word");
#define ZXC_LIT_RLE_FLAG 0x80U
#define ZXC_LIT_LEN_MASK (ZXC_LIT_RLE_FLAG - 1)
#define ZXC_LZ_HASH_BITS 15
#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS)
#define ZXC_LZ_WINDOW_SIZE (1U << 16)
#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U)
#define ZXC_LZ_MIN_MATCH_LEN 5
#define ZXC_MAX_VARINT_VALUE ((uint32_t)(ZXC_BLOCK_SIZE_MAX - 1U))
static_assert(ZXC_MAX_VARINT_VALUE < (1U << 21), "varint values must fit the 3-byte form");
static_assert(ZXC_BLOCK_SIZE_MIN <= ZXC_BLOCK_SIZE_DEFAULT &&
                  ZXC_BLOCK_SIZE_DEFAULT <= ZXC_BLOCK_SIZE_MAX &&
                  (ZXC_BLOCK_SIZE_DEFAULT & (ZXC_BLOCK_SIZE_DEFAULT - 1)) == 0,
              "ZXC_BLOCK_SIZE_DEFAULT must be a power of two within the block size range");
static_assert((ZXC_LZ_WINDOW_SIZE & ZXC_LZ_WINDOW_MASK) == 0,
              "ZXC_LZ_WINDOW_SIZE must be a power of two (ring mask)");
static_assert(ZXC_LZ_WINDOW_SIZE <= (1U << ZXC_SEQ_OFF_BITS),
              "chain_table holds window positions in 16 bits");
 
static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) {
    const uint8_t* p = *ptr;
    if (UNLIKELY(p >= end)) return 0;
 
    const uint32_t b0 = p[0];
 
    // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7)
    if (LIKELY(b0 < 0x80)) {
        *ptr = p + 1;
        return b0;
    }
 
    // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14)
    if (LIKELY(b0 < 0xC0)) {
        if (UNLIKELY(p + 1 >= end)) {
            *ptr = end;
            return 0;
        }
        *ptr = p + 2;
        return (b0 & 0x3F) | ((uint32_t)p[1] << 6);
    }
 
    // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest
    // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always
    // strictly below block_size_max = 2^21.
    if (LIKELY(b0 < 0xE0)) {
        if (UNLIKELY(p + 2 >= end)) {
            *ptr = end;
            return 0;
        }
        *ptr = p + 3;
        return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13);
    }
 
    // extra encoding: out-of-spec for the current format, reject.
    *ptr = end;
    return 0;
}
 
#define ZXC_GLO_MAX_INLINE_OUT_PER_SEQ ((ZXC_TOKEN_LL_MASK - 1U) + ZXC_GLO_MAX_INLINE_ML) /* 33 */
#define ZXC_GLO_INLINE_ML_CODE (ZXC_TOKEN_ML_MASK - 1U) /* 14 */
#define ZXC_GLO_MAX_INLINE_ML (ZXC_GLO_INLINE_ML_CODE + ZXC_LZ_MIN_MATCH_LEN) /* 19 */
#define ZXC_GHI_MAX_INLINE_OUT_PER_SEQ \
    ((ZXC_SEQ_LL_MASK - 1U) + (ZXC_SEQ_ML_MASK - 1U) + ZXC_LZ_MIN_MATCH_LEN) /* 513 */
#define ZXC_LZ_OFFSET_BIAS 1
#define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1)
static_assert(ZXC_GLO_MAX_INLINE_ML <= ZXC_PAD_SIZE, "an inline GLO match is one 32-byte store");
static_assert(ZXC_LZ_WINDOW_SIZE - ZXC_LZ_OFFSET_BIAS <= ZXC_SEQ_OFF_MASK,
              "a biased offset must fit the 16-bit wire field");
static_assert(4 * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE <= ZXC_DECOMPRESS_TAIL_PAD,
              "ZXC_DECOMPRESS_TAIL_PAD must hold a 4x GHI batch and its wild-copy overshoot");
static_assert(ZXC_DICT_SIZE_MAX <= ZXC_LZ_MAX_DIST, "a dictionary must stay within offset reach");
 
#define ZXC_LZ_MINDIST 32
 
#define ZXC_LZ_MINDIST_PROBE_PER_KB 1024
#define ZXC_LZ_MINDIST_PROBE_MIN 16
#define ZXC_LZ_MINDIST_PROBE_MAX 64
 
#define ZXC_LZ_MINDIST_MAX_SHORT_PCT 20
#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t))
#define ZXC_OPT_MATCH_COST_BASE ((uint32_t)(3U * CHAR_BIT))
#define ZXC_OPT_LONG_MATCH_SKIP ((size_t)256)
static_assert(ZXC_OPT_LONG_MATCH_SKIP <= ZXC_LZ_MIN_MATCH_LEN + ZXC_TOKEN_ML_MASK + 128 * 128,
              "covered positions relax only the first two match cost classes");
#define ZXC_OPT_LIT_SAMPLE_MIN 1024
 
#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL
#define ZXC_HASH_XORSHIFT64 0x2545F4914F6CDD1DULL
#define ZXC_HASH_MULT32 0x2D35182DU
#define ZXC_HASH_MULT64 0xD2D84A61D2D84A61ULL
#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11
#define ZXC_HUF_MAX_CODE_LEN_DENSITY 8
#define ZXC_HUF_NUM_SYMBOLS 256
static_assert(ZXC_HUF_MAX_CODE_LEN_DENSITY <= ZXC_HUF_MAX_CODE_LEN_ULTRA,
              "the decoder accepts code lengths up to ZXC_HUF_MAX_CODE_LEN_ULTRA only");
static_assert(ZXC_HUF_MAX_CODE_LEN_ULTRA <= 0x0F && ZXC_HUF_TABLE_SIZE * 2 == ZXC_HUF_NUM_SYMBOLS,
              "code lengths are packed one nibble per symbol");
 
#define ZXC_PIVCO_MAX_NODES (2 * ZXC_HUF_NUM_SYMBOLS - 1)
#define ZXC_PIVCO_NODE_SLOTS (ZXC_PIVCO_MAX_NODES + 1)
#define ZXC_PIVCO_UNPACK_FLAT_SIMD_MAX 6
 
#define ZXC_PIVCO_COVERED 0xFFU
 
typedef struct {
    uint16_t node_base[ZXC_HUF_MAX_CODE_LEN_ULTRA + 2]; 
    uint16_t leaf_base[ZXC_HUF_MAX_CODE_LEN_ULTRA + 2]; 
    int16_t base[ZXC_HUF_MAX_CODE_LEN_ULTRA + 1];       
    int max_depth;
    uint8_t syms[ZXC_HUF_NUM_SYMBOLS];    
    uint8_t flat_d[ZXC_PIVCO_NODE_SLOTS]; 
} zxc_pivco_tree_t;
 
typedef struct {
    zxc_pivco_tree_t tree;                 
    uint32_t codes[ZXC_HUF_NUM_SYMBOLS];   
    uint8_t code_len[ZXC_HUF_NUM_SYMBOLS]; 
} zxc_dict_huf_state_t;
#define ZXC_RLE_MARGIN_SHIFT 5
#define ZXC_HUF_MARGIN_SHIFT 5
 
#define ZXC_HUF_NUDGE_LAMBDA_Q8 26
#define ZXC_HUF_NUDGE_BITS_PERMIL 1015
#define ZXC_HUF_NUDGE_MERGE_Q8 230
#define ZXC_HUF_NUDGE_DEEP_FLAT_PENALTY 24
#define ZXC_HUF_NUDGE_LEVEL_COST 64
#define ZXC_SS_TAX(n, prem_q8) (((size_t)(n) * (size_t)(prem_q8)) >> 8)
static inline int zxc_ss_prem_rle_q8(const int level) {
    return (level >= ZXC_LEVEL_DENSITY) ? 1 : (256 >> ZXC_RLE_MARGIN_SHIFT);
}
static inline int zxc_ss_prem_huf_q8(const int level) {
    return (level >= ZXC_LEVEL_DENSITY) ? 4 : (256 >> ZXC_HUF_MARGIN_SHIFT);
}
#define ZXC_HUF_MIN_LITERALS 139
static inline int zxc_level_clamp(const int level) {
    return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level;
}
 
#define ZXC_OPTS_DICT_SIZE(o) (((o) && (o)->dict) ? (o)->dict_size : (size_t)0)
#define ZXC_OPTS_DICT_HUF(o) \
    (((o) && (o)->dict && (o)->dict_size > 0) ? (const uint8_t*)(o)->dict_huf : NULL)
#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt))
#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt))
#define ZXC_GLO_SPLIT_CTX_BITS 8
#define ZXC_GLO_SPLIT_MIN_MISPREDICT_PCT 80
typedef uint32_t zxc_glo_split_hist_t[4][2][1U << ZXC_GLO_SPLIT_CTX_BITS];
static_assert(ZXC_GLO_SPLIT_CTX_BITS <= CHAR_BIT, "a context is recorded in one byte of buf_split");
static inline int zxc_huf_enc_max_code_len(const int level) {
    return (level >= ZXC_LEVEL_ULTRA) ? ZXC_HUF_MAX_CODE_LEN_ULTRA : ZXC_HUF_MAX_CODE_LEN_DENSITY;
}
 
typedef struct {
    uint32_t weight; 
    int16_t left;    
    int16_t right;   
    int16_t sym;     
} zxc_huf_pm_item_t;
 
typedef struct {
    int8_t lvl;  
    int16_t idx; 
} zxc_huf_pm_frame_t;
 
typedef struct {
    uint32_t w;  
    int16_t sym; 
} zxc_huf_pm_leaf_t;
 
#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS)
 
#define ZXC_HUF_BUILD_SCRATCH_SIZE                                         \
    ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \
         sizeof(zxc_huf_pm_item_t) +                                       \
     8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U +          \
     (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \
         sizeof(zxc_huf_pm_frame_t))
 
#define ZXC_HUF_NUDGE_DP_M 64
 
typedef struct {
    uint64_t pf[ZXC_HUF_NUM_SYMBOLS + 1];      
    uint64_t pf_rank[ZXC_HUF_NUM_SYMBOLS + 1]; 
    uint64_t pfg[ZXC_HUF_NUDGE_DP_M + 1];      
    uint64_t jcur[(ZXC_HUF_NUDGE_DP_M + 1) * (ZXC_HUF_NUDGE_DP_M + 1)]; 
    uint64_t jnxt[(ZXC_HUF_NUDGE_DP_M + 1) * (ZXC_HUF_NUDGE_DP_M + 1)]; 
    uint8_t arrive[(ZXC_HUF_MAX_CODE_LEN_ULTRA + 1) * (ZXC_HUF_NUDGE_DP_M + 1) *
                   (ZXC_HUF_NUDGE_DP_M + 1)];
    zxc_huf_pm_leaf_t leaves[ZXC_HUF_NUM_SYMBOLS]; 
    int16_t sym_order[ZXC_HUF_NUM_SYMBOLS];        
    uint8_t cand[4][ZXC_HUF_NUM_SYMBOLS];          
} zxc_huf_nudge_ws_t;
 
#define ZXC_HUF_NUDGE_SCRATCH_OFF ZXC_ALIGN_CL(ZXC_HUF_BUILD_SCRATCH_SIZE)
#define ZXC_HUF_NUDGE_SCRATCH_SIZE (ZXC_HUF_NUDGE_SCRATCH_OFF + sizeof(zxc_huf_nudge_ws_t))
 
static ZXC_ALWAYS_INLINE void zxc_opt_dp_sizes(const size_t chunk_size, size_t* RESTRICT sz_dp,
                                               size_t* RESTRICT sz_pl, size_t* RESTRICT sz_po,
                                               size_t* RESTRICT sz_bm) {
    *sz_dp = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint32_t));
    *sz_pl = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t));
    *sz_po = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t));
    *sz_bm = ZXC_ALIGN_CL(ZXC_BITMAP_WORDS(chunk_size + 1) * sizeof(uint64_t));
}
 
static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) {
#ifdef _MSC_VER
    unsigned long index;
    return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0);
#else
    return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v));
#endif
}
 
static ZXC_ALWAYS_INLINE size_t zxc_block_size_ceil(const size_t v) {
    uint64_t x = (uint64_t)v - 1;
    x |= x >> 1;
    x |= x >> 2;
    x |= x >> 4;
    x |= x >> 8;
    x |= x >> 16;
    x |= x >> 32;
    x++;
    const size_t bs = (size_t)x;
    return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs;
}
 
static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) {
    return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0;
}
 
typedef struct {
    int search_depth;
 
    int sufficient_len;
 
    int use_lazy;
 
    int lazy_attempts;
 
    int lazy_len_threshold;
 
    uint32_t step_base;
 
    uint32_t step_shift;
 
    uint32_t min_offset;
 
    uint8_t split_max;
} zxc_lz77_params_t;
 
static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) {
    // The distance floor stops at level 5: the slow levels keep every distance.
    // Match splitting at levels 3-5 only: 1-2 are GHI, 6-7 keep every escape.
    // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base,
    // step_shift, min_offset, split_max
    static const zxc_lz77_params_t table[7] = {
        {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST, 0},        // fallback
        {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST, 0},        // level 1
        {3, 18, 0, 0, 0, 3, 6, ZXC_LZ_MINDIST, 0},        // level 2
        {3, 16, 1, 4, 128, 1, 4, ZXC_LZ_MINDIST, 33},     // level 3
        {3, 18, 1, 4, 128, 1, 5, ZXC_LZ_MINDIST, 24},     // level 4
        {64, 256, 1, 16, 128, 1, 8, ZXC_LZ_MINDIST, 23},  // level 5
        {64, 256, 0, 0, 0, 1, 8, 1, 0}                    // level 6
    };
    return (level >= ZXC_LEVEL_ULTRA)
               ? (zxc_lz77_params_t){128, 256, 0, 0, 0, 1, 8, 1, 0}
               : table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level];
}
 
typedef enum {
    ZXC_BLOCK_RAW = 0,
    ZXC_BLOCK_GLO = 1,
    ZXC_BLOCK_GHI = 2,
    ZXC_BLOCK_SEK = 254,
    ZXC_BLOCK_EOF = 255
} zxc_block_type_t;
 
typedef enum {
    ZXC_SECTION_ENCODING_RAW = 0,
    ZXC_SECTION_ENCODING_RLE = 1,
    ZXC_SECTION_ENCODING_HUFFMAN = 2,
    ZXC_SECTION_ENCODING_HUFFMAN_DICT = 3
} zxc_section_encoding_t;
 
typedef struct {
    uint32_t n_sequences;  // Number of sequences
    uint32_t n_literals;   // Number of literals
    uint8_t enc_lit;       // Literal stream encoding
    uint8_t enc_tok;       // Token section encoding (GLO only)
    uint8_t enc_mlen;      // Reserved (see above)
    uint8_t enc_off;       // Offset stream width (GLO only; ignored on decode in GHI)
} zxc_gnr_header_t;
 
static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) {
    uint16_t v;
    ZXC_MEMCPY(&v, p, sizeof(v));
#ifdef ZXC_BIG_ENDIAN
    return ZXC_BSWAP16(v);
#else
    return v;
#endif
}
 
static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) {
    uint32_t v;
    ZXC_MEMCPY(&v, p, sizeof(v));
#ifdef ZXC_BIG_ENDIAN
    return ZXC_BSWAP32(v);
#else
    return v;
#endif
}
 
static ZXC_ALWAYS_INLINE int zxc_block_is_short_dist_bound(const uint8_t* const blk,
                                                           const size_t size) {
    const size_t d_max = ZXC_LZ_MINDIST;
    // Too small to sample, and too small to gain: keep every distance.
    if (size < d_max + sizeof(uint32_t)) return 1;
 
    size_t want = size / ZXC_LZ_MINDIST_PROBE_PER_KB;
    if (want < ZXC_LZ_MINDIST_PROBE_MIN) want = ZXC_LZ_MINDIST_PROBE_MIN;
    if (want > ZXC_LZ_MINDIST_PROBE_MAX) want = ZXC_LZ_MINDIST_PROBE_MAX;
    size_t stride = size / want;
    if (stride < d_max) stride = d_max;
 
    const size_t planned = (size - sizeof(uint32_t) - d_max) / stride + 1;
    const size_t bar = (size_t)ZXC_LZ_MINDIST_MAX_SHORT_PCT * planned;
 
    size_t samples = 0;
    size_t hits = 0;
    for (size_t i = d_max; i + sizeof(uint32_t) <= size; i += stride) {
        const uint32_t cur = zxc_le32(blk + i);
        samples++;
        for (size_t d = 1; d < d_max; d++) {
            if (zxc_le32(blk + i - d) == cur) {
                hits++;
                break;
            }
        }
        if (hits * 100U > bar) return 1;                           // can only rise
        if ((hits + (planned - samples)) * 100U <= bar) return 0;  // out of reach
    }
    return hits * 100U > bar;
}
 
static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) {
    uint64_t v;
    ZXC_MEMCPY(&v, p, sizeof(v));
#ifdef ZXC_BIG_ENDIAN
    return ZXC_BSWAP64(v);
#else
    return v;
#endif
}
 
static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) {
#ifdef ZXC_BIG_ENDIAN
    const uint16_t s = ZXC_BSWAP16(v);
    ZXC_MEMCPY(p, &s, sizeof(s));
#else
    ZXC_MEMCPY(p, &v, sizeof(v));
#endif
}
 
static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) {
#ifdef ZXC_BIG_ENDIAN
    const uint32_t s = ZXC_BSWAP32(v);
    ZXC_MEMCPY(p, &s, sizeof(s));
#else
    ZXC_MEMCPY(p, &v, sizeof(v));
#endif
}
 
static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) {
#ifdef ZXC_BIG_ENDIAN
    const uint64_t s = ZXC_BSWAP64(v);
    ZXC_MEMCPY(p, &s, sizeof(s));
#else
    ZXC_MEMCPY(p, &v, sizeof(v));
#endif
}
 
static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) {
    const uint64_t h = (zxc_le64(p) ^ ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64;
 
    return (uint8_t)(h >> 56);
}
 
static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) {
    const uint64_t h1 = (zxc_le64(p) ^ ZXC_HASH_MULT64) * ZXC_HASH_MULT64;
 
    return (uint16_t)(((h1 + zxc_le64(p + 8) + ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64) >> 48);
}
 
static ZXC_ALWAYS_INLINE void zxc_file_header_sign(uint8_t* h) {
    zxc_store_le16(h + 14, 0);
    zxc_store_le16(h + 14, zxc_hash16(h));
}
 
static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) {
#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2)
    // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store
    _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32)
    vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
#else
    ZXC_MEMCPY(dst, src, 16);
#endif
}
 
static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) {
#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512)
    // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store
    _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src));
#elif defined(ZXC_USE_SSE2)
    const __m128i a = _mm_loadu_si128((const __m128i*)src);
    const __m128i b = _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16));
    _mm_storeu_si128((__m128i*)dst, a);
    _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), b);
#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32)
    const uint8x16_t a = vld1q_u8((const uint8_t*)src);
    const uint8x16_t b = vld1q_u8((const uint8_t*)src + 16);
    vst1q_u8((uint8_t*)dst, a);
    vst1q_u8((uint8_t*)dst + 16, b);
#else
    ZXC_MEMCPY(dst, src, 32);
#endif
}
 
static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) {
    if (x == 0) return 32;
#if defined(__GNUC__) || defined(__clang__)
    return __builtin_ctz(x);
#elif defined(_MSC_VER)
    unsigned long r;
    _BitScanForward(&r, x);
    return (int)r;
#else
    // Fallback De Bruijn (32 bits)
    static const int DeBruijn32[32] = {0,  1,  28, 2,  29, 14, 24, 3,  30, 22, 20,
                                       15, 25, 17, 4,  8,  31, 27, 13, 23, 21, 19,
                                       16, 7,  26, 12, 18, 6,  11, 5,  10, 9};
    return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27];
#endif
}
 
static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) {
    if (x == 0) return 64;
#if defined(__GNUC__) || defined(__clang__)
    return __builtin_ctzll(x);
#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64))
    unsigned long r;
    _BitScanForward64(&r, x);
    return (int)r;
#elif defined(_MSC_VER)
    // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication.
    unsigned long r;
    const uint32_t lo = (uint32_t)x;
    if (_BitScanForward(&r, lo)) return (int)r;
    _BitScanForward(&r, (uint32_t)(x >> 32));
    return 32 + (int)r;
#else
    // Fallback De Bruijn for non-GCC/non-MSVC compilers
    static const int Debruijn64[64] = {
        0,  1,  48, 2,  57, 49, 28, 3,  61, 58, 50, 42, 38, 29, 17, 4,  62, 55, 59, 36, 53, 51,
        43, 22, 45, 39, 33, 30, 24, 18, 12, 5,  63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21,
        44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9,  13, 8,  7,  6};
    return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58];
#endif
}
 
static ZXC_ALWAYS_INLINE uint64_t zxc_seek_block_count(const uint64_t total_decomp,
                                                       const size_t block_size) {
    // A shift, block_size being a header power of two: no 64-bit division on 32-bit
    // hosts. Not (total + bs - 1) >> s, which wraps near 2^64 on a forged footer.
    if (UNLIKELY(!block_size)) return 0;
    const int s = zxc_ctz32((uint32_t)block_size);
    return (total_decomp >> s) + ((total_decomp & (block_size - 1)) != 0);
}
 
// ============================================================================
// COMPRESSION CONTEXT & STRUCTS
// ============================================================================
 
// INTERNAL API
// ------------
 
static ZXC_ALWAYS_INLINE uint32_t zxc_checksum(const void* RESTRICT input, const size_t len,
                                               const uint64_t seed, const uint8_t hash_method) {
    (void)hash_method; /* single algorithm for now; extend when adding more */
    const uint64_t hash = rapidhash_withSeed(input, len, seed);
 
    return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT)));
}
 
static ZXC_ALWAYS_INLINE uint64_t zxc_digest_combine(const uint64_t acc,
                                                     const uint32_t block_checksum) {
    const uint64_t a = acc ^ ((uint64_t)block_checksum + 1U) * ZXC_HASH_GOLDEN64;
    const uint64_t b = ZXC_HASH_XORSHIFT64;
#if defined(__SIZEOF_INT128__)
    const __uint128_t r = (__uint128_t)a * b;
    return (uint64_t)r ^ (uint64_t)(r >> 64);
#else
    const uint64_t alo = (uint32_t)a, ahi = a >> 32, blo = (uint32_t)b, bhi = b >> 32;
    const uint64_t lolo = alo * blo, lohi = alo * bhi, hilo = ahi * blo, hihi = ahi * bhi;
    const uint64_t cross = (lolo >> 32) + (uint32_t)lohi + (uint32_t)hilo;
    const uint64_t hi = hihi + (lohi >> 32) + (hilo >> 32) + (cross >> 32);
    const uint64_t lo = (cross << 32) | (uint32_t)lolo;
    return lo ^ hi;
#endif
}
 
int zxc_write_glo_header_and_desc(uint8_t* RESTRICT dst, const size_t rem,
                                  const zxc_gnr_header_t* RESTRICT gh, const uint32_t lit_comp,
                                  const uint32_t tok_comp);
 
int zxc_read_glo_header_and_desc(const uint8_t* RESTRICT src, const size_t len,
                                 zxc_gnr_header_t* RESTRICT gh, uint32_t* RESTRICT lit_comp,
                                 uint32_t* RESTRICT tok_comp);
 
int zxc_write_ghi_header(uint8_t* RESTRICT dst, const size_t rem,
                         const zxc_gnr_header_t* RESTRICT gh);
 
int zxc_read_ghi_header(const uint8_t* RESTRICT src, const size_t len,
                        zxc_gnr_header_t* RESTRICT gh);
 
// ============================================================================
// Huffman codec for the GLO literal stream (level >= 6).
//
// On-disk layout, decoder geometry and tunables: see
// @ref ZXC_HUF_MAX_CODE_LEN_ULTRA and the surrounding "Huffman Codec Constants"
// group above.
// ============================================================================
 
int zxc_huf_build_code_lengths(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len,
                               void* RESTRICT scratch, int max_code_len);
 
int zxc_huf_nudge_code_lengths(const uint32_t* RESTRICT freq, uint8_t* RESTRICT code_len,
                               void* RESTRICT scratch, size_t scratch_cap, int max_code_len);
 
void zxc_huf_nudge_cost(const uint8_t* RESTRICT code_len, const uint32_t* RESTRICT freq,
                        uint64_t* RESTRICT bits, uint64_t* RESTRICT touches);
 
void zxc_huf_pack_lengths(const uint8_t* RESTRICT code_len, uint8_t* RESTRICT out);
 
int zxc_huf_unpack_lengths(const uint8_t* RESTRICT in, uint8_t* RESTRICT code_len);
 
// --------------------------------------------------------------------------
// PivCo-Huffman section codec (enc 2/3)
//
// Layout from PivCo-Huffman by Marcin Zukowski
// (https://github.com/MarcinZukowski/pivco-huffman); implemented
// independently here. See zxc_huffman.c for the codec.
//
// Same code bits as canonical Huffman, reordered by tree LEVEL: for each
// internal node in BFS order, its branch bits (one per symbol routed through
// it, LSB-first, byte-aligned). No size fields - the decoder derives every run
// length from the root count and popcounts. Decoding is bottom-up level merges,
// shuffle-parallel and gather-free.
// --------------------------------------------------------------------------
 
#define ZXC_PIVCO_SCRATCH_PAD 32
 
size_t zxc_huf_calc_size(const uint32_t* RESTRICT freq, const uint8_t* RESTRICT code_len,
                         int with_header);
 
int zxc_huf_encode_section(const uint8_t* RESTRICT literals, size_t n_literals,
                           const uint32_t* RESTRICT freq, const uint8_t* RESTRICT code_len,
                           uint8_t* RESTRICT dst, size_t dst_cap);
 
int zxc_huf_dict_tree_build(const uint8_t* RESTRICT packed_lengths, zxc_pivco_tree_t* RESTRICT tree,
                            uint32_t* RESTRICT codes, uint8_t* RESTRICT code_len);
 
size_t zxc_huf_calc_size_dict(const uint32_t* RESTRICT freq, const uint8_t* RESTRICT code_len,
                              const zxc_pivco_tree_t* RESTRICT tree);
 
int zxc_huf_encode_section_dict(const uint8_t* RESTRICT literals, size_t n_literals,
                                const uint32_t* RESTRICT freq, const uint8_t* RESTRICT code_len,
                                const zxc_pivco_tree_t* RESTRICT tree,
                                const uint32_t* RESTRICT codes, uint8_t* RESTRICT dst,
                                size_t dst_cap);
 
int zxc_huf_decode_section(const uint8_t* RESTRICT payload, size_t payload_size,
                           uint8_t* RESTRICT dst, size_t n, uint8_t* RESTRICT scratch);
 
int zxc_huf_decode_section_dict(const uint8_t* RESTRICT payload, size_t payload_size,
                                uint8_t* RESTRICT dst, size_t n,
                                const zxc_pivco_tree_t* RESTRICT tree, uint8_t* RESTRICT scratch);
 
// ---------------------------------------------------------------------------
// Compression / decompression context.
//
// The context owns the working buffers (hash table, sequence buffers, scratch)
// that encoder and decoder reuse across blocks. It stays private - the public
// APIs already wrap it opaquely - so the layout can evolve (cache-line
// placement, extra scratch arenas) without breaking the ABI.
// ---------------------------------------------------------------------------
 
typedef struct {
    // Hot zone: random access / high frequency.
    // Kept at the start to ensure they reside in the first cache line (64 bytes).
    uint32_t* hash_table;  
    uint8_t* hash_tags;    
    uint16_t* chain_table; 
    void* memory_block;    
    uint32_t epoch;        
    // Warm zone: sequential access per sequence.
    uint32_t* buf_sequences; 
    uint8_t* buf_tokens;     
    uint16_t* buf_offsets;   
    uint8_t* buf_extras;     
    uint8_t* buf_split;      
    zxc_glo_split_hist_t* buf_split_hist; 
    uint8_t* literals;                    
    // Cold zone: configuration / scratch / resizeable.
    uint8_t* lit_buffer;            
    size_t lit_buffer_cap;          
    uint8_t* work_buf;              
    size_t work_buf_cap;            
    uint8_t* tok_buffer;            
    size_t tok_buffer_cap;          
    uint8_t* pivco_scratch;         
    size_t pivco_scratch_cap;       
    void* entropy_block;            
    uint8_t* opt_scratch;           
    size_t opt_scratch_cap;         
    int checksum_enabled;           
    int compression_level;          
    size_t dict_size;               
    uint8_t* dict_buffer;           
    size_t dict_buffer_cap;         
    zxc_dict_huf_state_t* dict_huf; 
    int dict_huf_tree_ok;           
    uint32_t* lit_freq_acc;         
    // Block-size derived parameters (computed once at init).
    size_t chunk_size;    
    uint32_t offset_bits; 
    uint32_t offset_mask; 
    uint32_t max_epoch;   
} zxc_cctx_t;
 
int zxc_cctx_init(zxc_cctx_t* ctx, const size_t chunk_size, const int mode, const int level,
                  const int checksum_enabled, const size_t dict_size);
 
int zxc_cctx_attach_dict_huf(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT lengths);
 
size_t zxc_cctx_compute_workspace_size(const size_t chunk_size, const int mode, const int level,
                                       const size_t dict_size);
 
int zxc_cctx_init_in_workspace(zxc_cctx_t* RESTRICT ctx, void* RESTRICT workspace,
                               const size_t workspace_size, const size_t chunk_size, const int mode,
                               const int level, const int checksum_enabled, const size_t dict_size,
                               const int defer_entropy_scratch);
 
int zxc_cctx_alloc_entropy_scratch(zxc_cctx_t* ctx);
 
void zxc_cctx_free(zxc_cctx_t* ctx);
 
int zxc_decompress_chunk_wrapper(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
                                 const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap,
                                 const uint64_t block_index);
int zxc_decompress_chunk_wrapper_dict(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
                                      const size_t src_sz, uint8_t* RESTRICT dst,
                                      const size_t dst_cap, const uint64_t block_index);
int zxc_decompress_chunk_wrapper_safe(const zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
                                      const size_t src_sz, uint8_t* RESTRICT dst,
                                      const size_t dst_cap, const uint64_t block_index);
 
int zxc_compress_chunk_wrapper(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT src,
                               const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap,
                               const uint64_t block_index);
 
int64_t zxc_compress_block_at(zxc_cctx* cctx, const void* RESTRICT src, size_t src_size,
                              void* RESTRICT dst, size_t dst_capacity,
                              const zxc_compress_opts_t* opts, uint64_t block_index);
 
// ---------------------------------------------------------------------------
// Internal frame primitives.
//
// Read/write the ZXC file header, block header and footer. Kept internal:
// exposing them would freeze on-disk details (block_flags layout, footer
// composition) that stay free to evolve until the format is declared stable.
// ---------------------------------------------------------------------------
 
typedef struct {
    uint8_t block_type;      
    uint8_t block_flags;     
    uint8_t reserved;        
    uint8_t header_checksum; 
    uint32_t comp_size;      
} zxc_block_header_t;
 
int zxc_write_file_header(uint8_t* RESTRICT dst, const size_t dst_capacity, const size_t chunk_size,
                          const int has_checksum, const uint32_t dict_id, const int has_seek);
 
int zxc_read_file_header(const uint8_t* RESTRICT src, const size_t src_size, size_t* out_block_size,
                         int* out_has_checksum, uint32_t* out_dict_id, int* out_has_seek);
 
static inline int zxc_short_input_verdict(const uint8_t* p, const size_t n) {
    size_t chunk = 0;
    const int rc = zxc_read_file_header(p, n < ZXC_FILE_HEADER_SIZE ? n : ZXC_FILE_HEADER_SIZE,
                                        &chunk, NULL, NULL, NULL);
    return rc != ZXC_OK ? rc : ZXC_ERROR_SRC_TOO_SMALL;
}
 
// ---------------------------------------------------------------------------
// Container scan: frames back to back, measured without decoding (FORMAT.md,
// Sec 2.1).
// ---------------------------------------------------------------------------
 
typedef struct zxc_scan_src_s {
    int (*read_at)(const struct zxc_scan_src_s* src, uint64_t off, void* dst, size_t len);
    const uint8_t* data; 
    void* ctx;           
    uint64_t size;       
} zxc_scan_src_t;
 
static ZXC_ALWAYS_INLINE zxc_scan_src_t zxc_scan_src_mem(const uint8_t* data, const size_t size) {
    zxc_scan_src_t src = {NULL, data, NULL, (uint64_t)size};
    return src;
}
 
typedef struct {
    uint64_t dsize;         
    uint64_t frames;        
    uint32_t first_dict_id; 
    int uses_dict;          
} zxc_container_info_t;
 
int zxc_container_next(const zxc_scan_src_t* src, uint64_t pos);
 
int zxc_scan_frame(const zxc_scan_src_t* src, uint64_t end, uint64_t* start, zxc_frame_info_t* info,
                   size_t* footer_len);
 
int zxc_scan_whole_frame(const zxc_scan_src_t* src, zxc_frame_info_t* info, size_t* footer_len);
 
int zxc_scan_container(const zxc_scan_src_t* src, size_t req_chunk, zxc_container_info_t* info);
 
int zxc_write_block_header(uint8_t* RESTRICT dst, const size_t dst_capacity,
                           const zxc_block_header_t* bh);
 
int zxc_read_block_header(const uint8_t* RESTRICT src, const size_t src_size,
                          zxc_block_header_t* bh);
 
static ZXC_ALWAYS_INLINE size_t zxc_uint_bytes(const uint64_t v) {
    size_t n = 1;
    while (n < sizeof(uint64_t) && (v >> (8 * n)) != 0) n++;
    return n;
}
 
typedef struct {
    size_t skip;              
    size_t nd;                
    size_t nf;                
    size_t len;               
    uint64_t compressed_size; 
} zxc_footer_layout_t;
 
static ZXC_ALWAYS_INLINE zxc_footer_layout_t zxc_footer_layout(const uint64_t prefix,
                                                               const uint64_t src_size,
                                                               const int checksum_enabled) {
    zxc_footer_layout_t l;
    l.skip = checksum_enabled ? (size_t)ZXC_FILE_DIGEST_SIZE : 0U;
    l.nd = zxc_uint_bytes(src_size);
    const uint64_t base = prefix + l.skip + l.nd + 1;
    l.nf = 1;
    while (zxc_uint_bytes(base + l.nf) > l.nf) l.nf++;
    l.compressed_size = base + l.nf;
    l.len = l.skip + l.nd + l.nf + 1;
    return l;
}
 
int zxc_write_file_footer(uint8_t* RESTRICT dst, const size_t dst_capacity, const uint64_t prefix,
                          const uint64_t src_size, const uint64_t digest,
                          const int checksum_enabled);
 
int zxc_check_file_footer(const uint8_t* footer, size_t avail, const zxc_footer_layout_t* l,
                          uint64_t src_size, uint64_t* digest);
 
int zxc_parse_file_footer(const uint8_t* end, size_t avail, uint64_t* src_size,
                          uint64_t* compressed_size, size_t* sizes_len);
 
#define ZXC_FOOTER_MAX_SIZE_WITH_DIGEST (ZXC_FILE_DIGEST_SIZE + ZXC_FILE_FOOTER_MAX_SIZE)
 
#define ZXC_FRAME_MIN_SIZE (ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_MIN_SIZE)
 
static inline size_t zxc_frame_tail_len(const uint64_t total) {
    const uint64_t body = total - (ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE);
    return body < ZXC_FOOTER_MAX_SIZE_WITH_DIGEST ? (size_t)body : ZXC_FOOTER_MAX_SIZE_WITH_DIGEST;
}
 
int zxc_read_frame_info(const uint8_t* header, const uint8_t* tail, size_t tail_len, uint64_t total,
                        zxc_frame_info_t* info, size_t* footer_len);
 
static ZXC_ALWAYS_INLINE void zxc_frame_info_copy(zxc_frame_info_t* info, const size_t info_size,
                                                  const zxc_frame_info_t* got) {
    ZXC_MEMCPY(info, got, info_size < sizeof(*got) ? info_size : sizeof(*got));
}
static ZXC_ALWAYS_INLINE int zxc_footer_dsize_plausible(const uint64_t dsize,
                                                        const size_t chunk_size,
                                                        const uint64_t comp_size) {
    return zxc_seek_block_count(dsize, chunk_size) <= comp_size / ZXC_BLOCK_HEADER_SIZE;
}
 
int zxc_seek_table_header(uint8_t* dst, size_t dst_capacity, uint64_t num_blocks);
 
int zxc_check_eof_header(const uint8_t* hdr);
 
int zxc_check_seek_header(const uint8_t* hdr, size_t avail, uint64_t total_out, size_t block_size,
                          uint64_t* sek_bytes);
 
// ---------------------------------------------------------------------------
// Seekable cross-TU hooks (defined in zxc_seekable.c, consumed by the
// FILE*-flavored open helper in zxc_driver.c).
// -------------------------------------------------------------------------
 
void zxc_seekable_attach_owned_ctx(zxc_seekable* s, void* ctx);
 
size_t zxc_seek_write_group(uint8_t* RESTRICT dst, uint64_t* RESTRICT anchor,
                            const uint32_t* RESTRICT sizes, uint32_t cnt);
 
 
#ifdef __cplusplus
}
#endif
 
#endif  // ZXC_INTERNAL_H