Pre-trained dictionary API for ZXC compression. More...

#include <stddef.h>
#include <stdint.h>
#include "zxc_export.h"

Functions
uint32_t	zxc_dict_id (const void dict, size_t dict_size, const void huf_lengths)
	Computes the dictionary ID for the given content and optional table.

int	zxc_dict_load (const void buf, size_t buf_size, const void content_out, size_t content_size_out, const void *huf_out, uint32_t dict_id_out)
	Loads and validates a `.zxd` dictionary file from a memory buffer.

int64_t	zxc_dict_save (const void content, size_t content_size, const void huf_lengths, void *buf, size_t buf_capacity)
	Serializes dictionary content and its shared Huffman table to `.zxd`.

size_t	zxc_dict_save_bound (size_t content_size)
	Returns the `.zxd` file size for a given content size.

uint32_t	zxc_dict_get_id (const void *buf, size_t buf_size)
	Returns the dictionary ID stored in a `.zxd` file buffer.

int64_t	zxc_train_dict (const void const samples, const size_t sample_sizes, size_t n_samples, void dict_buf, size_t dict_capacity)
	Trains a dictionary from a corpus of samples.

int	zxc_train_dict_huf (const void const samples, const size_t sample_sizes, size_t n_samples, const void dict, size_t dict_size, uint8_t *huf_lengths_out)
	Trains the shared literal Huffman table for an already-trained dictionary.

int64_t	zxc_dict_train (const void const samples, const size_t sample_sizes, size_t n_samples, void zxd_buf, size_t zxd_capacity)
	One-call dictionary creation: content + shared table, serialized to ready-to-write `.zxd` bytes.

const void *	zxc_dict_huf (const void *buf, size_t buf_size)
	Returns a pointer to the shared Huffman table inside a `.zxd` buffer.

Detailed Description

Pre-trained dictionary API for ZXC compression.

Train, save, load and identify dictionaries, which buy back compression ratio on small, similar payloads. A dictionary is raw byte content that prefills the LZ77 window at the start of every block, so the compressor starts out already knowing the patterns instead of waiting for them to show up in the input. Dictionaries live in .zxd files and are referenced by a 32-bit ID in the ZXC file header.

// Train a dictionary from a corpus of JSON samples
void* dict_buf = malloc(32768);
int64_t dict_sz = zxc_train_dict(samples, sizes, n, dict_buf, 32768);
 
// Train the shared literal Huffman table on the same corpus
uint8_t huf[ZXC_HUF_TABLE_SIZE];
zxc_train_dict_huf(samples, sizes, n, dict_buf, dict_sz, huf);
 
// Save to .zxd file (content + table)
void* zxd = malloc(zxc_dict_save_bound(dict_sz));
int64_t zxd_sz = zxc_dict_save(dict_buf, dict_sz, huf, zxd, ...);
 
// Use for compression
zxc_compress_opts_t opts = {
    .level = 6, .dict = dict_buf, .dict_size = dict_sz, .dict_huf = huf };
zxc_compress(src, src_size, dst, dst_capacity, &opts);

Definition in file zxc_dict.h.

Functions

Detailed Description