2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
11 /* *****************************************************************************
12 * Constructs a dictionary using a heuristic based on the following paper:
14 * Liao, Petri, Moffat, Wirth
15 * Effective Construction of Relative Lempel-Ziv Dictionaries
16 * Published in WWW 2016.
18 * Adapted from code originally written by @ot (Giuseppe Ottaviano).
19 ******************************************************************************/
21 /*-*************************************
23 ***************************************/
24 #include <stdio.h> /* fprintf */
25 #include <stdlib.h> /* malloc, free, qsort */
26 #include <string.h> /* memset */
27 #include <time.h> /* clock */
29 #ifndef ZDICT_STATIC_LINKING_ONLY
30 # define ZDICT_STATIC_LINKING_ONLY
33 #include "../common/mem.h" /* read */
34 #include "../common/pool.h" /* POOL_ctx */
35 #include "../common/threading.h" /* ZSTD_pthread_mutex_t */
36 #include "../common/zstd_internal.h" /* includes zstd.h */
37 #include "../common/bits.h" /* ZSTD_highbit32 */
41 /*-*************************************
43 ***************************************/
45 * There are 32bit indexes used to ref samples, so limit samples size to 4GB
47 * For 32bit builds we choose 1 GB.
48 * Most 32bit platforms have 2GB user-mode addressable space and we allocate a large
49 * contiguous buffer, so 1GB is already a high limit.
51 #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
52 #define COVER_DEFAULT_SPLITPOINT 1.0
54 /*-*************************************
56 ***************************************/
57 #ifndef LOCALDISPLAYLEVEL
58 static int g_displayLevel = 0;
61 #define DISPLAY(...) \
63 fprintf(stderr, __VA_ARGS__); \
66 #undef LOCALDISPLAYLEVEL
67 #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
68 if (displayLevel >= l) { \
69 DISPLAY(__VA_ARGS__); \
70 } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
72 #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
74 #ifndef LOCALDISPLAYUPDATE
75 static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
76 static clock_t g_time = 0;
78 #undef LOCALDISPLAYUPDATE
79 #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
80 if (displayLevel >= l) { \
81 if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
83 DISPLAY(__VA_ARGS__); \
87 #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
89 /*-*************************************
91 ***************************************
92 * A small specialized hash map for storing activeDmers.
93 * The map does not resize, so if it becomes full it will loop forever.
94 * Thus, the map must be large enough to store every value.
95 * The map implements linear probing and keeps its load less than 0.5.
98 #define MAP_EMPTY_VALUE ((U32)-1)
99 typedef struct COVER_map_pair_t_s {
104 typedef struct COVER_map_s {
105 COVER_map_pair_t *data;
114 static void COVER_map_clear(COVER_map_t *map) {
115 memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t));
119 * Initializes a map of the given size.
120 * Returns 1 on success and 0 on failure.
121 * The map must be destroyed with COVER_map_destroy().
122 * The map is only guaranteed to be large enough to hold size elements.
124 static int COVER_map_init(COVER_map_t *map, U32 size) {
125 map->sizeLog = ZSTD_highbit32(size) + 2;
126 map->size = (U32)1 << map->sizeLog;
127 map->sizeMask = map->size - 1;
128 map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t));
134 COVER_map_clear(map);
139 * Internal hash function
141 static const U32 COVER_prime4bytes = 2654435761U;
142 static U32 COVER_map_hash(COVER_map_t *map, U32 key) {
143 return (key * COVER_prime4bytes) >> (32 - map->sizeLog);
147 * Helper function that returns the index that a key should be placed into.
149 static U32 COVER_map_index(COVER_map_t *map, U32 key) {
150 const U32 hash = COVER_map_hash(map, key);
152 for (i = hash;; i = (i + 1) & map->sizeMask) {
153 COVER_map_pair_t *pos = &map->data[i];
154 if (pos->value == MAP_EMPTY_VALUE) {
157 if (pos->key == key) {
164 * Returns the pointer to the value for key.
165 * If key is not in the map, it is inserted and the value is set to 0.
166 * The map must not be full.
168 static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
169 COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)];
170 if (pos->value == MAP_EMPTY_VALUE) {
178 * Deletes key from the map if present.
180 static void COVER_map_remove(COVER_map_t *map, U32 key) {
181 U32 i = COVER_map_index(map, key);
182 COVER_map_pair_t *del = &map->data[i];
184 if (del->value == MAP_EMPTY_VALUE) {
187 for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) {
188 COVER_map_pair_t *const pos = &map->data[i];
189 /* If the position is empty we are done */
190 if (pos->value == MAP_EMPTY_VALUE) {
191 del->value = MAP_EMPTY_VALUE;
194 /* If pos can be moved to del do so */
195 if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) {
197 del->value = pos->value;
207 * Destroys a map that is inited with COVER_map_init().
209 static void COVER_map_destroy(COVER_map_t *map) {
217 /*-*************************************
219 ***************************************/
224 const size_t *samplesSizes;
226 size_t nbTrainSamples;
227 size_t nbTestSamples;
235 /* We need a global context for qsort... */
236 static COVER_ctx_t *g_coverCtx = NULL;
238 /*-*************************************
240 ***************************************/
243 * Returns the sum of the sample sizes.
245 size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
248 for (i = 0; i < nbSamples; ++i) {
249 sum += samplesSizes[i];
255 * Returns -1 if the dmer at lp is less than the dmer at rp.
256 * Return 0 if the dmers at lp and rp are equal.
257 * Returns 1 if the dmer at lp is greater than the dmer at rp.
259 static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) {
260 U32 const lhs = *(U32 const *)lp;
261 U32 const rhs = *(U32 const *)rp;
262 return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d);
265 * Faster version for d <= 8.
267 static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
268 U64 const mask = (ctx->d == 8) ? (U64)-1 : (((U64)1 << (8 * ctx->d)) - 1);
269 U64 const lhs = MEM_readLE64(ctx->samples + *(U32 const *)lp) & mask;
270 U64 const rhs = MEM_readLE64(ctx->samples + *(U32 const *)rp) & mask;
278 * Same as COVER_cmp() except ties are broken by pointer value
279 * NOTE: g_coverCtx must be set to call this function. A global is required because
280 * qsort doesn't take an opaque pointer.
282 static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
283 int result = COVER_cmp(g_coverCtx, lp, rp);
285 result = lp < rp ? -1 : 1;
290 * Faster version for d <= 8.
292 static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
293 int result = COVER_cmp8(g_coverCtx, lp, rp);
295 result = lp < rp ? -1 : 1;
301 * Returns the first pointer in [first, last) whose element does not compare
302 * less than value. If no such element exists it returns last.
304 static const size_t *COVER_lower_bound(const size_t* first, const size_t* last,
306 size_t count = (size_t)(last - first);
307 assert(last >= first);
309 size_t step = count / 2;
310 const size_t *ptr = first;
323 * Generic groupBy function.
324 * Groups an array sorted by cmp into groups with equivalent values.
325 * Calls grp for each group.
328 COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx,
329 int (*cmp)(COVER_ctx_t *, const void *, const void *),
330 void (*grp)(COVER_ctx_t *, const void *, const void *)) {
331 const BYTE *ptr = (const BYTE *)data;
333 while (num < count) {
334 const BYTE *grpEnd = ptr + size;
336 while (num < count && cmp(ctx, ptr, grpEnd) == 0) {
340 grp(ctx, ptr, grpEnd);
345 /*-*************************************
347 ***************************************/
350 * Called on each group of positions with the same dmer.
351 * Counts the frequency of each dmer and saves it in the suffix array.
352 * Fills `ctx->dmerAt`.
354 static void COVER_group(COVER_ctx_t *ctx, const void *group,
355 const void *groupEnd) {
356 /* The group consists of all the positions with the same first d bytes. */
357 const U32 *grpPtr = (const U32 *)group;
358 const U32 *grpEnd = (const U32 *)groupEnd;
359 /* The dmerId is how we will reference this dmer.
360 * This allows us to map the whole dmer space to a much smaller space, the
361 * size of the suffix array.
363 const U32 dmerId = (U32)(grpPtr - ctx->suffix);
364 /* Count the number of samples this dmer shows up in */
367 const size_t *curOffsetPtr = ctx->offsets;
368 const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples;
369 /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a
370 * different sample than the last.
372 size_t curSampleEnd = ctx->offsets[0];
373 for (; grpPtr != grpEnd; ++grpPtr) {
374 /* Save the dmerId for this position so we can get back to it. */
375 ctx->dmerAt[*grpPtr] = dmerId;
376 /* Dictionaries only help for the first reference to the dmer.
377 * After that zstd can reference the match from the previous reference.
378 * So only count each dmer once for each sample it is in.
380 if (*grpPtr < curSampleEnd) {
384 /* Binary search to find the end of the sample *grpPtr is in.
385 * In the common case that grpPtr + 1 == grpEnd we can skip the binary
386 * search because the loop is over.
388 if (grpPtr + 1 != grpEnd) {
389 const size_t *sampleEndPtr =
390 COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr);
391 curSampleEnd = *sampleEndPtr;
392 curOffsetPtr = sampleEndPtr + 1;
395 /* At this point we are never going to look at this segment of the suffix
396 * array again. We take advantage of this fact to save memory.
397 * We store the frequency of the dmer in the first position of the group,
400 ctx->suffix[dmerId] = freq;
405 * Selects the best segment in an epoch.
406 * Segments of are scored according to the function:
408 * Let F(d) be the frequency of dmer d.
409 * Let S_i be the dmer at position i of segment S which has length k.
411 * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
413 * Once the dmer d is in the dictionary we set F(d) = 0.
415 static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs,
416 COVER_map_t *activeDmers, U32 begin,
418 ZDICT_cover_params_t parameters) {
420 const U32 k = parameters.k;
421 const U32 d = parameters.d;
422 const U32 dmersInK = k - d + 1;
423 /* Try each segment (activeSegment) and save the best (bestSegment) */
424 COVER_segment_t bestSegment = {0, 0, 0};
425 COVER_segment_t activeSegment;
426 /* Reset the activeDmers in the segment */
427 COVER_map_clear(activeDmers);
428 /* The activeSegment starts at the beginning of the epoch. */
429 activeSegment.begin = begin;
430 activeSegment.end = begin;
431 activeSegment.score = 0;
432 /* Slide the activeSegment through the whole epoch.
433 * Save the best segment in bestSegment.
435 while (activeSegment.end < end) {
436 /* The dmerId for the dmer at the next position */
437 U32 newDmer = ctx->dmerAt[activeSegment.end];
438 /* The entry in activeDmers for this dmerId */
439 U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer);
440 /* If the dmer isn't already present in the segment add its score. */
441 if (*newDmerOcc == 0) {
442 /* The paper suggest using the L-0.5 norm, but experiments show that it
445 activeSegment.score += freqs[newDmer];
447 /* Add the dmer to the segment */
448 activeSegment.end += 1;
451 /* If the window is now too large, drop the first position */
452 if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
453 U32 delDmer = ctx->dmerAt[activeSegment.begin];
454 U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer);
455 activeSegment.begin += 1;
457 /* If this is the last occurrence of the dmer, subtract its score */
458 if (*delDmerOcc == 0) {
459 COVER_map_remove(activeDmers, delDmer);
460 activeSegment.score -= freqs[delDmer];
464 /* If this segment is the best so far save it */
465 if (activeSegment.score > bestSegment.score) {
466 bestSegment = activeSegment;
470 /* Trim off the zero frequency head and tail from the segment. */
471 U32 newBegin = bestSegment.end;
472 U32 newEnd = bestSegment.begin;
474 for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
475 U32 freq = freqs[ctx->dmerAt[pos]];
477 newBegin = MIN(newBegin, pos);
481 bestSegment.begin = newBegin;
482 bestSegment.end = newEnd;
485 /* Zero out the frequency of each dmer covered by the chosen segment. */
487 for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
488 freqs[ctx->dmerAt[pos]] = 0;
495 * Check the validity of the parameters.
496 * Returns non-zero if the parameters are valid and 0 otherwise.
498 static int COVER_checkParameters(ZDICT_cover_params_t parameters,
499 size_t maxDictSize) {
500 /* k and d are required parameters */
501 if (parameters.d == 0 || parameters.k == 0) {
504 /* k <= maxDictSize */
505 if (parameters.k > maxDictSize) {
509 if (parameters.d > parameters.k) {
512 /* 0 < splitPoint <= 1 */
513 if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){
520 * Clean up a context initialized with `COVER_ctx_init()`.
522 static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
545 * Prepare a context for dictionary building.
546 * The context is only dependent on the parameter `d` and can be used multiple
548 * Returns 0 on success or error code on error.
549 * The context must be destroyed with `COVER_ctx_destroy()`.
551 static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
552 const size_t *samplesSizes, unsigned nbSamples,
553 unsigned d, double splitPoint)
555 const BYTE *const samples = (const BYTE *)samplesBuffer;
556 const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
557 /* Split samples into testing and training sets */
558 const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
559 const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
560 const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
561 const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
563 if (totalSamplesSize < MAX(d, sizeof(U64)) ||
564 totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
565 DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
566 (unsigned)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
567 return ERROR(srcSize_wrong);
569 /* Check if there are at least 5 training samples */
570 if (nbTrainSamples < 5) {
571 DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
572 return ERROR(srcSize_wrong);
574 /* Check if there's testing sample */
575 if (nbTestSamples < 1) {
576 DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
577 return ERROR(srcSize_wrong);
579 /* Zero the context */
580 memset(ctx, 0, sizeof(*ctx));
581 DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
582 (unsigned)trainingSamplesSize);
583 DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
584 (unsigned)testSamplesSize);
585 ctx->samples = samples;
586 ctx->samplesSizes = samplesSizes;
587 ctx->nbSamples = nbSamples;
588 ctx->nbTrainSamples = nbTrainSamples;
589 ctx->nbTestSamples = nbTestSamples;
590 /* Partial suffix array */
591 ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
592 ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
593 /* Maps index to the dmerID */
594 ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
595 /* The offsets of each file */
596 ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
597 if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
598 DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
599 COVER_ctx_destroy(ctx);
600 return ERROR(memory_allocation);
605 /* Fill offsets from the samplesSizes */
609 for (i = 1; i <= nbSamples; ++i) {
610 ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
613 DISPLAYLEVEL(2, "Constructing partial suffix array\n");
615 /* suffix is a partial suffix array.
616 * It only sorts suffixes by their first parameters.d bytes.
617 * The sort is stable, so each dmer group is sorted by position in input.
620 for (i = 0; i < ctx->suffixSize; ++i) {
623 /* qsort doesn't take an opaque pointer, so pass as a global.
624 * On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
627 #if defined(__OpenBSD__)
628 mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
629 (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
631 qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
632 (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
635 DISPLAYLEVEL(2, "Computing frequencies\n");
636 /* For each dmer group (group of positions with the same first d bytes):
637 * 1. For each position we set dmerAt[position] = dmerID. The dmerID is
638 * (groupBeginPtr - suffix). This allows us to go from position to
639 * dmerID so we can look up values in freq.
640 * 2. We calculate how many samples the dmer occurs in and save it in
643 COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx,
644 (ctx->d <= 8 ? &COVER_cmp8 : &COVER_cmp), &COVER_group);
645 ctx->freqs = ctx->suffix;
650 void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
652 const double ratio = (double)nbDmers / (double)maxDictSize;
656 LOCALDISPLAYLEVEL(displayLevel, 1,
657 "WARNING: The maximum dictionary size %u is too large "
658 "compared to the source size %u! "
659 "size(source)/size(dictionary) = %f, but it should be >= "
660 "10! This may lead to a subpar dictionary! We recommend "
661 "training on sources at least 10x, and preferably 100x "
662 "the size of the dictionary! \n", (U32)maxDictSize,
663 (U32)nbDmers, ratio);
666 COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize,
667 U32 nbDmers, U32 k, U32 passes)
669 const U32 minEpochSize = k * 10;
670 COVER_epoch_info_t epochs;
671 epochs.num = MAX(1, maxDictSize / k / passes);
672 epochs.size = nbDmers / epochs.num;
673 if (epochs.size >= minEpochSize) {
674 assert(epochs.size * epochs.num <= nbDmers);
677 epochs.size = MIN(minEpochSize, nbDmers);
678 epochs.num = nbDmers / epochs.size;
679 assert(epochs.size * epochs.num <= nbDmers);
684 * Given the prepared context build the dictionary.
686 static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
687 COVER_map_t *activeDmers, void *dictBuffer,
688 size_t dictBufferCapacity,
689 ZDICT_cover_params_t parameters) {
690 BYTE *const dict = (BYTE *)dictBuffer;
691 size_t tail = dictBufferCapacity;
692 /* Divide the data into epochs. We will select one segment from each epoch. */
693 const COVER_epoch_info_t epochs = COVER_computeEpochs(
694 (U32)dictBufferCapacity, (U32)ctx->suffixSize, parameters.k, 4);
695 const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3));
696 size_t zeroScoreRun = 0;
698 DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
699 (U32)epochs.num, (U32)epochs.size);
700 /* Loop through the epochs until there are no more segments or the dictionary
703 for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs.num) {
704 const U32 epochBegin = (U32)(epoch * epochs.size);
705 const U32 epochEnd = epochBegin + epochs.size;
707 /* Select a segment */
708 COVER_segment_t segment = COVER_selectSegment(
709 ctx, freqs, activeDmers, epochBegin, epochEnd, parameters);
710 /* If the segment covers no dmers, then we are out of content.
711 * There may be new content in other epochs, for continue for some time.
713 if (segment.score == 0) {
714 if (++zeroScoreRun >= maxZeroScoreRun) {
720 /* Trim the segment if necessary and if it is too small then we are done */
721 segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
722 if (segmentSize < parameters.d) {
725 /* We fill the dictionary from the back to allow the best segments to be
726 * referenced with the smallest offsets.
729 memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
732 (unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
734 DISPLAYLEVEL(2, "\r%79s\r", "");
738 ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
739 void *dictBuffer, size_t dictBufferCapacity,
740 const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
741 ZDICT_cover_params_t parameters)
743 BYTE* const dict = (BYTE*)dictBuffer;
745 COVER_map_t activeDmers;
746 parameters.splitPoint = 1.0;
747 /* Initialize global data */
748 g_displayLevel = (int)parameters.zParams.notificationLevel;
750 if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
751 DISPLAYLEVEL(1, "Cover parameters incorrect\n");
752 return ERROR(parameter_outOfBound);
754 if (nbSamples == 0) {
755 DISPLAYLEVEL(1, "Cover must have at least one input file\n");
756 return ERROR(srcSize_wrong);
758 if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
759 DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
761 return ERROR(dstSize_tooSmall);
763 /* Initialize context and activeDmers */
765 size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
766 parameters.d, parameters.splitPoint);
767 if (ZSTD_isError(initVal)) {
771 COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
772 if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
773 DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
774 COVER_ctx_destroy(&ctx);
775 return ERROR(memory_allocation);
778 DISPLAYLEVEL(2, "Building dictionary\n");
781 COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer,
782 dictBufferCapacity, parameters);
783 const size_t dictionarySize = ZDICT_finalizeDictionary(
784 dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
785 samplesBuffer, samplesSizes, nbSamples, parameters.zParams);
786 if (!ZSTD_isError(dictionarySize)) {
787 DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
788 (unsigned)dictionarySize);
790 COVER_ctx_destroy(&ctx);
791 COVER_map_destroy(&activeDmers);
792 return dictionarySize;
798 size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
799 const size_t *samplesSizes, const BYTE *samples,
801 size_t nbTrainSamples, size_t nbSamples,
802 BYTE *const dict, size_t dictBufferCapacity) {
803 size_t totalCompressedSize = ERROR(GENERIC);
808 /* Local variables */
811 /* Allocate dst with enough space to compress the maximum sized sample */
813 size_t maxSampleSize = 0;
814 i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
815 for (; i < nbSamples; ++i) {
816 maxSampleSize = MAX(samplesSizes[i], maxSampleSize);
818 dstCapacity = ZSTD_compressBound(maxSampleSize);
819 dst = malloc(dstCapacity);
821 /* Create the cctx and cdict */
822 cctx = ZSTD_createCCtx();
823 cdict = ZSTD_createCDict(dict, dictBufferCapacity,
824 parameters.zParams.compressionLevel);
825 if (!dst || !cctx || !cdict) {
826 goto _compressCleanup;
828 /* Compress each sample and sum their sizes (or error) */
829 totalCompressedSize = dictBufferCapacity;
830 i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
831 for (; i < nbSamples; ++i) {
832 const size_t size = ZSTD_compress_usingCDict(
833 cctx, dst, dstCapacity, samples + offsets[i],
834 samplesSizes[i], cdict);
835 if (ZSTD_isError(size)) {
836 totalCompressedSize = size;
837 goto _compressCleanup;
839 totalCompressedSize += size;
843 ZSTD_freeCDict(cdict);
847 return totalCompressedSize;
852 * Initialize the `COVER_best_t`.
854 void COVER_best_init(COVER_best_t *best) {
855 if (best==NULL) return; /* compatible with init on NULL */
856 (void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
857 (void)ZSTD_pthread_cond_init(&best->cond, NULL);
861 best->compressedSize = (size_t)-1;
862 memset(&best->parameters, 0, sizeof(best->parameters));
866 * Wait until liveJobs == 0.
868 void COVER_best_wait(COVER_best_t *best) {
872 ZSTD_pthread_mutex_lock(&best->mutex);
873 while (best->liveJobs != 0) {
874 ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
876 ZSTD_pthread_mutex_unlock(&best->mutex);
880 * Call COVER_best_wait() and then destroy the COVER_best_t.
882 void COVER_best_destroy(COVER_best_t *best) {
886 COVER_best_wait(best);
890 ZSTD_pthread_mutex_destroy(&best->mutex);
891 ZSTD_pthread_cond_destroy(&best->cond);
895 * Called when a thread is about to be launched.
896 * Increments liveJobs.
898 void COVER_best_start(COVER_best_t *best) {
902 ZSTD_pthread_mutex_lock(&best->mutex);
904 ZSTD_pthread_mutex_unlock(&best->mutex);
908 * Called when a thread finishes executing, both on error or success.
909 * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
910 * If this dictionary is the best so far save it and its parameters.
912 void COVER_best_finish(COVER_best_t* best,
913 ZDICT_cover_params_t parameters,
914 COVER_dictSelection_t selection)
916 void* dict = selection.dictContent;
917 size_t compressedSize = selection.totalCompressedSize;
918 size_t dictSize = selection.dictSize;
924 ZSTD_pthread_mutex_lock(&best->mutex);
926 liveJobs = best->liveJobs;
927 /* If the new dictionary is better */
928 if (compressedSize < best->compressedSize) {
929 /* Allocate space if necessary */
930 if (!best->dict || best->dictSize < dictSize) {
934 best->dict = malloc(dictSize);
936 best->compressedSize = ERROR(GENERIC);
938 ZSTD_pthread_cond_signal(&best->cond);
939 ZSTD_pthread_mutex_unlock(&best->mutex);
943 /* Save the dictionary, parameters, and size */
945 memcpy(best->dict, dict, dictSize);
946 best->dictSize = dictSize;
947 best->parameters = parameters;
948 best->compressedSize = compressedSize;
952 ZSTD_pthread_cond_broadcast(&best->cond);
954 ZSTD_pthread_mutex_unlock(&best->mutex);
958 static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz)
960 COVER_dictSelection_t ds;
961 ds.dictContent = buf;
963 ds.totalCompressedSize = csz;
967 COVER_dictSelection_t COVER_dictSelectionError(size_t error) {
968 return setDictSelection(NULL, 0, error);
971 unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) {
972 return (ZSTD_isError(selection.totalCompressedSize) || !selection.dictContent);
975 void COVER_dictSelectionFree(COVER_dictSelection_t selection){
976 free(selection.dictContent);
979 COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
980 size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
981 size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
983 size_t largestDict = 0;
984 size_t largestCompressed = 0;
985 BYTE* customDictContentEnd = customDictContent + dictContentSize;
987 BYTE* largestDictbuffer = (BYTE*)malloc(dictBufferCapacity);
988 BYTE* candidateDictBuffer = (BYTE*)malloc(dictBufferCapacity);
989 double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
991 if (!largestDictbuffer || !candidateDictBuffer) {
992 free(largestDictbuffer);
993 free(candidateDictBuffer);
994 return COVER_dictSelectionError(dictContentSize);
997 /* Initial dictionary size and compressed size */
998 memcpy(largestDictbuffer, customDictContent, dictContentSize);
999 dictContentSize = ZDICT_finalizeDictionary(
1000 largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
1001 samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
1003 if (ZDICT_isError(dictContentSize)) {
1004 free(largestDictbuffer);
1005 free(candidateDictBuffer);
1006 return COVER_dictSelectionError(dictContentSize);
1009 totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes,
1010 samplesBuffer, offsets,
1011 nbCheckSamples, nbSamples,
1012 largestDictbuffer, dictContentSize);
1014 if (ZSTD_isError(totalCompressedSize)) {
1015 free(largestDictbuffer);
1016 free(candidateDictBuffer);
1017 return COVER_dictSelectionError(totalCompressedSize);
1020 if (params.shrinkDict == 0) {
1021 free(candidateDictBuffer);
1022 return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize);
1025 largestDict = dictContentSize;
1026 largestCompressed = totalCompressedSize;
1027 dictContentSize = ZDICT_DICTSIZE_MIN;
1029 /* Largest dict is initially at least ZDICT_DICTSIZE_MIN */
1030 while (dictContentSize < largestDict) {
1031 memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
1032 dictContentSize = ZDICT_finalizeDictionary(
1033 candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
1034 samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
1036 if (ZDICT_isError(dictContentSize)) {
1037 free(largestDictbuffer);
1038 free(candidateDictBuffer);
1039 return COVER_dictSelectionError(dictContentSize);
1043 totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes,
1044 samplesBuffer, offsets,
1045 nbCheckSamples, nbSamples,
1046 candidateDictBuffer, dictContentSize);
1048 if (ZSTD_isError(totalCompressedSize)) {
1049 free(largestDictbuffer);
1050 free(candidateDictBuffer);
1051 return COVER_dictSelectionError(totalCompressedSize);
1054 if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) {
1055 free(largestDictbuffer);
1056 return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize );
1058 dictContentSize *= 2;
1060 dictContentSize = largestDict;
1061 totalCompressedSize = largestCompressed;
1062 free(candidateDictBuffer);
1063 return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize );
1067 * Parameters for COVER_tryParameters().
1069 typedef struct COVER_tryParameters_data_s {
1070 const COVER_ctx_t *ctx;
1072 size_t dictBufferCapacity;
1073 ZDICT_cover_params_t parameters;
1074 } COVER_tryParameters_data_t;
1077 * Tries a set of parameters and updates the COVER_best_t with the results.
1078 * This function is thread safe if zstd is compiled with multithreaded support.
1079 * It takes its parameters as an *OWNING* opaque pointer to support threading.
1081 static void COVER_tryParameters(void *opaque)
1083 /* Save parameters as local variables */
1084 COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t*)opaque;
1085 const COVER_ctx_t *const ctx = data->ctx;
1086 const ZDICT_cover_params_t parameters = data->parameters;
1087 size_t dictBufferCapacity = data->dictBufferCapacity;
1088 size_t totalCompressedSize = ERROR(GENERIC);
1089 /* Allocate space for hash table, dict, and freqs */
1090 COVER_map_t activeDmers;
1091 BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
1092 COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
1093 U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
1094 if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
1095 DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
1098 if (!dict || !freqs) {
1099 DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
1102 /* Copy the frequencies because we need to modify them */
1103 memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32));
1104 /* Build the dictionary */
1106 const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
1107 dictBufferCapacity, parameters);
1108 selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
1109 ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
1110 totalCompressedSize);
1112 if (COVER_dictSelectionIsError(selection)) {
1113 DISPLAYLEVEL(1, "Failed to select dictionary\n");
1119 COVER_best_finish(data->best, parameters, selection);
1121 COVER_map_destroy(&activeDmers);
1122 COVER_dictSelectionFree(selection);
1126 ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
1127 void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer,
1128 const size_t* samplesSizes, unsigned nbSamples,
1129 ZDICT_cover_params_t* parameters)
1132 const unsigned nbThreads = parameters->nbThreads;
1133 const double splitPoint =
1134 parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint;
1135 const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
1136 const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
1137 const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
1138 const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
1139 const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
1140 const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
1141 const unsigned kIterations =
1142 (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
1143 const unsigned shrinkDict = 0;
1144 /* Local variables */
1145 const int displayLevel = parameters->zParams.notificationLevel;
1146 unsigned iteration = 1;
1150 POOL_ctx *pool = NULL;
1154 if (splitPoint <= 0 || splitPoint > 1) {
1155 LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
1156 return ERROR(parameter_outOfBound);
1158 if (kMinK < kMaxD || kMaxK < kMinK) {
1159 LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
1160 return ERROR(parameter_outOfBound);
1162 if (nbSamples == 0) {
1163 DISPLAYLEVEL(1, "Cover must have at least one input file\n");
1164 return ERROR(srcSize_wrong);
1166 if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
1167 DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
1168 ZDICT_DICTSIZE_MIN);
1169 return ERROR(dstSize_tooSmall);
1171 if (nbThreads > 1) {
1172 pool = POOL_create(nbThreads, 1);
1174 return ERROR(memory_allocation);
1177 /* Initialization */
1178 COVER_best_init(&best);
1179 /* Turn down global display level to clean up display at level 2 and below */
1180 g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
1181 /* Loop through d first because each new value needs a new context */
1182 LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
1184 for (d = kMinD; d <= kMaxD; d += 2) {
1185 /* Initialize the context for this value of d */
1187 LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
1189 const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint);
1190 if (ZSTD_isError(initVal)) {
1191 LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
1192 COVER_best_destroy(&best);
1198 COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
1201 /* Loop through k reusing the same context */
1202 for (k = kMinK; k <= kMaxK; k += kStepSize) {
1203 /* Prepare the arguments */
1204 COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
1205 sizeof(COVER_tryParameters_data_t));
1206 LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
1208 LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
1209 COVER_best_destroy(&best);
1210 COVER_ctx_destroy(&ctx);
1212 return ERROR(memory_allocation);
1216 data->dictBufferCapacity = dictBufferCapacity;
1217 data->parameters = *parameters;
1218 data->parameters.k = k;
1219 data->parameters.d = d;
1220 data->parameters.splitPoint = splitPoint;
1221 data->parameters.steps = kSteps;
1222 data->parameters.shrinkDict = shrinkDict;
1223 data->parameters.zParams.notificationLevel = g_displayLevel;
1224 /* Check the parameters */
1225 if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) {
1226 DISPLAYLEVEL(1, "Cover parameters incorrect\n");
1230 /* Call the function and pass ownership of data to it */
1231 COVER_best_start(&best);
1233 POOL_add(pool, &COVER_tryParameters, data);
1235 COVER_tryParameters(data);
1238 LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
1239 (unsigned)((iteration * 100) / kIterations));
1242 COVER_best_wait(&best);
1243 COVER_ctx_destroy(&ctx);
1245 LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
1246 /* Fill the output buffer and parameters with output of the best parameters */
1248 const size_t dictSize = best.dictSize;
1249 if (ZSTD_isError(best.compressedSize)) {
1250 const size_t compressedSize = best.compressedSize;
1251 COVER_best_destroy(&best);
1253 return compressedSize;
1255 *parameters = best.parameters;
1256 memcpy(dictBuffer, best.dict, dictSize);
1257 COVER_best_destroy(&best);