648db22b |
1 | /* |
2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | * You may select, at your option, one of the above-listed licenses. |
9 | */ |
10 | |
11 | #include "config.h" |
12 | |
13 | /* Define a config for each fast level we want to test with. */ |
14 | #define FAST_LEVEL(x) \ |
15 | param_value_t const level_fast##x##_param_values[] = { \ |
16 | {.param = ZSTD_c_compressionLevel, .value = -x}, \ |
17 | }; \ |
18 | config_t const level_fast##x = { \ |
19 | .name = "level -" #x, \ |
20 | .cli_args = "--fast=" #x, \ |
21 | .param_values = PARAM_VALUES(level_fast##x##_param_values), \ |
22 | }; \ |
23 | config_t const level_fast##x##_dict = { \ |
24 | .name = "level -" #x " with dict", \ |
25 | .cli_args = "--fast=" #x, \ |
26 | .param_values = PARAM_VALUES(level_fast##x##_param_values), \ |
27 | .use_dictionary = 1, \ |
28 | }; |
29 | |
30 | /* Define a config for each level we want to test with. */ |
31 | #define LEVEL(x) \ |
32 | param_value_t const level_##x##_param_values[] = { \ |
33 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
34 | }; \ |
35 | param_value_t const level_##x##_param_values_dms[] = { \ |
36 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
37 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
38 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \ |
39 | }; \ |
40 | param_value_t const level_##x##_param_values_dds[] = { \ |
41 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
42 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \ |
43 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \ |
44 | }; \ |
45 | param_value_t const level_##x##_param_values_dictcopy[] = { \ |
46 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
47 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
48 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \ |
49 | }; \ |
50 | param_value_t const level_##x##_param_values_dictload[] = { \ |
51 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
52 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
53 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \ |
54 | }; \ |
55 | config_t const level_##x = { \ |
56 | .name = "level " #x, \ |
57 | .cli_args = "-" #x, \ |
58 | .param_values = PARAM_VALUES(level_##x##_param_values), \ |
59 | }; \ |
60 | config_t const level_##x##_dict = { \ |
61 | .name = "level " #x " with dict", \ |
62 | .cli_args = "-" #x, \ |
63 | .param_values = PARAM_VALUES(level_##x##_param_values), \ |
64 | .use_dictionary = 1, \ |
65 | }; \ |
66 | config_t const level_##x##_dict_dms = { \ |
67 | .name = "level " #x " with dict dms", \ |
68 | .cli_args = "-" #x, \ |
69 | .param_values = PARAM_VALUES(level_##x##_param_values_dms), \ |
70 | .use_dictionary = 1, \ |
71 | .advanced_api_only = 1, \ |
72 | }; \ |
73 | config_t const level_##x##_dict_dds = { \ |
74 | .name = "level " #x " with dict dds", \ |
75 | .cli_args = "-" #x, \ |
76 | .param_values = PARAM_VALUES(level_##x##_param_values_dds), \ |
77 | .use_dictionary = 1, \ |
78 | .advanced_api_only = 1, \ |
79 | }; \ |
80 | config_t const level_##x##_dict_copy = { \ |
81 | .name = "level " #x " with dict copy", \ |
82 | .cli_args = "-" #x, \ |
83 | .param_values = PARAM_VALUES(level_##x##_param_values_dictcopy), \ |
84 | .use_dictionary = 1, \ |
85 | .advanced_api_only = 1, \ |
86 | }; \ |
87 | config_t const level_##x##_dict_load = { \ |
88 | .name = "level " #x " with dict load", \ |
89 | .cli_args = "-" #x, \ |
90 | .param_values = PARAM_VALUES(level_##x##_param_values_dictload), \ |
91 | .use_dictionary = 1, \ |
92 | .advanced_api_only = 1, \ |
93 | }; |
94 | |
95 | /* Define a config specifically to test row hash based levels and settings. |
96 | */ |
97 | #define ROW_LEVEL(x, y) \ |
98 | param_value_t const row_##y##_level_##x##_param_values[] = { \ |
99 | {.param = ZSTD_c_useRowMatchFinder, .value = y}, \ |
100 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
101 | }; \ |
102 | param_value_t const row_##y##_level_##x##_param_values_dms[] = { \ |
103 | {.param = ZSTD_c_useRowMatchFinder, .value = y}, \ |
104 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
105 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
106 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \ |
107 | }; \ |
108 | param_value_t const row_##y##_level_##x##_param_values_dds[] = { \ |
109 | {.param = ZSTD_c_useRowMatchFinder, .value = y}, \ |
110 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
111 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \ |
112 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \ |
113 | }; \ |
114 | param_value_t const row_##y##_level_##x##_param_values_dictcopy[] = { \ |
115 | {.param = ZSTD_c_useRowMatchFinder, .value = y}, \ |
116 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
117 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
118 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \ |
119 | }; \ |
120 | param_value_t const row_##y##_level_##x##_param_values_dictload[] = { \ |
121 | {.param = ZSTD_c_useRowMatchFinder, .value = y}, \ |
122 | {.param = ZSTD_c_compressionLevel, .value = x}, \ |
123 | {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \ |
124 | {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \ |
125 | }; \ |
126 | config_t const row_##y##_level_##x = { \ |
127 | .name = "level " #x " row " #y, \ |
128 | .cli_args = "-" #x, \ |
129 | .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values), \ |
130 | .advanced_api_only = 1, \ |
131 | }; \ |
132 | config_t const row_##y##_level_##x##_dict_dms = { \ |
133 | .name = "level " #x " row " #y " with dict dms", \ |
134 | .cli_args = "-" #x, \ |
135 | .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dms), \ |
136 | .use_dictionary = 1, \ |
137 | .advanced_api_only = 1, \ |
138 | }; \ |
139 | config_t const row_##y##_level_##x##_dict_dds = { \ |
140 | .name = "level " #x " row " #y " with dict dds", \ |
141 | .cli_args = "-" #x, \ |
142 | .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dds), \ |
143 | .use_dictionary = 1, \ |
144 | .advanced_api_only = 1, \ |
145 | }; \ |
146 | config_t const row_##y##_level_##x##_dict_copy = { \ |
147 | .name = "level " #x " row " #y" with dict copy", \ |
148 | .cli_args = "-" #x, \ |
149 | .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictcopy), \ |
150 | .use_dictionary = 1, \ |
151 | .advanced_api_only = 1, \ |
152 | }; \ |
153 | config_t const row_##y##_level_##x##_dict_load = { \ |
154 | .name = "level " #x " row " #y " with dict load", \ |
155 | .cli_args = "-" #x, \ |
156 | .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictload), \ |
157 | .use_dictionary = 1, \ |
158 | .advanced_api_only = 1, \ |
159 | }; |
160 | |
161 | #define PARAM_VALUES(pv) \ |
162 | { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) } |
163 | |
164 | #include "levels.h" |
165 | |
166 | #undef LEVEL |
167 | #undef FAST_LEVEL |
168 | #undef ROW_LEVEL |
169 | |
170 | static config_t no_pledged_src_size = { |
171 | .name = "no source size", |
172 | .cli_args = "", |
173 | .param_values = PARAM_VALUES(level_0_param_values), |
174 | .no_pledged_src_size = 1, |
175 | }; |
176 | |
177 | static config_t no_pledged_src_size_with_dict = { |
178 | .name = "no source size with dict", |
179 | .cli_args = "", |
180 | .param_values = PARAM_VALUES(level_0_param_values), |
181 | .no_pledged_src_size = 1, |
182 | .use_dictionary = 1, |
183 | }; |
184 | |
185 | static param_value_t const ldm_param_values[] = { |
186 | {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable}, |
187 | }; |
188 | |
189 | static config_t ldm = { |
190 | .name = "long distance mode", |
191 | .cli_args = "--long", |
192 | .param_values = PARAM_VALUES(ldm_param_values), |
193 | }; |
194 | |
195 | static param_value_t const mt_param_values[] = { |
196 | {.param = ZSTD_c_nbWorkers, .value = 2}, |
197 | }; |
198 | |
199 | static config_t mt = { |
200 | .name = "multithreaded", |
201 | .cli_args = "-T2", |
202 | .param_values = PARAM_VALUES(mt_param_values), |
203 | }; |
204 | |
205 | static param_value_t const mt_ldm_param_values[] = { |
206 | {.param = ZSTD_c_nbWorkers, .value = 2}, |
207 | {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable}, |
208 | }; |
209 | |
210 | static config_t mt_ldm = { |
211 | .name = "multithreaded long distance mode", |
212 | .cli_args = "-T2 --long", |
213 | .param_values = PARAM_VALUES(mt_ldm_param_values), |
214 | }; |
215 | |
216 | static param_value_t mt_advanced_param_values[] = { |
217 | {.param = ZSTD_c_nbWorkers, .value = 2}, |
218 | {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, |
219 | }; |
220 | |
221 | static config_t mt_advanced = { |
222 | .name = "multithreaded with advanced params", |
223 | .cli_args = "-T2 --no-compress-literals", |
224 | .param_values = PARAM_VALUES(mt_advanced_param_values), |
225 | }; |
226 | |
227 | static param_value_t const small_wlog_param_values[] = { |
228 | {.param = ZSTD_c_windowLog, .value = 10}, |
229 | }; |
230 | |
231 | static config_t small_wlog = { |
232 | .name = "small window log", |
233 | .cli_args = "--zstd=wlog=10", |
234 | .param_values = PARAM_VALUES(small_wlog_param_values), |
235 | }; |
236 | |
237 | static param_value_t const small_hlog_param_values[] = { |
238 | {.param = ZSTD_c_hashLog, .value = 6}, |
239 | {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, |
240 | }; |
241 | |
242 | static config_t small_hlog = { |
243 | .name = "small hash log", |
244 | .cli_args = "--zstd=hlog=6,strat=7", |
245 | .param_values = PARAM_VALUES(small_hlog_param_values), |
246 | }; |
247 | |
248 | static param_value_t const small_clog_param_values[] = { |
249 | {.param = ZSTD_c_chainLog, .value = 6}, |
250 | {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, |
251 | }; |
252 | |
253 | static config_t small_clog = { |
254 | .name = "small chain log", |
255 | .cli_args = "--zstd=clog=6,strat=7", |
256 | .param_values = PARAM_VALUES(small_clog_param_values), |
257 | }; |
258 | |
259 | static param_value_t const uncompressed_literals_param_values[] = { |
260 | {.param = ZSTD_c_compressionLevel, .value = 3}, |
261 | {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, |
262 | }; |
263 | |
264 | static config_t uncompressed_literals = { |
265 | .name = "uncompressed literals", |
266 | .cli_args = "-3 --no-compress-literals", |
267 | .param_values = PARAM_VALUES(uncompressed_literals_param_values), |
268 | }; |
269 | |
270 | static param_value_t const uncompressed_literals_opt_param_values[] = { |
271 | {.param = ZSTD_c_compressionLevel, .value = 19}, |
272 | {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable}, |
273 | }; |
274 | |
275 | static config_t uncompressed_literals_opt = { |
276 | .name = "uncompressed literals optimal", |
277 | .cli_args = "-19 --no-compress-literals", |
278 | .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values), |
279 | }; |
280 | |
281 | static param_value_t const huffman_literals_param_values[] = { |
282 | {.param = ZSTD_c_compressionLevel, .value = -1}, |
283 | {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_enable}, |
284 | }; |
285 | |
286 | static config_t huffman_literals = { |
287 | .name = "huffman literals", |
288 | .cli_args = "--fast=1 --compress-literals", |
289 | .param_values = PARAM_VALUES(huffman_literals_param_values), |
290 | }; |
291 | |
292 | static param_value_t const explicit_params_param_values[] = { |
293 | {.param = ZSTD_c_checksumFlag, .value = 1}, |
294 | {.param = ZSTD_c_contentSizeFlag, .value = 0}, |
295 | {.param = ZSTD_c_dictIDFlag, .value = 0}, |
296 | {.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy}, |
297 | {.param = ZSTD_c_windowLog, .value = 18}, |
298 | {.param = ZSTD_c_hashLog, .value = 21}, |
299 | {.param = ZSTD_c_chainLog, .value = 21}, |
300 | {.param = ZSTD_c_targetLength, .value = 100}, |
301 | }; |
302 | |
303 | static config_t explicit_params = { |
304 | .name = "explicit params", |
305 | .cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100", |
306 | .param_values = PARAM_VALUES(explicit_params_param_values), |
307 | }; |
308 | |
309 | static config_t const* g_configs[] = { |
310 | |
311 | #define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict, |
312 | #define LEVEL(x) &level_##x, &level_##x##_dict, &level_##x##_dict_dms, &level_##x##_dict_dds, &level_##x##_dict_copy, &level_##x##_dict_load, |
313 | #define ROW_LEVEL(x, y) &row_##y##_level_##x, &row_##y##_level_##x##_dict_dms, &row_##y##_level_##x##_dict_dds, &row_##y##_level_##x##_dict_copy, &row_##y##_level_##x##_dict_load, |
314 | #include "levels.h" |
315 | #undef ROW_LEVEL |
316 | #undef LEVEL |
317 | #undef FAST_LEVEL |
318 | |
319 | &no_pledged_src_size, |
320 | &no_pledged_src_size_with_dict, |
321 | &ldm, |
322 | &mt, |
323 | &mt_ldm, |
324 | &small_wlog, |
325 | &small_hlog, |
326 | &small_clog, |
327 | &explicit_params, |
328 | &uncompressed_literals, |
329 | &uncompressed_literals_opt, |
330 | &huffman_literals, |
331 | &mt_advanced, |
332 | NULL, |
333 | }; |
334 | |
335 | config_t const* const* configs = g_configs; |
336 | |
337 | int config_skip_data(config_t const* config, data_t const* data) { |
338 | return config->use_dictionary && !data_has_dict(data); |
339 | } |
340 | |
341 | int config_get_level(config_t const* config) |
342 | { |
343 | param_values_t const params = config->param_values; |
344 | size_t i; |
345 | for (i = 0; i < params.size; ++i) { |
346 | if (params.data[i].param == ZSTD_c_compressionLevel) |
347 | return (int)params.data[i].value; |
348 | } |
349 | return CONFIG_NO_LEVEL; |
350 | } |
351 | |
352 | ZSTD_parameters config_get_zstd_params( |
353 | config_t const* config, |
354 | uint64_t srcSize, |
355 | size_t dictSize) |
356 | { |
357 | ZSTD_parameters zparams = {}; |
358 | param_values_t const params = config->param_values; |
359 | int level = config_get_level(config); |
360 | if (level == CONFIG_NO_LEVEL) |
361 | level = 3; |
362 | zparams = ZSTD_getParams( |
363 | level, |
364 | config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize, |
365 | dictSize); |
366 | for (size_t i = 0; i < params.size; ++i) { |
367 | unsigned const value = params.data[i].value; |
368 | switch (params.data[i].param) { |
369 | case ZSTD_c_contentSizeFlag: |
370 | zparams.fParams.contentSizeFlag = value; |
371 | break; |
372 | case ZSTD_c_checksumFlag: |
373 | zparams.fParams.checksumFlag = value; |
374 | break; |
375 | case ZSTD_c_dictIDFlag: |
376 | zparams.fParams.noDictIDFlag = !value; |
377 | break; |
378 | case ZSTD_c_windowLog: |
379 | zparams.cParams.windowLog = value; |
380 | break; |
381 | case ZSTD_c_chainLog: |
382 | zparams.cParams.chainLog = value; |
383 | break; |
384 | case ZSTD_c_hashLog: |
385 | zparams.cParams.hashLog = value; |
386 | break; |
387 | case ZSTD_c_searchLog: |
388 | zparams.cParams.searchLog = value; |
389 | break; |
390 | case ZSTD_c_minMatch: |
391 | zparams.cParams.minMatch = value; |
392 | break; |
393 | case ZSTD_c_targetLength: |
394 | zparams.cParams.targetLength = value; |
395 | break; |
396 | case ZSTD_c_strategy: |
397 | zparams.cParams.strategy = (ZSTD_strategy)value; |
398 | break; |
399 | default: |
400 | break; |
401 | } |
402 | } |
403 | return zparams; |
404 | } |