648db22b |
1 | #include <stddef.h> |
2 | #include <stdint.h> |
3 | #include <stdlib.h> // malloc |
4 | #include <stdio.h> |
5 | #include <assert.h> |
6 | #include <string.h> |
7 | |
8 | #include "../zstd_seekable.h" |
9 | |
10 | |
11 | /* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */ |
12 | typedef struct { |
13 | const void *ptr; |
14 | size_t size; |
15 | size_t pos; |
16 | size_t totalRead; |
17 | } buffWrapperWithTotal_t; |
18 | |
19 | static int readBuffWithTotal(void* opaque, void* buffer, size_t n) |
20 | { |
21 | buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque; |
22 | assert(buff != NULL); |
23 | if (buff->pos + n > buff->size) return -1; |
24 | memcpy(buffer, (const char*)buff->ptr + buff->pos, n); |
25 | buff->pos += n; |
26 | buff->totalRead += n; |
27 | return 0; |
28 | } |
29 | |
30 | static int seekBuffWithTotal(void* opaque, long long offset, int origin) |
31 | { |
32 | buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque; |
33 | unsigned long long newOffset; |
34 | assert(buff != NULL); |
35 | switch (origin) { |
36 | case SEEK_SET: |
37 | assert(offset >= 0); |
38 | newOffset = (unsigned long long)offset; |
39 | break; |
40 | case SEEK_CUR: |
41 | newOffset = (unsigned long long)((long long)buff->pos + offset); |
42 | break; |
43 | case SEEK_END: |
44 | newOffset = (unsigned long long)((long long)buff->size + offset); |
45 | break; |
46 | default: |
47 | assert(0); /* not possible */ |
48 | } |
49 | if (newOffset > buff->size) { |
50 | return -1; |
51 | } |
52 | buff->pos = newOffset; |
53 | return 0; |
54 | } |
55 | |
56 | /* Basic unit tests for zstd seekable format */ |
57 | int main(int argc, const char** argv) |
58 | { |
59 | unsigned testNb = 1; |
60 | (void)argc; (void)argv; |
61 | printf("Beginning zstd seekable format tests...\n"); |
62 | |
63 | printf("Test %u - simple round trip: ", testNb++); |
64 | { size_t const inSize = 4000; |
65 | void* const inBuffer = malloc(inSize); |
66 | assert(inBuffer != NULL); |
67 | |
68 | size_t const seekCapacity = 5000; |
69 | void* const seekBuffer = malloc(seekCapacity); |
70 | assert(seekBuffer != NULL); |
71 | size_t seekSize; |
72 | |
73 | size_t const outCapacity = inSize; |
74 | void* const outBuffer = malloc(outCapacity); |
75 | assert(outBuffer != NULL); |
76 | |
77 | ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
78 | assert(zscs != NULL); |
79 | |
80 | { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */); |
81 | assert(!ZSTD_isError(initStatus)); |
82 | } |
83 | |
84 | { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
85 | ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
86 | |
87 | size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
88 | assert(!ZSTD_isError(cStatus)); |
89 | assert(inb.pos == inb.size); |
90 | |
91 | size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
92 | assert(!ZSTD_isError(endStatus)); |
93 | seekSize = outb.pos; |
94 | } |
95 | |
96 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
97 | assert(stream != NULL); |
98 | { size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize); |
99 | assert(!ZSTD_isError(initStatus)); } |
100 | |
101 | { size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0); |
102 | assert(decStatus == inSize); } |
103 | |
104 | /* unit test ZSTD_seekTable functions */ |
105 | ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream); |
106 | assert(zst != NULL); |
107 | |
108 | unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst); |
109 | assert(nbFrames > 0); |
110 | |
111 | unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0); |
112 | assert(frame0Offset == 0); |
113 | |
114 | unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0); |
115 | assert(content0Offset == 0); |
116 | |
117 | size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0); |
118 | assert(!ZSTD_isError(cSize)); |
119 | assert(cSize <= seekCapacity); |
120 | |
121 | size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0); |
122 | assert(origSize == inSize); |
123 | |
124 | unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1); |
125 | assert(fo1idx == 0); |
126 | |
127 | free(inBuffer); |
128 | free(seekBuffer); |
129 | free(outBuffer); |
130 | ZSTD_seekable_freeCStream(zscs); |
131 | ZSTD_seekTable_free(zst); |
132 | ZSTD_seekable_free(stream); |
133 | } |
134 | printf("Success!\n"); |
135 | |
136 | |
137 | printf("Test %u - check that seekable decompress does not hang: ", testNb++); |
138 | { /* Github issue #2335 */ |
139 | const size_t compressed_size = 17; |
140 | const uint8_t compressed_data[17] = { |
141 | '^', |
142 | '*', |
143 | 'M', |
144 | '\x18', |
145 | '\t', |
146 | '\x00', |
147 | '\x00', |
148 | '\x00', |
149 | '\x00', |
150 | '\x00', |
151 | '\x00', |
152 | '\x00', |
153 | (uint8_t)('\x03'), |
154 | (uint8_t)('\xb1'), |
155 | (uint8_t)('\xea'), |
156 | (uint8_t)('\x92'), |
157 | (uint8_t)('\x8f'), |
158 | }; |
159 | const size_t uncompressed_size = 32; |
160 | uint8_t uncompressed_data[32]; |
161 | |
162 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
163 | assert(stream != NULL); |
164 | { size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
165 | if (ZSTD_isError(status)) { |
166 | ZSTD_seekable_free(stream); |
167 | goto _test_error; |
168 | } } |
169 | |
170 | /* Should return an error, but not hang */ |
171 | { const size_t offset = 2; |
172 | size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
173 | if (!ZSTD_isError(status)) { |
174 | ZSTD_seekable_free(stream); |
175 | goto _test_error; |
176 | } } |
177 | |
178 | ZSTD_seekable_free(stream); |
179 | } |
180 | printf("Success!\n"); |
181 | |
182 | printf("Test %u - check #2 that seekable decompress does not hang: ", testNb++); |
183 | { /* Github issue #FIXME */ |
184 | const size_t compressed_size = 27; |
185 | const uint8_t compressed_data[27] = { |
186 | (uint8_t)'\x28', |
187 | (uint8_t)'\xb5', |
188 | (uint8_t)'\x2f', |
189 | (uint8_t)'\xfd', |
190 | (uint8_t)'\x00', |
191 | (uint8_t)'\x32', |
192 | (uint8_t)'\x91', |
193 | (uint8_t)'\x00', |
194 | (uint8_t)'\x00', |
195 | (uint8_t)'\x00', |
196 | (uint8_t)'\x5e', |
197 | (uint8_t)'\x2a', |
198 | (uint8_t)'\x4d', |
199 | (uint8_t)'\x18', |
200 | (uint8_t)'\x09', |
201 | (uint8_t)'\x00', |
202 | (uint8_t)'\x00', |
203 | (uint8_t)'\x00', |
204 | (uint8_t)'\x00', |
205 | (uint8_t)'\x00', |
206 | (uint8_t)'\x00', |
207 | (uint8_t)'\x00', |
208 | (uint8_t)'\x00', |
209 | (uint8_t)'\xb1', |
210 | (uint8_t)'\xea', |
211 | (uint8_t)'\x92', |
212 | (uint8_t)'\x8f', |
213 | }; |
214 | const size_t uncompressed_size = 400; |
215 | uint8_t uncompressed_data[400]; |
216 | |
217 | ZSTD_seekable* stream = ZSTD_seekable_create(); |
218 | size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
219 | if (ZSTD_isError(status)) { |
220 | ZSTD_seekable_free(stream); |
221 | goto _test_error; |
222 | } |
223 | |
224 | const size_t offset = 2; |
225 | /* Should return an error, but not hang */ |
226 | status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
227 | if (!ZSTD_isError(status)) { |
228 | ZSTD_seekable_free(stream); |
229 | goto _test_error; |
230 | } |
231 | |
232 | ZSTD_seekable_free(stream); |
233 | } |
234 | printf("Success!\n"); |
235 | |
236 | |
237 | printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++); |
238 | { // compressing empty string should return a zstd header |
239 | size_t const capacity = 255; |
240 | char* inBuffer = malloc(capacity); |
241 | assert(inBuffer != NULL); |
242 | inBuffer[0] = '\0'; |
243 | void* const outBuffer = malloc(capacity); |
244 | assert(outBuffer != NULL); |
245 | |
246 | ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream(); |
247 | ZSTD_seekable_initCStream(s, 1, 1, 255); |
248 | |
249 | ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 }; |
250 | ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity }; |
251 | |
252 | ZSTD_seekable_compressStream(s, &output, &input); |
253 | ZSTD_seekable_endStream(s, &output); |
254 | |
255 | if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) { |
256 | printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] ); |
257 | |
258 | free(inBuffer); |
259 | free(outBuffer); |
260 | ZSTD_seekable_freeCStream(s); |
261 | goto _test_error; |
262 | } |
263 | |
264 | free(inBuffer); |
265 | free(outBuffer); |
266 | ZSTD_seekable_freeCStream(s); |
267 | } |
268 | printf("Success!\n"); |
269 | |
270 | |
271 | printf("Test %u - multiple decompress calls: ", testNb++); |
272 | { char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt"; |
273 | size_t const inSize = sizeof(inBuffer); |
274 | |
275 | size_t const seekCapacity = 5000; |
276 | void* const seekBuffer = malloc(seekCapacity); |
277 | assert(seekBuffer != NULL); |
278 | size_t seekSize; |
279 | |
280 | size_t const outCapacity = inSize; |
281 | char* const outBuffer = malloc(outCapacity); |
282 | assert(outBuffer != NULL); |
283 | |
284 | ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
285 | assert(zscs != NULL); |
286 | |
287 | /* compress test data with a small frame size to ensure multiple frames in the output */ |
288 | unsigned const maxFrameSize = 40; |
289 | { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize); |
290 | assert(!ZSTD_isError(initStatus)); |
291 | } |
292 | |
293 | { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
294 | ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
295 | |
296 | while (inb.pos < inb.size) { |
297 | size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
298 | assert(!ZSTD_isError(cStatus)); |
299 | } |
300 | |
301 | size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
302 | assert(!ZSTD_isError(endStatus)); |
303 | seekSize = outb.pos; |
304 | } |
305 | |
306 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
307 | assert(stream != NULL); |
308 | buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0}; |
309 | { ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal}; |
310 | size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile); |
311 | assert(!ZSTD_isError(initStatus)); } |
312 | |
313 | /* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte) |
314 | and check that we didn't reread input data unnecessarily */ |
315 | size_t pos; |
316 | for (pos = 0; pos < inSize; pos += 2) { |
317 | size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos); |
318 | if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) { |
319 | goto _test_error; |
320 | } |
321 | } |
322 | if (buffWrapper.totalRead > seekSize) { |
323 | /* We read more than the compressed size, meaning there were some rereads. |
324 | This is unneeded because we only seeked forward. */ |
325 | printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize); |
326 | goto _test_error; |
327 | } |
328 | |
329 | /* Perform some reads and seeks to ensure correctness */ |
330 | struct { |
331 | size_t offset; |
332 | size_t size; |
333 | } const tests[] = { /* Assume the frame size is 40 */ |
334 | {20, 40}, /* read partial data from two frames */ |
335 | {60, 10}, /* continue reading from the same offset */ |
336 | {50, 20}, /* seek backward within the same frame */ |
337 | {10, 10}, /* seek backward to a different frame */ |
338 | {25, 10}, /* seek forward within the same frame */ |
339 | {60, 10}, /* seek forward to a different frame */ |
340 | }; |
341 | size_t idx; |
342 | for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) { |
343 | size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset); |
344 | if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) { |
345 | goto _test_error; |
346 | } |
347 | } |
348 | |
349 | free(seekBuffer); |
350 | free(outBuffer); |
351 | ZSTD_seekable_freeCStream(zscs); |
352 | ZSTD_seekable_free(stream); |
353 | } |
354 | printf("Success!\n"); |
355 | |
356 | /* TODO: Add more tests */ |
357 | printf("Finished tests\n"); |
358 | return 0; |
359 | |
360 | _test_error: |
361 | printf("test failed! Exiting..\n"); |
362 | return 1; |
363 | } |