| 1 | #include <stddef.h> |
| 2 | #include <stdint.h> |
| 3 | #include <stdlib.h> // malloc |
| 4 | #include <stdio.h> |
| 5 | #include <assert.h> |
| 6 | #include <string.h> |
| 7 | |
| 8 | #include "../zstd_seekable.h" |
| 9 | |
| 10 | |
| 11 | /* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */ |
| 12 | typedef struct { |
| 13 | const void *ptr; |
| 14 | size_t size; |
| 15 | size_t pos; |
| 16 | size_t totalRead; |
| 17 | } buffWrapperWithTotal_t; |
| 18 | |
| 19 | static int readBuffWithTotal(void* opaque, void* buffer, size_t n) |
| 20 | { |
| 21 | buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque; |
| 22 | assert(buff != NULL); |
| 23 | if (buff->pos + n > buff->size) return -1; |
| 24 | memcpy(buffer, (const char*)buff->ptr + buff->pos, n); |
| 25 | buff->pos += n; |
| 26 | buff->totalRead += n; |
| 27 | return 0; |
| 28 | } |
| 29 | |
| 30 | static int seekBuffWithTotal(void* opaque, long long offset, int origin) |
| 31 | { |
| 32 | buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque; |
| 33 | unsigned long long newOffset; |
| 34 | assert(buff != NULL); |
| 35 | switch (origin) { |
| 36 | case SEEK_SET: |
| 37 | assert(offset >= 0); |
| 38 | newOffset = (unsigned long long)offset; |
| 39 | break; |
| 40 | case SEEK_CUR: |
| 41 | newOffset = (unsigned long long)((long long)buff->pos + offset); |
| 42 | break; |
| 43 | case SEEK_END: |
| 44 | newOffset = (unsigned long long)((long long)buff->size + offset); |
| 45 | break; |
| 46 | default: |
| 47 | assert(0); /* not possible */ |
| 48 | } |
| 49 | if (newOffset > buff->size) { |
| 50 | return -1; |
| 51 | } |
| 52 | buff->pos = newOffset; |
| 53 | return 0; |
| 54 | } |
| 55 | |
| 56 | /* Basic unit tests for zstd seekable format */ |
| 57 | int main(int argc, const char** argv) |
| 58 | { |
| 59 | unsigned testNb = 1; |
| 60 | (void)argc; (void)argv; |
| 61 | printf("Beginning zstd seekable format tests...\n"); |
| 62 | |
| 63 | printf("Test %u - simple round trip: ", testNb++); |
| 64 | { size_t const inSize = 4000; |
| 65 | void* const inBuffer = malloc(inSize); |
| 66 | assert(inBuffer != NULL); |
| 67 | |
| 68 | size_t const seekCapacity = 5000; |
| 69 | void* const seekBuffer = malloc(seekCapacity); |
| 70 | assert(seekBuffer != NULL); |
| 71 | size_t seekSize; |
| 72 | |
| 73 | size_t const outCapacity = inSize; |
| 74 | void* const outBuffer = malloc(outCapacity); |
| 75 | assert(outBuffer != NULL); |
| 76 | |
| 77 | ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
| 78 | assert(zscs != NULL); |
| 79 | |
| 80 | { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */); |
| 81 | assert(!ZSTD_isError(initStatus)); |
| 82 | } |
| 83 | |
| 84 | { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
| 85 | ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
| 86 | |
| 87 | size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
| 88 | assert(!ZSTD_isError(cStatus)); |
| 89 | assert(inb.pos == inb.size); |
| 90 | |
| 91 | size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
| 92 | assert(!ZSTD_isError(endStatus)); |
| 93 | seekSize = outb.pos; |
| 94 | } |
| 95 | |
| 96 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 97 | assert(stream != NULL); |
| 98 | { size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize); |
| 99 | assert(!ZSTD_isError(initStatus)); } |
| 100 | |
| 101 | { size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0); |
| 102 | assert(decStatus == inSize); } |
| 103 | |
| 104 | /* unit test ZSTD_seekTable functions */ |
| 105 | ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream); |
| 106 | assert(zst != NULL); |
| 107 | |
| 108 | unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst); |
| 109 | assert(nbFrames > 0); |
| 110 | |
| 111 | unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0); |
| 112 | assert(frame0Offset == 0); |
| 113 | |
| 114 | unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0); |
| 115 | assert(content0Offset == 0); |
| 116 | |
| 117 | size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0); |
| 118 | assert(!ZSTD_isError(cSize)); |
| 119 | assert(cSize <= seekCapacity); |
| 120 | |
| 121 | size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0); |
| 122 | assert(origSize == inSize); |
| 123 | |
| 124 | unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1); |
| 125 | assert(fo1idx == 0); |
| 126 | |
| 127 | free(inBuffer); |
| 128 | free(seekBuffer); |
| 129 | free(outBuffer); |
| 130 | ZSTD_seekable_freeCStream(zscs); |
| 131 | ZSTD_seekTable_free(zst); |
| 132 | ZSTD_seekable_free(stream); |
| 133 | } |
| 134 | printf("Success!\n"); |
| 135 | |
| 136 | |
| 137 | printf("Test %u - check that seekable decompress does not hang: ", testNb++); |
| 138 | { /* Github issue #2335 */ |
| 139 | const size_t compressed_size = 17; |
| 140 | const uint8_t compressed_data[17] = { |
| 141 | '^', |
| 142 | '*', |
| 143 | 'M', |
| 144 | '\x18', |
| 145 | '\t', |
| 146 | '\x00', |
| 147 | '\x00', |
| 148 | '\x00', |
| 149 | '\x00', |
| 150 | '\x00', |
| 151 | '\x00', |
| 152 | '\x00', |
| 153 | (uint8_t)('\x03'), |
| 154 | (uint8_t)('\xb1'), |
| 155 | (uint8_t)('\xea'), |
| 156 | (uint8_t)('\x92'), |
| 157 | (uint8_t)('\x8f'), |
| 158 | }; |
| 159 | const size_t uncompressed_size = 32; |
| 160 | uint8_t uncompressed_data[32]; |
| 161 | |
| 162 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 163 | assert(stream != NULL); |
| 164 | { size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
| 165 | if (ZSTD_isError(status)) { |
| 166 | ZSTD_seekable_free(stream); |
| 167 | goto _test_error; |
| 168 | } } |
| 169 | |
| 170 | /* Should return an error, but not hang */ |
| 171 | { const size_t offset = 2; |
| 172 | size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
| 173 | if (!ZSTD_isError(status)) { |
| 174 | ZSTD_seekable_free(stream); |
| 175 | goto _test_error; |
| 176 | } } |
| 177 | |
| 178 | ZSTD_seekable_free(stream); |
| 179 | } |
| 180 | printf("Success!\n"); |
| 181 | |
| 182 | printf("Test %u - check #2 that seekable decompress does not hang: ", testNb++); |
| 183 | { /* Github issue #FIXME */ |
| 184 | const size_t compressed_size = 27; |
| 185 | const uint8_t compressed_data[27] = { |
| 186 | (uint8_t)'\x28', |
| 187 | (uint8_t)'\xb5', |
| 188 | (uint8_t)'\x2f', |
| 189 | (uint8_t)'\xfd', |
| 190 | (uint8_t)'\x00', |
| 191 | (uint8_t)'\x32', |
| 192 | (uint8_t)'\x91', |
| 193 | (uint8_t)'\x00', |
| 194 | (uint8_t)'\x00', |
| 195 | (uint8_t)'\x00', |
| 196 | (uint8_t)'\x5e', |
| 197 | (uint8_t)'\x2a', |
| 198 | (uint8_t)'\x4d', |
| 199 | (uint8_t)'\x18', |
| 200 | (uint8_t)'\x09', |
| 201 | (uint8_t)'\x00', |
| 202 | (uint8_t)'\x00', |
| 203 | (uint8_t)'\x00', |
| 204 | (uint8_t)'\x00', |
| 205 | (uint8_t)'\x00', |
| 206 | (uint8_t)'\x00', |
| 207 | (uint8_t)'\x00', |
| 208 | (uint8_t)'\x00', |
| 209 | (uint8_t)'\xb1', |
| 210 | (uint8_t)'\xea', |
| 211 | (uint8_t)'\x92', |
| 212 | (uint8_t)'\x8f', |
| 213 | }; |
| 214 | const size_t uncompressed_size = 400; |
| 215 | uint8_t uncompressed_data[400]; |
| 216 | |
| 217 | ZSTD_seekable* stream = ZSTD_seekable_create(); |
| 218 | size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
| 219 | if (ZSTD_isError(status)) { |
| 220 | ZSTD_seekable_free(stream); |
| 221 | goto _test_error; |
| 222 | } |
| 223 | |
| 224 | const size_t offset = 2; |
| 225 | /* Should return an error, but not hang */ |
| 226 | status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
| 227 | if (!ZSTD_isError(status)) { |
| 228 | ZSTD_seekable_free(stream); |
| 229 | goto _test_error; |
| 230 | } |
| 231 | |
| 232 | ZSTD_seekable_free(stream); |
| 233 | } |
| 234 | printf("Success!\n"); |
| 235 | |
| 236 | |
| 237 | printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++); |
| 238 | { // compressing empty string should return a zstd header |
| 239 | size_t const capacity = 255; |
| 240 | char* inBuffer = malloc(capacity); |
| 241 | assert(inBuffer != NULL); |
| 242 | inBuffer[0] = '\0'; |
| 243 | void* const outBuffer = malloc(capacity); |
| 244 | assert(outBuffer != NULL); |
| 245 | |
| 246 | ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream(); |
| 247 | ZSTD_seekable_initCStream(s, 1, 1, 255); |
| 248 | |
| 249 | ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 }; |
| 250 | ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity }; |
| 251 | |
| 252 | ZSTD_seekable_compressStream(s, &output, &input); |
| 253 | ZSTD_seekable_endStream(s, &output); |
| 254 | |
| 255 | if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) { |
| 256 | printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] ); |
| 257 | |
| 258 | free(inBuffer); |
| 259 | free(outBuffer); |
| 260 | ZSTD_seekable_freeCStream(s); |
| 261 | goto _test_error; |
| 262 | } |
| 263 | |
| 264 | free(inBuffer); |
| 265 | free(outBuffer); |
| 266 | ZSTD_seekable_freeCStream(s); |
| 267 | } |
| 268 | printf("Success!\n"); |
| 269 | |
| 270 | |
| 271 | printf("Test %u - multiple decompress calls: ", testNb++); |
| 272 | { char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt"; |
| 273 | size_t const inSize = sizeof(inBuffer); |
| 274 | |
| 275 | size_t const seekCapacity = 5000; |
| 276 | void* const seekBuffer = malloc(seekCapacity); |
| 277 | assert(seekBuffer != NULL); |
| 278 | size_t seekSize; |
| 279 | |
| 280 | size_t const outCapacity = inSize; |
| 281 | char* const outBuffer = malloc(outCapacity); |
| 282 | assert(outBuffer != NULL); |
| 283 | |
| 284 | ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
| 285 | assert(zscs != NULL); |
| 286 | |
| 287 | /* compress test data with a small frame size to ensure multiple frames in the output */ |
| 288 | unsigned const maxFrameSize = 40; |
| 289 | { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize); |
| 290 | assert(!ZSTD_isError(initStatus)); |
| 291 | } |
| 292 | |
| 293 | { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
| 294 | ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
| 295 | |
| 296 | while (inb.pos < inb.size) { |
| 297 | size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
| 298 | assert(!ZSTD_isError(cStatus)); |
| 299 | } |
| 300 | |
| 301 | size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
| 302 | assert(!ZSTD_isError(endStatus)); |
| 303 | seekSize = outb.pos; |
| 304 | } |
| 305 | |
| 306 | ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 307 | assert(stream != NULL); |
| 308 | buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0}; |
| 309 | { ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal}; |
| 310 | size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile); |
| 311 | assert(!ZSTD_isError(initStatus)); } |
| 312 | |
| 313 | /* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte) |
| 314 | and check that we didn't reread input data unnecessarily */ |
| 315 | size_t pos; |
| 316 | for (pos = 0; pos < inSize; pos += 2) { |
| 317 | size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos); |
| 318 | if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) { |
| 319 | goto _test_error; |
| 320 | } |
| 321 | } |
| 322 | if (buffWrapper.totalRead > seekSize) { |
| 323 | /* We read more than the compressed size, meaning there were some rereads. |
| 324 | This is unneeded because we only seeked forward. */ |
| 325 | printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize); |
| 326 | goto _test_error; |
| 327 | } |
| 328 | |
| 329 | /* Perform some reads and seeks to ensure correctness */ |
| 330 | struct { |
| 331 | size_t offset; |
| 332 | size_t size; |
| 333 | } const tests[] = { /* Assume the frame size is 40 */ |
| 334 | {20, 40}, /* read partial data from two frames */ |
| 335 | {60, 10}, /* continue reading from the same offset */ |
| 336 | {50, 20}, /* seek backward within the same frame */ |
| 337 | {10, 10}, /* seek backward to a different frame */ |
| 338 | {25, 10}, /* seek forward within the same frame */ |
| 339 | {60, 10}, /* seek forward to a different frame */ |
| 340 | }; |
| 341 | size_t idx; |
| 342 | for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) { |
| 343 | size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset); |
| 344 | if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) { |
| 345 | goto _test_error; |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | free(seekBuffer); |
| 350 | free(outBuffer); |
| 351 | ZSTD_seekable_freeCStream(zscs); |
| 352 | ZSTD_seekable_free(stream); |
| 353 | } |
| 354 | printf("Success!\n"); |
| 355 | |
| 356 | /* TODO: Add more tests */ |
| 357 | printf("Finished tests\n"); |
| 358 | return 0; |
| 359 | |
| 360 | _test_error: |
| 361 | printf("test failed! Exiting..\n"); |
| 362 | return 1; |
| 363 | } |