ce188d4d |
1 | /* Lzma2Enc.c -- LZMA2 Encoder\r |
2 | 2015-10-04 : Igor Pavlov : Public domain */\r |
3 | \r |
4 | #include "Precomp.h"\r |
5 | \r |
6 | /* #include <stdio.h> */\r |
7 | #include <string.h>\r |
8 | \r |
9 | /* #define _7ZIP_ST */\r |
10 | \r |
11 | #include "Lzma2Enc.h"\r |
12 | \r |
13 | #ifndef _7ZIP_ST\r |
14 | #include "MtCoder.h"\r |
15 | #else\r |
16 | #define NUM_MT_CODER_THREADS_MAX 1\r |
17 | #endif\r |
18 | \r |
19 | #define LZMA2_CONTROL_LZMA (1 << 7)\r |
20 | #define LZMA2_CONTROL_COPY_NO_RESET 2\r |
21 | #define LZMA2_CONTROL_COPY_RESET_DIC 1\r |
22 | #define LZMA2_CONTROL_EOF 0\r |
23 | \r |
24 | #define LZMA2_LCLP_MAX 4\r |
25 | \r |
26 | #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))\r |
27 | \r |
28 | #define LZMA2_PACK_SIZE_MAX (1 << 16)\r |
29 | #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX\r |
30 | #define LZMA2_UNPACK_SIZE_MAX (1 << 21)\r |
31 | #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX\r |
32 | \r |
33 | #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)\r |
34 | \r |
35 | \r |
36 | #define PRF(x) /* x */\r |
37 | \r |
38 | /* ---------- CLzma2EncInt ---------- */\r |
39 | \r |
40 | typedef struct\r |
41 | {\r |
42 | CLzmaEncHandle enc;\r |
43 | UInt64 srcPos;\r |
44 | Byte props;\r |
45 | Bool needInitState;\r |
46 | Bool needInitProp;\r |
47 | } CLzma2EncInt;\r |
48 | \r |
49 | static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)\r |
50 | {\r |
51 | Byte propsEncoded[LZMA_PROPS_SIZE];\r |
52 | SizeT propsSize = LZMA_PROPS_SIZE;\r |
53 | RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));\r |
54 | RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));\r |
55 | p->srcPos = 0;\r |
56 | p->props = propsEncoded[0];\r |
57 | p->needInitState = True;\r |
58 | p->needInitProp = True;\r |
59 | return SZ_OK;\r |
60 | }\r |
61 | \r |
62 | SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,\r |
63 | ISzAlloc *alloc, ISzAlloc *allocBig);\r |
64 | SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,\r |
65 | UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);\r |
66 | SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,\r |
67 | Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);\r |
68 | const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);\r |
69 | void LzmaEnc_Finish(CLzmaEncHandle pp);\r |
70 | void LzmaEnc_SaveState(CLzmaEncHandle pp);\r |
71 | void LzmaEnc_RestoreState(CLzmaEncHandle pp);\r |
72 | \r |
73 | \r |
74 | static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,\r |
75 | size_t *packSizeRes, ISeqOutStream *outStream)\r |
76 | {\r |
77 | size_t packSizeLimit = *packSizeRes;\r |
78 | size_t packSize = packSizeLimit;\r |
79 | UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;\r |
80 | unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);\r |
81 | Bool useCopyBlock;\r |
82 | SRes res;\r |
83 | \r |
84 | *packSizeRes = 0;\r |
85 | if (packSize < lzHeaderSize)\r |
86 | return SZ_ERROR_OUTPUT_EOF;\r |
87 | packSize -= lzHeaderSize;\r |
88 | \r |
89 | LzmaEnc_SaveState(p->enc);\r |
90 | res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,\r |
91 | outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);\r |
92 | \r |
93 | PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));\r |
94 | \r |
95 | if (unpackSize == 0)\r |
96 | return res;\r |
97 | \r |
98 | if (res == SZ_OK)\r |
99 | useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));\r |
100 | else\r |
101 | {\r |
102 | if (res != SZ_ERROR_OUTPUT_EOF)\r |
103 | return res;\r |
104 | res = SZ_OK;\r |
105 | useCopyBlock = True;\r |
106 | }\r |
107 | \r |
108 | if (useCopyBlock)\r |
109 | {\r |
110 | size_t destPos = 0;\r |
111 | PRF(printf("################# COPY "));\r |
112 | \r |
113 | while (unpackSize > 0)\r |
114 | {\r |
115 | UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;\r |
116 | if (packSizeLimit - destPos < u + 3)\r |
117 | return SZ_ERROR_OUTPUT_EOF;\r |
118 | outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);\r |
119 | outBuf[destPos++] = (Byte)((u - 1) >> 8);\r |
120 | outBuf[destPos++] = (Byte)(u - 1);\r |
121 | memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);\r |
122 | unpackSize -= u;\r |
123 | destPos += u;\r |
124 | p->srcPos += u;\r |
125 | \r |
126 | if (outStream)\r |
127 | {\r |
128 | *packSizeRes += destPos;\r |
129 | if (outStream->Write(outStream, outBuf, destPos) != destPos)\r |
130 | return SZ_ERROR_WRITE;\r |
131 | destPos = 0;\r |
132 | }\r |
133 | else\r |
134 | *packSizeRes = destPos;\r |
135 | /* needInitState = True; */\r |
136 | }\r |
137 | \r |
138 | LzmaEnc_RestoreState(p->enc);\r |
139 | return SZ_OK;\r |
140 | }\r |
141 | \r |
142 | {\r |
143 | size_t destPos = 0;\r |
144 | UInt32 u = unpackSize - 1;\r |
145 | UInt32 pm = (UInt32)(packSize - 1);\r |
146 | unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);\r |
147 | \r |
148 | PRF(printf(" "));\r |
149 | \r |
150 | outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));\r |
151 | outBuf[destPos++] = (Byte)(u >> 8);\r |
152 | outBuf[destPos++] = (Byte)u;\r |
153 | outBuf[destPos++] = (Byte)(pm >> 8);\r |
154 | outBuf[destPos++] = (Byte)pm;\r |
155 | \r |
156 | if (p->needInitProp)\r |
157 | outBuf[destPos++] = p->props;\r |
158 | \r |
159 | p->needInitProp = False;\r |
160 | p->needInitState = False;\r |
161 | destPos += packSize;\r |
162 | p->srcPos += unpackSize;\r |
163 | \r |
164 | if (outStream)\r |
165 | if (outStream->Write(outStream, outBuf, destPos) != destPos)\r |
166 | return SZ_ERROR_WRITE;\r |
167 | \r |
168 | *packSizeRes = destPos;\r |
169 | return SZ_OK;\r |
170 | }\r |
171 | }\r |
172 | \r |
173 | \r |
174 | /* ---------- Lzma2 Props ---------- */\r |
175 | \r |
176 | void Lzma2EncProps_Init(CLzma2EncProps *p)\r |
177 | {\r |
178 | LzmaEncProps_Init(&p->lzmaProps);\r |
179 | p->numTotalThreads = -1;\r |
180 | p->numBlockThreads = -1;\r |
181 | p->blockSize = 0;\r |
182 | }\r |
183 | \r |
184 | void Lzma2EncProps_Normalize(CLzma2EncProps *p)\r |
185 | {\r |
186 | int t1, t1n, t2, t3;\r |
187 | {\r |
188 | CLzmaEncProps lzmaProps = p->lzmaProps;\r |
189 | LzmaEncProps_Normalize(&lzmaProps);\r |
190 | t1n = lzmaProps.numThreads;\r |
191 | }\r |
192 | \r |
193 | t1 = p->lzmaProps.numThreads;\r |
194 | t2 = p->numBlockThreads;\r |
195 | t3 = p->numTotalThreads;\r |
196 | \r |
197 | if (t2 > NUM_MT_CODER_THREADS_MAX)\r |
198 | t2 = NUM_MT_CODER_THREADS_MAX;\r |
199 | \r |
200 | if (t3 <= 0)\r |
201 | {\r |
202 | if (t2 <= 0)\r |
203 | t2 = 1;\r |
204 | t3 = t1n * t2;\r |
205 | }\r |
206 | else if (t2 <= 0)\r |
207 | {\r |
208 | t2 = t3 / t1n;\r |
209 | if (t2 == 0)\r |
210 | {\r |
211 | t1 = 1;\r |
212 | t2 = t3;\r |
213 | }\r |
214 | if (t2 > NUM_MT_CODER_THREADS_MAX)\r |
215 | t2 = NUM_MT_CODER_THREADS_MAX;\r |
216 | }\r |
217 | else if (t1 <= 0)\r |
218 | {\r |
219 | t1 = t3 / t2;\r |
220 | if (t1 == 0)\r |
221 | t1 = 1;\r |
222 | }\r |
223 | else\r |
224 | t3 = t1n * t2;\r |
225 | \r |
226 | p->lzmaProps.numThreads = t1;\r |
227 | \r |
228 | LzmaEncProps_Normalize(&p->lzmaProps);\r |
229 | \r |
230 | t1 = p->lzmaProps.numThreads;\r |
231 | \r |
232 | if (p->blockSize == 0)\r |
233 | {\r |
234 | UInt32 dictSize = p->lzmaProps.dictSize;\r |
235 | UInt64 blockSize = (UInt64)dictSize << 2;\r |
236 | const UInt32 kMinSize = (UInt32)1 << 20;\r |
237 | const UInt32 kMaxSize = (UInt32)1 << 28;\r |
238 | if (blockSize < kMinSize) blockSize = kMinSize;\r |
239 | if (blockSize > kMaxSize) blockSize = kMaxSize;\r |
240 | if (blockSize < dictSize) blockSize = dictSize;\r |
241 | p->blockSize = (size_t)blockSize;\r |
242 | }\r |
243 | \r |
244 | if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)\r |
245 | {\r |
246 | UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;\r |
247 | if (temp > p->lzmaProps.reduceSize)\r |
248 | {\r |
249 | UInt64 numBlocks = temp / p->blockSize;\r |
250 | if (numBlocks < (unsigned)t2)\r |
251 | {\r |
252 | t2 = (unsigned)numBlocks;\r |
253 | if (t2 == 0)\r |
254 | t2 = 1;\r |
255 | t3 = t1 * t2;\r |
256 | }\r |
257 | }\r |
258 | }\r |
259 | \r |
260 | p->numBlockThreads = t2;\r |
261 | p->numTotalThreads = t3;\r |
262 | }\r |
263 | \r |
264 | \r |
265 | static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)\r |
266 | {\r |
267 | return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;\r |
268 | }\r |
269 | \r |
270 | \r |
271 | /* ---------- Lzma2 ---------- */\r |
272 | \r |
273 | typedef struct\r |
274 | {\r |
275 | Byte propEncoded;\r |
276 | CLzma2EncProps props;\r |
277 | \r |
278 | Byte *outBuf;\r |
279 | \r |
280 | ISzAlloc *alloc;\r |
281 | ISzAlloc *allocBig;\r |
282 | \r |
283 | CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];\r |
284 | \r |
285 | #ifndef _7ZIP_ST\r |
286 | CMtCoder mtCoder;\r |
287 | #endif\r |
288 | \r |
289 | } CLzma2Enc;\r |
290 | \r |
291 | \r |
292 | /* ---------- Lzma2EncThread ---------- */\r |
293 | \r |
294 | static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,\r |
295 | ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)\r |
296 | {\r |
297 | UInt64 packTotal = 0;\r |
298 | SRes res = SZ_OK;\r |
299 | \r |
300 | if (!mainEncoder->outBuf)\r |
301 | {\r |
302 | mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);\r |
303 | if (!mainEncoder->outBuf)\r |
304 | return SZ_ERROR_MEM;\r |
305 | }\r |
306 | \r |
307 | RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));\r |
308 | RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,\r |
309 | mainEncoder->alloc, mainEncoder->allocBig));\r |
310 | \r |
311 | for (;;)\r |
312 | {\r |
313 | size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;\r |
314 | res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);\r |
315 | if (res != SZ_OK)\r |
316 | break;\r |
317 | packTotal += packSize;\r |
318 | res = Progress(progress, p->srcPos, packTotal);\r |
319 | if (res != SZ_OK)\r |
320 | break;\r |
321 | if (packSize == 0)\r |
322 | break;\r |
323 | }\r |
324 | \r |
325 | LzmaEnc_Finish(p->enc);\r |
326 | \r |
327 | if (res == SZ_OK)\r |
328 | {\r |
329 | Byte b = 0;\r |
330 | if (outStream->Write(outStream, &b, 1) != 1)\r |
331 | return SZ_ERROR_WRITE;\r |
332 | }\r |
333 | \r |
334 | return res;\r |
335 | }\r |
336 | \r |
337 | \r |
338 | #ifndef _7ZIP_ST\r |
339 | \r |
340 | typedef struct\r |
341 | {\r |
342 | IMtCoderCallback funcTable;\r |
343 | CLzma2Enc *lzma2Enc;\r |
344 | } CMtCallbackImp;\r |
345 | \r |
346 | static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,\r |
347 | const Byte *src, size_t srcSize, int finished)\r |
348 | {\r |
349 | CMtCallbackImp *imp = (CMtCallbackImp *)pp;\r |
350 | CLzma2Enc *mainEncoder = imp->lzma2Enc;\r |
351 | CLzma2EncInt *p = &mainEncoder->coders[index];\r |
352 | \r |
353 | SRes res = SZ_OK;\r |
354 | {\r |
355 | size_t destLim = *destSize;\r |
356 | *destSize = 0;\r |
357 | \r |
358 | if (srcSize != 0)\r |
359 | {\r |
360 | RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));\r |
361 | \r |
362 | RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,\r |
363 | mainEncoder->alloc, mainEncoder->allocBig));\r |
364 | \r |
365 | while (p->srcPos < srcSize)\r |
366 | {\r |
367 | size_t packSize = destLim - *destSize;\r |
368 | res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);\r |
369 | if (res != SZ_OK)\r |
370 | break;\r |
371 | *destSize += packSize;\r |
372 | \r |
373 | if (packSize == 0)\r |
374 | {\r |
375 | res = SZ_ERROR_FAIL;\r |
376 | break;\r |
377 | }\r |
378 | \r |
379 | if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)\r |
380 | {\r |
381 | res = SZ_ERROR_PROGRESS;\r |
382 | break;\r |
383 | }\r |
384 | }\r |
385 | \r |
386 | LzmaEnc_Finish(p->enc);\r |
387 | if (res != SZ_OK)\r |
388 | return res;\r |
389 | }\r |
390 | \r |
391 | if (finished)\r |
392 | {\r |
393 | if (*destSize == destLim)\r |
394 | return SZ_ERROR_OUTPUT_EOF;\r |
395 | dest[(*destSize)++] = 0;\r |
396 | }\r |
397 | }\r |
398 | return res;\r |
399 | }\r |
400 | \r |
401 | #endif\r |
402 | \r |
403 | \r |
404 | /* ---------- Lzma2Enc ---------- */\r |
405 | \r |
406 | CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)\r |
407 | {\r |
408 | CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));\r |
409 | if (!p)\r |
410 | return NULL;\r |
411 | Lzma2EncProps_Init(&p->props);\r |
412 | Lzma2EncProps_Normalize(&p->props);\r |
413 | p->outBuf = 0;\r |
414 | p->alloc = alloc;\r |
415 | p->allocBig = allocBig;\r |
416 | {\r |
417 | unsigned i;\r |
418 | for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)\r |
419 | p->coders[i].enc = 0;\r |
420 | }\r |
421 | \r |
422 | #ifndef _7ZIP_ST\r |
423 | MtCoder_Construct(&p->mtCoder);\r |
424 | #endif\r |
425 | \r |
426 | return p;\r |
427 | }\r |
428 | \r |
429 | void Lzma2Enc_Destroy(CLzma2EncHandle pp)\r |
430 | {\r |
431 | CLzma2Enc *p = (CLzma2Enc *)pp;\r |
432 | unsigned i;\r |
433 | for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)\r |
434 | {\r |
435 | CLzma2EncInt *t = &p->coders[i];\r |
436 | if (t->enc)\r |
437 | {\r |
438 | LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);\r |
439 | t->enc = 0;\r |
440 | }\r |
441 | }\r |
442 | \r |
443 | #ifndef _7ZIP_ST\r |
444 | MtCoder_Destruct(&p->mtCoder);\r |
445 | #endif\r |
446 | \r |
447 | IAlloc_Free(p->alloc, p->outBuf);\r |
448 | IAlloc_Free(p->alloc, pp);\r |
449 | }\r |
450 | \r |
451 | SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)\r |
452 | {\r |
453 | CLzma2Enc *p = (CLzma2Enc *)pp;\r |
454 | CLzmaEncProps lzmaProps = props->lzmaProps;\r |
455 | LzmaEncProps_Normalize(&lzmaProps);\r |
456 | if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)\r |
457 | return SZ_ERROR_PARAM;\r |
458 | p->props = *props;\r |
459 | Lzma2EncProps_Normalize(&p->props);\r |
460 | return SZ_OK;\r |
461 | }\r |
462 | \r |
463 | Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)\r |
464 | {\r |
465 | CLzma2Enc *p = (CLzma2Enc *)pp;\r |
466 | unsigned i;\r |
467 | UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);\r |
468 | for (i = 0; i < 40; i++)\r |
469 | if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))\r |
470 | break;\r |
471 | return (Byte)i;\r |
472 | }\r |
473 | \r |
474 | SRes Lzma2Enc_Encode(CLzma2EncHandle pp,\r |
475 | ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)\r |
476 | {\r |
477 | CLzma2Enc *p = (CLzma2Enc *)pp;\r |
478 | int i;\r |
479 | \r |
480 | for (i = 0; i < p->props.numBlockThreads; i++)\r |
481 | {\r |
482 | CLzma2EncInt *t = &p->coders[(unsigned)i];\r |
483 | if (!t->enc)\r |
484 | {\r |
485 | t->enc = LzmaEnc_Create(p->alloc);\r |
486 | if (!t->enc)\r |
487 | return SZ_ERROR_MEM;\r |
488 | }\r |
489 | }\r |
490 | \r |
491 | #ifndef _7ZIP_ST\r |
492 | if (p->props.numBlockThreads > 1)\r |
493 | {\r |
494 | CMtCallbackImp mtCallback;\r |
495 | \r |
496 | mtCallback.funcTable.Code = MtCallbackImp_Code;\r |
497 | mtCallback.lzma2Enc = p;\r |
498 | \r |
499 | p->mtCoder.progress = progress;\r |
500 | p->mtCoder.inStream = inStream;\r |
501 | p->mtCoder.outStream = outStream;\r |
502 | p->mtCoder.alloc = p->alloc;\r |
503 | p->mtCoder.mtCallback = &mtCallback.funcTable;\r |
504 | \r |
505 | p->mtCoder.blockSize = p->props.blockSize;\r |
506 | p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;\r |
507 | if (p->mtCoder.destBlockSize < p->props.blockSize)\r |
508 | {\r |
509 | p->mtCoder.destBlockSize = (size_t)0 - 1;\r |
510 | if (p->mtCoder.destBlockSize < p->props.blockSize)\r |
511 | return SZ_ERROR_FAIL;\r |
512 | }\r |
513 | p->mtCoder.numThreads = p->props.numBlockThreads;\r |
514 | \r |
515 | return MtCoder_Code(&p->mtCoder);\r |
516 | }\r |
517 | #endif\r |
518 | \r |
519 | return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);\r |
520 | }\r |