1 /* Lzma2Enc.c -- LZMA2 Encoder
\r
2 2015-10-04 : Igor Pavlov : Public domain */
\r
6 /* #include <stdio.h> */
\r
9 /* #define _7ZIP_ST */
\r
11 #include "Lzma2Enc.h"
\r
14 #include "MtCoder.h"
\r
16 #define NUM_MT_CODER_THREADS_MAX 1
\r
19 #define LZMA2_CONTROL_LZMA (1 << 7)
\r
20 #define LZMA2_CONTROL_COPY_NO_RESET 2
\r
21 #define LZMA2_CONTROL_COPY_RESET_DIC 1
\r
22 #define LZMA2_CONTROL_EOF 0
\r
24 #define LZMA2_LCLP_MAX 4
\r
26 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
\r
28 #define LZMA2_PACK_SIZE_MAX (1 << 16)
\r
29 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
\r
30 #define LZMA2_UNPACK_SIZE_MAX (1 << 21)
\r
31 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX
\r
33 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)
\r
36 #define PRF(x) /* x */
\r
38 /* ---------- CLzma2EncInt ---------- */
\r
49 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
\r
51 Byte propsEncoded[LZMA_PROPS_SIZE];
\r
52 SizeT propsSize = LZMA_PROPS_SIZE;
\r
53 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
\r
54 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
\r
56 p->props = propsEncoded[0];
\r
57 p->needInitState = True;
\r
58 p->needInitProp = True;
\r
62 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
\r
63 ISzAlloc *alloc, ISzAlloc *allocBig);
\r
64 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
\r
65 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
\r
66 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
\r
67 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
\r
68 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
\r
69 void LzmaEnc_Finish(CLzmaEncHandle pp);
\r
70 void LzmaEnc_SaveState(CLzmaEncHandle pp);
\r
71 void LzmaEnc_RestoreState(CLzmaEncHandle pp);
\r
74 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
\r
75 size_t *packSizeRes, ISeqOutStream *outStream)
\r
77 size_t packSizeLimit = *packSizeRes;
\r
78 size_t packSize = packSizeLimit;
\r
79 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
\r
80 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
\r
85 if (packSize < lzHeaderSize)
\r
86 return SZ_ERROR_OUTPUT_EOF;
\r
87 packSize -= lzHeaderSize;
\r
89 LzmaEnc_SaveState(p->enc);
\r
90 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
\r
91 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
\r
93 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));
\r
95 if (unpackSize == 0)
\r
99 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
\r
102 if (res != SZ_ERROR_OUTPUT_EOF)
\r
105 useCopyBlock = True;
\r
110 size_t destPos = 0;
\r
111 PRF(printf("################# COPY "));
\r
113 while (unpackSize > 0)
\r
115 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
\r
116 if (packSizeLimit - destPos < u + 3)
\r
117 return SZ_ERROR_OUTPUT_EOF;
\r
118 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
\r
119 outBuf[destPos++] = (Byte)((u - 1) >> 8);
\r
120 outBuf[destPos++] = (Byte)(u - 1);
\r
121 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
\r
128 *packSizeRes += destPos;
\r
129 if (outStream->Write(outStream, outBuf, destPos) != destPos)
\r
130 return SZ_ERROR_WRITE;
\r
134 *packSizeRes = destPos;
\r
135 /* needInitState = True; */
\r
138 LzmaEnc_RestoreState(p->enc);
\r
143 size_t destPos = 0;
\r
144 UInt32 u = unpackSize - 1;
\r
145 UInt32 pm = (UInt32)(packSize - 1);
\r
146 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);
\r
150 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
\r
151 outBuf[destPos++] = (Byte)(u >> 8);
\r
152 outBuf[destPos++] = (Byte)u;
\r
153 outBuf[destPos++] = (Byte)(pm >> 8);
\r
154 outBuf[destPos++] = (Byte)pm;
\r
156 if (p->needInitProp)
\r
157 outBuf[destPos++] = p->props;
\r
159 p->needInitProp = False;
\r
160 p->needInitState = False;
\r
161 destPos += packSize;
\r
162 p->srcPos += unpackSize;
\r
165 if (outStream->Write(outStream, outBuf, destPos) != destPos)
\r
166 return SZ_ERROR_WRITE;
\r
168 *packSizeRes = destPos;
\r
174 /* ---------- Lzma2 Props ---------- */
\r
176 void Lzma2EncProps_Init(CLzma2EncProps *p)
\r
178 LzmaEncProps_Init(&p->lzmaProps);
\r
179 p->numTotalThreads = -1;
\r
180 p->numBlockThreads = -1;
\r
184 void Lzma2EncProps_Normalize(CLzma2EncProps *p)
\r
186 int t1, t1n, t2, t3;
\r
188 CLzmaEncProps lzmaProps = p->lzmaProps;
\r
189 LzmaEncProps_Normalize(&lzmaProps);
\r
190 t1n = lzmaProps.numThreads;
\r
193 t1 = p->lzmaProps.numThreads;
\r
194 t2 = p->numBlockThreads;
\r
195 t3 = p->numTotalThreads;
\r
197 if (t2 > NUM_MT_CODER_THREADS_MAX)
\r
198 t2 = NUM_MT_CODER_THREADS_MAX;
\r
214 if (t2 > NUM_MT_CODER_THREADS_MAX)
\r
215 t2 = NUM_MT_CODER_THREADS_MAX;
\r
226 p->lzmaProps.numThreads = t1;
\r
228 LzmaEncProps_Normalize(&p->lzmaProps);
\r
230 t1 = p->lzmaProps.numThreads;
\r
232 if (p->blockSize == 0)
\r
234 UInt32 dictSize = p->lzmaProps.dictSize;
\r
235 UInt64 blockSize = (UInt64)dictSize << 2;
\r
236 const UInt32 kMinSize = (UInt32)1 << 20;
\r
237 const UInt32 kMaxSize = (UInt32)1 << 28;
\r
238 if (blockSize < kMinSize) blockSize = kMinSize;
\r
239 if (blockSize > kMaxSize) blockSize = kMaxSize;
\r
240 if (blockSize < dictSize) blockSize = dictSize;
\r
241 p->blockSize = (size_t)blockSize;
\r
244 if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
\r
246 UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
\r
247 if (temp > p->lzmaProps.reduceSize)
\r
249 UInt64 numBlocks = temp / p->blockSize;
\r
250 if (numBlocks < (unsigned)t2)
\r
252 t2 = (unsigned)numBlocks;
\r
260 p->numBlockThreads = t2;
\r
261 p->numTotalThreads = t3;
\r
265 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
\r
267 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
\r
271 /* ---------- Lzma2 ---------- */
\r
276 CLzma2EncProps props;
\r
281 ISzAlloc *allocBig;
\r
283 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
\r
292 /* ---------- Lzma2EncThread ---------- */
\r
294 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
\r
295 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
\r
297 UInt64 packTotal = 0;
\r
300 if (!mainEncoder->outBuf)
\r
302 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
\r
303 if (!mainEncoder->outBuf)
\r
304 return SZ_ERROR_MEM;
\r
307 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
\r
308 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
\r
309 mainEncoder->alloc, mainEncoder->allocBig));
\r
313 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
\r
314 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
\r
317 packTotal += packSize;
\r
318 res = Progress(progress, p->srcPos, packTotal);
\r
325 LzmaEnc_Finish(p->enc);
\r
330 if (outStream->Write(outStream, &b, 1) != 1)
\r
331 return SZ_ERROR_WRITE;
\r
342 IMtCoderCallback funcTable;
\r
343 CLzma2Enc *lzma2Enc;
\r
346 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
\r
347 const Byte *src, size_t srcSize, int finished)
\r
349 CMtCallbackImp *imp = (CMtCallbackImp *)pp;
\r
350 CLzma2Enc *mainEncoder = imp->lzma2Enc;
\r
351 CLzma2EncInt *p = &mainEncoder->coders[index];
\r
355 size_t destLim = *destSize;
\r
360 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
\r
362 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
\r
363 mainEncoder->alloc, mainEncoder->allocBig));
\r
365 while (p->srcPos < srcSize)
\r
367 size_t packSize = destLim - *destSize;
\r
368 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
\r
371 *destSize += packSize;
\r
375 res = SZ_ERROR_FAIL;
\r
379 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
\r
381 res = SZ_ERROR_PROGRESS;
\r
386 LzmaEnc_Finish(p->enc);
\r
393 if (*destSize == destLim)
\r
394 return SZ_ERROR_OUTPUT_EOF;
\r
395 dest[(*destSize)++] = 0;
\r
404 /* ---------- Lzma2Enc ---------- */
\r
406 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
\r
408 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
\r
411 Lzma2EncProps_Init(&p->props);
\r
412 Lzma2EncProps_Normalize(&p->props);
\r
415 p->allocBig = allocBig;
\r
418 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
\r
419 p->coders[i].enc = 0;
\r
423 MtCoder_Construct(&p->mtCoder);
\r
429 void Lzma2Enc_Destroy(CLzma2EncHandle pp)
\r
431 CLzma2Enc *p = (CLzma2Enc *)pp;
\r
433 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
\r
435 CLzma2EncInt *t = &p->coders[i];
\r
438 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
\r
444 MtCoder_Destruct(&p->mtCoder);
\r
447 IAlloc_Free(p->alloc, p->outBuf);
\r
448 IAlloc_Free(p->alloc, pp);
\r
451 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
\r
453 CLzma2Enc *p = (CLzma2Enc *)pp;
\r
454 CLzmaEncProps lzmaProps = props->lzmaProps;
\r
455 LzmaEncProps_Normalize(&lzmaProps);
\r
456 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
\r
457 return SZ_ERROR_PARAM;
\r
459 Lzma2EncProps_Normalize(&p->props);
\r
463 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
\r
465 CLzma2Enc *p = (CLzma2Enc *)pp;
\r
467 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
\r
468 for (i = 0; i < 40; i++)
\r
469 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
\r
474 SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
\r
475 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
\r
477 CLzma2Enc *p = (CLzma2Enc *)pp;
\r
480 for (i = 0; i < p->props.numBlockThreads; i++)
\r
482 CLzma2EncInt *t = &p->coders[(unsigned)i];
\r
485 t->enc = LzmaEnc_Create(p->alloc);
\r
487 return SZ_ERROR_MEM;
\r
492 if (p->props.numBlockThreads > 1)
\r
494 CMtCallbackImp mtCallback;
\r
496 mtCallback.funcTable.Code = MtCallbackImp_Code;
\r
497 mtCallback.lzma2Enc = p;
\r
499 p->mtCoder.progress = progress;
\r
500 p->mtCoder.inStream = inStream;
\r
501 p->mtCoder.outStream = outStream;
\r
502 p->mtCoder.alloc = p->alloc;
\r
503 p->mtCoder.mtCallback = &mtCallback.funcTable;
\r
505 p->mtCoder.blockSize = p->props.blockSize;
\r
506 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
\r
507 if (p->mtCoder.destBlockSize < p->props.blockSize)
\r
509 p->mtCoder.destBlockSize = (size_t)0 - 1;
\r
510 if (p->mtCoder.destBlockSize < p->props.blockSize)
\r
511 return SZ_ERROR_FAIL;
\r
513 p->mtCoder.numThreads = p->props.numBlockThreads;
\r
515 return MtCoder_Code(&p->mtCoder);
\r
519 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
\r