1 /* XzEnc.c -- Xz Encode
\r
2 2015-09-16 : Igor Pavlov : Public domain */
\r
12 #include "CpuArch.h"
\r
15 #include "Bcj3Enc.c"
\r
22 #define XzBlock_ClearFlags(p) (p)->flags = 0;
\r
23 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
\r
24 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
\r
25 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
\r
27 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
\r
29 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
\r
32 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
\r
34 *crc = CrcUpdate(*crc, buf, size);
\r
35 return WriteBytes(s, buf, size);
\r
38 static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
\r
41 Byte header[XZ_STREAM_HEADER_SIZE];
\r
42 memcpy(header, XZ_SIG, XZ_SIG_SIZE);
\r
43 header[XZ_SIG_SIZE] = (Byte)(f >> 8);
\r
44 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
\r
45 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
\r
46 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
\r
47 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
\r
51 static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
\r
53 Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
\r
56 unsigned numFilters, i;
\r
57 header[pos++] = p->flags;
\r
59 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
\r
60 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
\r
61 numFilters = XzBlock_GetNumFilters(p);
\r
63 for (i = 0; i < numFilters; i++)
\r
65 const CXzFilter *f = &p->filters[i];
\r
66 pos += Xz_WriteVarInt(header + pos, f->id);
\r
67 pos += Xz_WriteVarInt(header + pos, f->propsSize);
\r
68 memcpy(header + pos, f->props, f->propsSize);
\r
69 pos += f->propsSize;
\r
72 while ((pos & 3) != 0)
\r
75 header[0] = (Byte)(pos >> 2);
\r
76 SetUi32(header + pos, CrcCalc(header, pos));
\r
77 return WriteBytes(s, header, pos + 4);
\r
81 static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
\r
86 UInt32 crc = CRC_INIT_VAL;
\r
87 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
\r
92 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
\r
94 for (i = 0; i < p->numBlocks; i++)
\r
96 const CXzBlockSizes *block = &p->blocks[i];
\r
97 pos = Xz_WriteVarInt(buf, block->totalSize);
\r
98 pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
\r
100 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
\r
103 pos = ((unsigned)globalPos & 3);
\r
107 buf[0] = buf[1] = buf[2] = 0;
\r
108 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
\r
109 globalPos += 4 - pos;
\r
112 SetUi32(buf, CRC_GET_DIGEST(crc));
\r
113 RINOK(WriteBytes(s, buf, 4));
\r
119 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
\r
120 SetUi32(buf + 4, indexSize);
\r
121 buf[8] = (Byte)(p->flags >> 8);
\r
122 buf[9] = (Byte)(p->flags & 0xFF);
\r
123 SetUi32(buf, CrcCalc(buf + 4, 6));
\r
124 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
\r
125 return WriteBytes(s, buf, 12);
\r
130 static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
\r
132 if (!p->blocks || p->numBlocksAllocated == p->numBlocks)
\r
134 size_t num = p->numBlocks * 2 + 1;
\r
135 size_t newSize = sizeof(CXzBlockSizes) * num;
\r
136 CXzBlockSizes *blocks;
\r
137 if (newSize / sizeof(CXzBlockSizes) != num)
\r
138 return SZ_ERROR_MEM;
\r
139 blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);
\r
141 return SZ_ERROR_MEM;
\r
142 if (p->numBlocks != 0)
\r
144 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
\r
145 alloc->Free(alloc, p->blocks);
\r
147 p->blocks = blocks;
\r
148 p->numBlocksAllocated = num;
\r
151 CXzBlockSizes *block = &p->blocks[p->numBlocks++];
\r
152 block->unpackSize = unpackSize;
\r
153 block->totalSize = totalSize;
\r
159 /* ---------- CSeqCheckInStream ---------- */
\r
164 ISeqInStream *realStream;
\r
167 } CSeqCheckInStream;
\r
169 static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode)
\r
172 XzCheck_Init(&p->check, mode);
\r
175 static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
\r
177 XzCheck_Final(&p->check, digest);
\r
180 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
\r
182 CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
\r
183 SRes res = p->realStream->Read(p->realStream, data, size);
\r
184 XzCheck_Update(&p->check, data, *size);
\r
185 p->processed += *size;
\r
190 /* ---------- CSeqSizeOutStream ---------- */
\r
195 ISeqOutStream *realStream;
\r
197 } CSeqSizeOutStream;
\r
199 static size_t MyWrite(void *pp, const void *data, size_t size)
\r
201 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
\r
202 size = p->realStream->Write(p->realStream, data, size);
\r
203 p->processed += size;
\r
208 /* ---------- CSeqInFilter ---------- */
\r
210 #define FILTER_BUF_SIZE (1 << 20)
\r
215 ISeqInStream *realStream;
\r
216 IStateCoder StateCoder;
\r
220 int srcWasFinished;
\r
223 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
\r
225 CSeqInFilter *p = (CSeqInFilter *)pp;
\r
226 size_t sizeOriginal = *size;
\r
227 if (sizeOriginal == 0)
\r
233 if (!p->srcWasFinished && p->curPos == p->endPos)
\r
236 p->endPos = FILTER_BUF_SIZE;
\r
237 RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));
\r
238 if (p->endPos == 0)
\r
239 p->srcWasFinished = 1;
\r
242 SizeT srcLen = p->endPos - p->curPos;
\r
245 *size = sizeOriginal;
\r
246 res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,
\r
247 p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);
\r
248 p->curPos += srcLen;
\r
249 if (*size != 0 || srcLen == 0 || res != 0)
\r
255 static void SeqInFilter_Construct(CSeqInFilter *p)
\r
258 p->p.Read = SeqInFilter_Read;
\r
261 static void SeqInFilter_Free(CSeqInFilter *p)
\r
265 g_Alloc.Free(&g_Alloc, p->buf);
\r
270 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);
\r
272 static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)
\r
276 p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);
\r
278 return SZ_ERROR_MEM;
\r
280 p->curPos = p->endPos = 0;
\r
281 p->srcWasFinished = 0;
\r
282 RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));
\r
283 RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));
\r
284 p->StateCoder.Init(p->StateCoder.p);
\r
289 /* ---------- CSbEncInStream ---------- */
\r
291 #ifdef USE_SUBBLOCK
\r
296 ISeqInStream *inStream;
\r
300 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
\r
302 CSbEncInStream *p = (CSbEncInStream *)pp;
\r
303 size_t sizeOriginal = *size;
\r
304 if (sizeOriginal == 0)
\r
309 if (p->enc.needRead && !p->enc.readWasFinished)
\r
311 size_t processed = p->enc.needReadSizeMax;
\r
312 RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));
\r
313 p->enc.readPos += processed;
\r
314 if (processed == 0)
\r
316 p->enc.readWasFinished = True;
\r
317 p->enc.isFinalFinished = True;
\r
319 p->enc.needRead = False;
\r
322 *size = sizeOriginal;
\r
323 RINOK(SbEnc_Read(&p->enc, data, size));
\r
324 if (*size != 0 || !p->enc.needRead)
\r
329 void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)
\r
331 SbEnc_Construct(&p->enc, alloc);
\r
332 p->p.Read = SbEncInStream_Read;
\r
335 SRes SbEncInStream_Init(CSbEncInStream *p)
\r
337 return SbEnc_Init(&p->enc);
\r
340 void SbEncInStream_Free(CSbEncInStream *p)
\r
342 SbEnc_Free(&p->enc);
\r
350 CLzma2EncHandle lzma2;
\r
351 #ifdef USE_SUBBLOCK
\r
354 CSeqInFilter filter;
\r
356 ISzAlloc *bigAlloc;
\r
357 } CLzma2WithFilters;
\r
360 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
\r
363 p->bigAlloc = bigAlloc;
\r
365 #ifdef USE_SUBBLOCK
\r
366 SbEncInStream_Construct(&p->sb, alloc);
\r
368 SeqInFilter_Construct(&p->filter);
\r
371 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
\r
373 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
\r
375 return SZ_ERROR_MEM;
\r
379 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
\r
381 SeqInFilter_Free(&p->filter);
\r
382 #ifdef USE_SUBBLOCK
\r
383 SbEncInStream_Free(&p->sb);
\r
387 Lzma2Enc_Destroy(p->lzma2);
\r
393 void XzProps_Init(CXzProps *p)
\r
395 p->lzma2Props = NULL;
\r
396 p->filterProps = NULL;
\r
397 p->checkId = XZ_CHECK_CRC32;
\r
400 void XzFilterProps_Init(CXzFilterProps *p)
\r
405 p->ipDefined = False;
\r
409 static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,
\r
410 ISeqOutStream *outStream, ISeqInStream *inStream,
\r
411 const CXzProps *props, ICompressProgress *progress)
\r
413 xz->flags = (Byte)props->checkId;
\r
415 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));
\r
416 RINOK(Xz_WriteHeader(xz->flags, outStream));
\r
419 CSeqCheckInStream checkInStream;
\r
420 CSeqSizeOutStream seqSizeOutStream;
\r
422 unsigned filterIndex = 0;
\r
423 CXzFilter *filter = NULL;
\r
424 const CXzFilterProps *fp = props->filterProps;
\r
426 XzBlock_ClearFlags(&block);
\r
427 XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));
\r
431 filter = &block.filters[filterIndex++];
\r
432 filter->id = fp->id;
\r
433 filter->propsSize = 0;
\r
435 if (fp->id == XZ_ID_Delta)
\r
437 filter->props[0] = (Byte)(fp->delta - 1);
\r
438 filter->propsSize = 1;
\r
440 else if (fp->ipDefined)
\r
442 SetUi32(filter->props, fp->ip);
\r
443 filter->propsSize = 4;
\r
448 CXzFilter *f = &block.filters[filterIndex++];
\r
449 f->id = XZ_ID_LZMA2;
\r
451 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
\r
454 seqSizeOutStream.p.Write = MyWrite;
\r
455 seqSizeOutStream.realStream = outStream;
\r
456 seqSizeOutStream.processed = 0;
\r
458 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
\r
460 checkInStream.p.Read = SeqCheckInStream_Read;
\r
461 checkInStream.realStream = inStream;
\r
462 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
\r
466 #ifdef USE_SUBBLOCK
\r
467 if (fp->id == XZ_ID_Subblock)
\r
469 lzmaf->sb.inStream = &checkInStream.p;
\r
470 RINOK(SbEncInStream_Init(&lzmaf->sb));
\r
475 lzmaf->filter.realStream = &checkInStream.p;
\r
476 RINOK(SeqInFilter_Init(&lzmaf->filter, filter));
\r
481 UInt64 packPos = seqSizeOutStream.processed;
\r
483 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
\r
485 #ifdef USE_SUBBLOCK
\r
486 (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:
\r
493 block.unpackSize = checkInStream.processed;
\r
494 block.packSize = seqSizeOutStream.processed - packPos;
\r
498 unsigned padSize = 0;
\r
500 while ((((unsigned)block.packSize + padSize) & 3) != 0)
\r
501 buf[padSize++] = 0;
\r
502 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
\r
503 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
\r
504 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
\r
507 return Xz_WriteFooter(xz, outStream);
\r
511 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
\r
512 const CXzProps *props, ICompressProgress *progress)
\r
516 CLzma2WithFilters lzmaf;
\r
518 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
\r
519 res = Lzma2WithFilters_Create(&lzmaf);
\r
521 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);
\r
522 Lzma2WithFilters_Free(&lzmaf);
\r
523 Xz_Free(&xz, &g_Alloc);
\r
528 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
\r
533 res = Xz_WriteHeader(xz.flags, outStream);
\r
535 res = Xz_WriteFooter(&xz, outStream);
\r
536 Xz_Free(&xz, &g_Alloc);
\r