ce188d4d |
1 | /* XzEnc.c -- Xz Encode\r |
2 | 2015-09-16 : Igor Pavlov : Public domain */\r |
3 | \r |
4 | #include "Precomp.h"\r |
5 | \r |
6 | #include <stdlib.h>\r |
7 | #include <string.h>\r |
8 | \r |
9 | #include "7zCrc.h"\r |
10 | #include "Alloc.h"\r |
11 | #include "Bra.h"\r |
12 | #include "CpuArch.h"\r |
13 | \r |
14 | #ifdef USE_SUBBLOCK\r |
15 | #include "Bcj3Enc.c"\r |
16 | #include "SbFind.c"\r |
17 | #include "SbEnc.c"\r |
18 | #endif\r |
19 | \r |
20 | #include "XzEnc.h"\r |
21 | \r |
22 | #define XzBlock_ClearFlags(p) (p)->flags = 0;\r |
23 | #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);\r |
24 | #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;\r |
25 | #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;\r |
26 | \r |
27 | static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)\r |
28 | {\r |
29 | return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;\r |
30 | }\r |
31 | \r |
32 | static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)\r |
33 | {\r |
34 | *crc = CrcUpdate(*crc, buf, size);\r |
35 | return WriteBytes(s, buf, size);\r |
36 | }\r |
37 | \r |
38 | static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)\r |
39 | {\r |
40 | UInt32 crc;\r |
41 | Byte header[XZ_STREAM_HEADER_SIZE];\r |
42 | memcpy(header, XZ_SIG, XZ_SIG_SIZE);\r |
43 | header[XZ_SIG_SIZE] = (Byte)(f >> 8);\r |
44 | header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);\r |
45 | crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);\r |
46 | SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);\r |
47 | return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);\r |
48 | }\r |
49 | \r |
50 | \r |
51 | static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)\r |
52 | {\r |
53 | Byte header[XZ_BLOCK_HEADER_SIZE_MAX];\r |
54 | \r |
55 | unsigned pos = 1;\r |
56 | unsigned numFilters, i;\r |
57 | header[pos++] = p->flags;\r |
58 | \r |
59 | if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);\r |
60 | if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);\r |
61 | numFilters = XzBlock_GetNumFilters(p);\r |
62 | \r |
63 | for (i = 0; i < numFilters; i++)\r |
64 | {\r |
65 | const CXzFilter *f = &p->filters[i];\r |
66 | pos += Xz_WriteVarInt(header + pos, f->id);\r |
67 | pos += Xz_WriteVarInt(header + pos, f->propsSize);\r |
68 | memcpy(header + pos, f->props, f->propsSize);\r |
69 | pos += f->propsSize;\r |
70 | }\r |
71 | \r |
72 | while ((pos & 3) != 0)\r |
73 | header[pos++] = 0;\r |
74 | \r |
75 | header[0] = (Byte)(pos >> 2);\r |
76 | SetUi32(header + pos, CrcCalc(header, pos));\r |
77 | return WriteBytes(s, header, pos + 4);\r |
78 | }\r |
79 | \r |
80 | \r |
81 | static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)\r |
82 | {\r |
83 | Byte buf[32];\r |
84 | UInt64 globalPos;\r |
85 | {\r |
86 | UInt32 crc = CRC_INIT_VAL;\r |
87 | unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);\r |
88 | size_t i;\r |
89 | \r |
90 | globalPos = pos;\r |
91 | buf[0] = 0;\r |
92 | RINOK(WriteBytesAndCrc(s, buf, pos, &crc));\r |
93 | \r |
94 | for (i = 0; i < p->numBlocks; i++)\r |
95 | {\r |
96 | const CXzBlockSizes *block = &p->blocks[i];\r |
97 | pos = Xz_WriteVarInt(buf, block->totalSize);\r |
98 | pos += Xz_WriteVarInt(buf + pos, block->unpackSize);\r |
99 | globalPos += pos;\r |
100 | RINOK(WriteBytesAndCrc(s, buf, pos, &crc));\r |
101 | }\r |
102 | \r |
103 | pos = ((unsigned)globalPos & 3);\r |
104 | \r |
105 | if (pos != 0)\r |
106 | {\r |
107 | buf[0] = buf[1] = buf[2] = 0;\r |
108 | RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));\r |
109 | globalPos += 4 - pos;\r |
110 | }\r |
111 | {\r |
112 | SetUi32(buf, CRC_GET_DIGEST(crc));\r |
113 | RINOK(WriteBytes(s, buf, 4));\r |
114 | globalPos += 4;\r |
115 | }\r |
116 | }\r |
117 | \r |
118 | {\r |
119 | UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);\r |
120 | SetUi32(buf + 4, indexSize);\r |
121 | buf[8] = (Byte)(p->flags >> 8);\r |
122 | buf[9] = (Byte)(p->flags & 0xFF);\r |
123 | SetUi32(buf, CrcCalc(buf + 4, 6));\r |
124 | memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);\r |
125 | return WriteBytes(s, buf, 12);\r |
126 | }\r |
127 | }\r |
128 | \r |
129 | \r |
130 | static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)\r |
131 | {\r |
132 | if (!p->blocks || p->numBlocksAllocated == p->numBlocks)\r |
133 | {\r |
134 | size_t num = p->numBlocks * 2 + 1;\r |
135 | size_t newSize = sizeof(CXzBlockSizes) * num;\r |
136 | CXzBlockSizes *blocks;\r |
137 | if (newSize / sizeof(CXzBlockSizes) != num)\r |
138 | return SZ_ERROR_MEM;\r |
139 | blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);\r |
140 | if (!blocks)\r |
141 | return SZ_ERROR_MEM;\r |
142 | if (p->numBlocks != 0)\r |
143 | {\r |
144 | memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));\r |
145 | alloc->Free(alloc, p->blocks);\r |
146 | }\r |
147 | p->blocks = blocks;\r |
148 | p->numBlocksAllocated = num;\r |
149 | }\r |
150 | {\r |
151 | CXzBlockSizes *block = &p->blocks[p->numBlocks++];\r |
152 | block->unpackSize = unpackSize;\r |
153 | block->totalSize = totalSize;\r |
154 | }\r |
155 | return SZ_OK;\r |
156 | }\r |
157 | \r |
158 | \r |
159 | /* ---------- CSeqCheckInStream ---------- */\r |
160 | \r |
161 | typedef struct\r |
162 | {\r |
163 | ISeqInStream p;\r |
164 | ISeqInStream *realStream;\r |
165 | UInt64 processed;\r |
166 | CXzCheck check;\r |
167 | } CSeqCheckInStream;\r |
168 | \r |
169 | static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode)\r |
170 | {\r |
171 | p->processed = 0;\r |
172 | XzCheck_Init(&p->check, mode);\r |
173 | }\r |
174 | \r |
175 | static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)\r |
176 | {\r |
177 | XzCheck_Final(&p->check, digest);\r |
178 | }\r |
179 | \r |
180 | static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)\r |
181 | {\r |
182 | CSeqCheckInStream *p = (CSeqCheckInStream *)pp;\r |
183 | SRes res = p->realStream->Read(p->realStream, data, size);\r |
184 | XzCheck_Update(&p->check, data, *size);\r |
185 | p->processed += *size;\r |
186 | return res;\r |
187 | }\r |
188 | \r |
189 | \r |
190 | /* ---------- CSeqSizeOutStream ---------- */\r |
191 | \r |
192 | typedef struct\r |
193 | {\r |
194 | ISeqOutStream p;\r |
195 | ISeqOutStream *realStream;\r |
196 | UInt64 processed;\r |
197 | } CSeqSizeOutStream;\r |
198 | \r |
199 | static size_t MyWrite(void *pp, const void *data, size_t size)\r |
200 | {\r |
201 | CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;\r |
202 | size = p->realStream->Write(p->realStream, data, size);\r |
203 | p->processed += size;\r |
204 | return size;\r |
205 | }\r |
206 | \r |
207 | \r |
208 | /* ---------- CSeqInFilter ---------- */\r |
209 | \r |
210 | #define FILTER_BUF_SIZE (1 << 20)\r |
211 | \r |
212 | typedef struct\r |
213 | {\r |
214 | ISeqInStream p;\r |
215 | ISeqInStream *realStream;\r |
216 | IStateCoder StateCoder;\r |
217 | Byte *buf;\r |
218 | size_t curPos;\r |
219 | size_t endPos;\r |
220 | int srcWasFinished;\r |
221 | } CSeqInFilter;\r |
222 | \r |
223 | static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)\r |
224 | {\r |
225 | CSeqInFilter *p = (CSeqInFilter *)pp;\r |
226 | size_t sizeOriginal = *size;\r |
227 | if (sizeOriginal == 0)\r |
228 | return SZ_OK;\r |
229 | *size = 0;\r |
230 | \r |
231 | for (;;)\r |
232 | {\r |
233 | if (!p->srcWasFinished && p->curPos == p->endPos)\r |
234 | {\r |
235 | p->curPos = 0;\r |
236 | p->endPos = FILTER_BUF_SIZE;\r |
237 | RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));\r |
238 | if (p->endPos == 0)\r |
239 | p->srcWasFinished = 1;\r |
240 | }\r |
241 | {\r |
242 | SizeT srcLen = p->endPos - p->curPos;\r |
243 | int wasFinished;\r |
244 | SRes res;\r |
245 | *size = sizeOriginal;\r |
246 | res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,\r |
247 | p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);\r |
248 | p->curPos += srcLen;\r |
249 | if (*size != 0 || srcLen == 0 || res != 0)\r |
250 | return res;\r |
251 | }\r |
252 | }\r |
253 | }\r |
254 | \r |
255 | static void SeqInFilter_Construct(CSeqInFilter *p)\r |
256 | {\r |
257 | p->buf = NULL;\r |
258 | p->p.Read = SeqInFilter_Read;\r |
259 | }\r |
260 | \r |
261 | static void SeqInFilter_Free(CSeqInFilter *p)\r |
262 | {\r |
263 | if (p->buf)\r |
264 | {\r |
265 | g_Alloc.Free(&g_Alloc, p->buf);\r |
266 | p->buf = NULL;\r |
267 | }\r |
268 | }\r |
269 | \r |
270 | SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);\r |
271 | \r |
272 | static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)\r |
273 | {\r |
274 | if (!p->buf)\r |
275 | {\r |
276 | p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);\r |
277 | if (!p->buf)\r |
278 | return SZ_ERROR_MEM;\r |
279 | }\r |
280 | p->curPos = p->endPos = 0;\r |
281 | p->srcWasFinished = 0;\r |
282 | RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));\r |
283 | RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));\r |
284 | p->StateCoder.Init(p->StateCoder.p);\r |
285 | return SZ_OK;\r |
286 | }\r |
287 | \r |
288 | \r |
289 | /* ---------- CSbEncInStream ---------- */\r |
290 | \r |
291 | #ifdef USE_SUBBLOCK\r |
292 | \r |
293 | typedef struct\r |
294 | {\r |
295 | ISeqInStream p;\r |
296 | ISeqInStream *inStream;\r |
297 | CSbEnc enc;\r |
298 | } CSbEncInStream;\r |
299 | \r |
300 | static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)\r |
301 | {\r |
302 | CSbEncInStream *p = (CSbEncInStream *)pp;\r |
303 | size_t sizeOriginal = *size;\r |
304 | if (sizeOriginal == 0)\r |
305 | return S_OK;\r |
306 | \r |
307 | for (;;)\r |
308 | {\r |
309 | if (p->enc.needRead && !p->enc.readWasFinished)\r |
310 | {\r |
311 | size_t processed = p->enc.needReadSizeMax;\r |
312 | RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));\r |
313 | p->enc.readPos += processed;\r |
314 | if (processed == 0)\r |
315 | {\r |
316 | p->enc.readWasFinished = True;\r |
317 | p->enc.isFinalFinished = True;\r |
318 | }\r |
319 | p->enc.needRead = False;\r |
320 | }\r |
321 | \r |
322 | *size = sizeOriginal;\r |
323 | RINOK(SbEnc_Read(&p->enc, data, size));\r |
324 | if (*size != 0 || !p->enc.needRead)\r |
325 | return S_OK;\r |
326 | }\r |
327 | }\r |
328 | \r |
329 | void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)\r |
330 | {\r |
331 | SbEnc_Construct(&p->enc, alloc);\r |
332 | p->p.Read = SbEncInStream_Read;\r |
333 | }\r |
334 | \r |
335 | SRes SbEncInStream_Init(CSbEncInStream *p)\r |
336 | {\r |
337 | return SbEnc_Init(&p->enc);\r |
338 | }\r |
339 | \r |
340 | void SbEncInStream_Free(CSbEncInStream *p)\r |
341 | {\r |
342 | SbEnc_Free(&p->enc);\r |
343 | }\r |
344 | \r |
345 | #endif\r |
346 | \r |
347 | \r |
348 | typedef struct\r |
349 | {\r |
350 | CLzma2EncHandle lzma2;\r |
351 | #ifdef USE_SUBBLOCK\r |
352 | CSbEncInStream sb;\r |
353 | #endif\r |
354 | CSeqInFilter filter;\r |
355 | ISzAlloc *alloc;\r |
356 | ISzAlloc *bigAlloc;\r |
357 | } CLzma2WithFilters;\r |
358 | \r |
359 | \r |
360 | static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)\r |
361 | {\r |
362 | p->alloc = alloc;\r |
363 | p->bigAlloc = bigAlloc;\r |
364 | p->lzma2 = NULL;\r |
365 | #ifdef USE_SUBBLOCK\r |
366 | SbEncInStream_Construct(&p->sb, alloc);\r |
367 | #endif\r |
368 | SeqInFilter_Construct(&p->filter);\r |
369 | }\r |
370 | \r |
371 | static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)\r |
372 | {\r |
373 | p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);\r |
374 | if (!p->lzma2)\r |
375 | return SZ_ERROR_MEM;\r |
376 | return SZ_OK;\r |
377 | }\r |
378 | \r |
379 | static void Lzma2WithFilters_Free(CLzma2WithFilters *p)\r |
380 | {\r |
381 | SeqInFilter_Free(&p->filter);\r |
382 | #ifdef USE_SUBBLOCK\r |
383 | SbEncInStream_Free(&p->sb);\r |
384 | #endif\r |
385 | if (p->lzma2)\r |
386 | {\r |
387 | Lzma2Enc_Destroy(p->lzma2);\r |
388 | p->lzma2 = NULL;\r |
389 | }\r |
390 | }\r |
391 | \r |
392 | \r |
393 | void XzProps_Init(CXzProps *p)\r |
394 | {\r |
395 | p->lzma2Props = NULL;\r |
396 | p->filterProps = NULL;\r |
397 | p->checkId = XZ_CHECK_CRC32;\r |
398 | }\r |
399 | \r |
400 | void XzFilterProps_Init(CXzFilterProps *p)\r |
401 | {\r |
402 | p->id = 0;\r |
403 | p->delta = 0;\r |
404 | p->ip = 0;\r |
405 | p->ipDefined = False;\r |
406 | }\r |
407 | \r |
408 | \r |
409 | static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,\r |
410 | ISeqOutStream *outStream, ISeqInStream *inStream,\r |
411 | const CXzProps *props, ICompressProgress *progress)\r |
412 | {\r |
413 | xz->flags = (Byte)props->checkId;\r |
414 | \r |
415 | RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));\r |
416 | RINOK(Xz_WriteHeader(xz->flags, outStream));\r |
417 | \r |
418 | {\r |
419 | CSeqCheckInStream checkInStream;\r |
420 | CSeqSizeOutStream seqSizeOutStream;\r |
421 | CXzBlock block;\r |
422 | unsigned filterIndex = 0;\r |
423 | CXzFilter *filter = NULL;\r |
424 | const CXzFilterProps *fp = props->filterProps;\r |
425 | \r |
426 | XzBlock_ClearFlags(&block);\r |
427 | XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));\r |
428 | \r |
429 | if (fp)\r |
430 | {\r |
431 | filter = &block.filters[filterIndex++];\r |
432 | filter->id = fp->id;\r |
433 | filter->propsSize = 0;\r |
434 | \r |
435 | if (fp->id == XZ_ID_Delta)\r |
436 | {\r |
437 | filter->props[0] = (Byte)(fp->delta - 1);\r |
438 | filter->propsSize = 1;\r |
439 | }\r |
440 | else if (fp->ipDefined)\r |
441 | {\r |
442 | SetUi32(filter->props, fp->ip);\r |
443 | filter->propsSize = 4;\r |
444 | }\r |
445 | }\r |
446 | \r |
447 | {\r |
448 | CXzFilter *f = &block.filters[filterIndex++];\r |
449 | f->id = XZ_ID_LZMA2;\r |
450 | f->propsSize = 1;\r |
451 | f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);\r |
452 | }\r |
453 | \r |
454 | seqSizeOutStream.p.Write = MyWrite;\r |
455 | seqSizeOutStream.realStream = outStream;\r |
456 | seqSizeOutStream.processed = 0;\r |
457 | \r |
458 | RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));\r |
459 | \r |
460 | checkInStream.p.Read = SeqCheckInStream_Read;\r |
461 | checkInStream.realStream = inStream;\r |
462 | SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));\r |
463 | \r |
464 | if (fp)\r |
465 | {\r |
466 | #ifdef USE_SUBBLOCK\r |
467 | if (fp->id == XZ_ID_Subblock)\r |
468 | {\r |
469 | lzmaf->sb.inStream = &checkInStream.p;\r |
470 | RINOK(SbEncInStream_Init(&lzmaf->sb));\r |
471 | }\r |
472 | else\r |
473 | #endif\r |
474 | {\r |
475 | lzmaf->filter.realStream = &checkInStream.p;\r |
476 | RINOK(SeqInFilter_Init(&lzmaf->filter, filter));\r |
477 | }\r |
478 | }\r |
479 | \r |
480 | {\r |
481 | UInt64 packPos = seqSizeOutStream.processed;\r |
482 | \r |
483 | SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,\r |
484 | fp ?\r |
485 | #ifdef USE_SUBBLOCK\r |
486 | (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:\r |
487 | #endif\r |
488 | &lzmaf->filter.p:\r |
489 | &checkInStream.p,\r |
490 | progress);\r |
491 | \r |
492 | RINOK(res);\r |
493 | block.unpackSize = checkInStream.processed;\r |
494 | block.packSize = seqSizeOutStream.processed - packPos;\r |
495 | }\r |
496 | \r |
497 | {\r |
498 | unsigned padSize = 0;\r |
499 | Byte buf[128];\r |
500 | while ((((unsigned)block.packSize + padSize) & 3) != 0)\r |
501 | buf[padSize++] = 0;\r |
502 | SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);\r |
503 | RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));\r |
504 | RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));\r |
505 | }\r |
506 | }\r |
507 | return Xz_WriteFooter(xz, outStream);\r |
508 | }\r |
509 | \r |
510 | \r |
511 | SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,\r |
512 | const CXzProps *props, ICompressProgress *progress)\r |
513 | {\r |
514 | SRes res;\r |
515 | CXzStream xz;\r |
516 | CLzma2WithFilters lzmaf;\r |
517 | Xz_Construct(&xz);\r |
518 | Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);\r |
519 | res = Lzma2WithFilters_Create(&lzmaf);\r |
520 | if (res == SZ_OK)\r |
521 | res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);\r |
522 | Lzma2WithFilters_Free(&lzmaf);\r |
523 | Xz_Free(&xz, &g_Alloc);\r |
524 | return res;\r |
525 | }\r |
526 | \r |
527 | \r |
528 | SRes Xz_EncodeEmpty(ISeqOutStream *outStream)\r |
529 | {\r |
530 | SRes res;\r |
531 | CXzStream xz;\r |
532 | Xz_Construct(&xz);\r |
533 | res = Xz_WriteHeader(xz.flags, outStream);\r |
534 | if (res == SZ_OK)\r |
535 | res = Xz_WriteFooter(&xz, outStream);\r |
536 | Xz_Free(&xz, &g_Alloc);\r |
537 | return res;\r |
538 | }\r |