9e052883 |
1 | /* gzappend -- command to append to a gzip file |
2 | |
3 | Copyright (C) 2003, 2012 Mark Adler, all rights reserved |
4 | version 1.2, 11 Oct 2012 |
5 | |
6 | This software is provided 'as-is', without any express or implied |
7 | warranty. In no event will the author be held liable for any damages |
8 | arising from the use of this software. |
9 | |
10 | Permission is granted to anyone to use this software for any purpose, |
11 | including commercial applications, and to alter it and redistribute it |
12 | freely, subject to the following restrictions: |
13 | |
14 | 1. The origin of this software must not be misrepresented; you must not |
15 | claim that you wrote the original software. If you use this software |
16 | in a product, an acknowledgment in the product documentation would be |
17 | appreciated but is not required. |
18 | 2. Altered source versions must be plainly marked as such, and must not be |
19 | misrepresented as being the original software. |
20 | 3. This notice may not be removed or altered from any source distribution. |
21 | |
22 | Mark Adler madler@alumni.caltech.edu |
23 | */ |
24 | |
25 | /* |
26 | * Change history: |
27 | * |
28 | * 1.0 19 Oct 2003 - First version |
29 | * 1.1 4 Nov 2003 - Expand and clarify some comments and notes |
30 | * - Add version and copyright to help |
31 | * - Send help to stdout instead of stderr |
32 | * - Add some preemptive typecasts |
33 | * - Add L to constants in lseek() calls |
34 | * - Remove some debugging information in error messages |
35 | * - Use new data_type definition for zlib 1.2.1 |
36 | * - Simplify and unify file operations |
37 | * - Finish off gzip file in gztack() |
38 | * - Use deflatePrime() instead of adding empty blocks |
39 | * - Keep gzip file clean on appended file read errors |
40 | * - Use in-place rotate instead of auxiliary buffer |
41 | * (Why you ask? Because it was fun to write!) |
42 | * 1.2 11 Oct 2012 - Fix for proper z_const usage |
43 | * - Check for input buffer malloc failure |
44 | */ |
45 | |
46 | /* |
47 | gzappend takes a gzip file and appends to it, compressing files from the |
48 | command line or data from stdin. The gzip file is written to directly, to |
49 | avoid copying that file, in case it's large. Note that this results in the |
50 | unfriendly behavior that if gzappend fails, the gzip file is corrupted. |
51 | |
52 | This program was written to illustrate the use of the new Z_BLOCK option of |
53 | zlib 1.2.x's inflate() function. This option returns from inflate() at each |
54 | block boundary to facilitate locating and modifying the last block bit at |
55 | the start of the final deflate block. Also whether using Z_BLOCK or not, |
56 | another required feature of zlib 1.2.x is that inflate() now provides the |
57 | number of unused bits in the last input byte used. gzappend will not work |
58 | with versions of zlib earlier than 1.2.1. |
59 | |
60 | gzappend first decompresses the gzip file internally, discarding all but |
61 | the last 32K of uncompressed data, and noting the location of the last block |
62 | bit and the number of unused bits in the last byte of the compressed data. |
63 | The gzip trailer containing the CRC-32 and length of the uncompressed data |
64 | is verified. This trailer will be later overwritten. |
65 | |
66 | Then the last block bit is cleared by seeking back in the file and rewriting |
67 | the byte that contains it. Seeking forward, the last byte of the compressed |
68 | data is saved along with the number of unused bits to initialize deflate. |
69 | |
70 | A deflate process is initialized, using the last 32K of the uncompressed |
71 | data from the gzip file to initialize the dictionary. If the total |
72 | uncompressed data was less than 32K, then all of it is used to initialize |
73 | the dictionary. The deflate output bit buffer is also initialized with the |
74 | last bits from the original deflate stream. From here on, the data to |
75 | append is simply compressed using deflate, and written to the gzip file. |
76 | When that is complete, the new CRC-32 and uncompressed length are written |
77 | as the trailer of the gzip file. |
78 | */ |
79 | |
80 | #include <stdio.h> |
81 | #include <stdlib.h> |
82 | #include <string.h> |
83 | #include <fcntl.h> |
84 | #include <unistd.h> |
85 | #include "zlib.h" |
86 | |
87 | #define local static |
88 | #define LGCHUNK 14 |
89 | #define CHUNK (1U << LGCHUNK) |
90 | #define DSIZE 32768U |
91 | |
92 | /* print an error message and terminate with extreme prejudice */ |
93 | local void bye(char *msg1, char *msg2) |
94 | { |
95 | fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2); |
96 | exit(1); |
97 | } |
98 | |
99 | /* return the greatest common divisor of a and b using Euclid's algorithm, |
100 | modified to be fast when one argument much greater than the other, and |
101 | coded to avoid unnecessary swapping */ |
102 | local unsigned gcd(unsigned a, unsigned b) |
103 | { |
104 | unsigned c; |
105 | |
106 | while (a && b) |
107 | if (a > b) { |
108 | c = b; |
109 | while (a - c >= c) |
110 | c <<= 1; |
111 | a -= c; |
112 | } |
113 | else { |
114 | c = a; |
115 | while (b - c >= c) |
116 | c <<= 1; |
117 | b -= c; |
118 | } |
119 | return a + b; |
120 | } |
121 | |
122 | /* rotate list[0..len-1] left by rot positions, in place */ |
123 | local void rotate(unsigned char *list, unsigned len, unsigned rot) |
124 | { |
125 | unsigned char tmp; |
126 | unsigned cycles; |
127 | unsigned char *start, *last, *to, *from; |
128 | |
129 | /* normalize rot and handle degenerate cases */ |
130 | if (len < 2) return; |
131 | if (rot >= len) rot %= len; |
132 | if (rot == 0) return; |
133 | |
134 | /* pointer to last entry in list */ |
135 | last = list + (len - 1); |
136 | |
137 | /* do simple left shift by one */ |
138 | if (rot == 1) { |
139 | tmp = *list; |
140 | memmove(list, list + 1, len - 1); |
141 | *last = tmp; |
142 | return; |
143 | } |
144 | |
145 | /* do simple right shift by one */ |
146 | if (rot == len - 1) { |
147 | tmp = *last; |
148 | memmove(list + 1, list, len - 1); |
149 | *list = tmp; |
150 | return; |
151 | } |
152 | |
153 | /* otherwise do rotate as a set of cycles in place */ |
154 | cycles = gcd(len, rot); /* number of cycles */ |
155 | do { |
156 | start = from = list + cycles; /* start index is arbitrary */ |
157 | tmp = *from; /* save entry to be overwritten */ |
158 | for (;;) { |
159 | to = from; /* next step in cycle */ |
160 | from += rot; /* go right rot positions */ |
161 | if (from > last) from -= len; /* (pointer better not wrap) */ |
162 | if (from == start) break; /* all but one shifted */ |
163 | *to = *from; /* shift left */ |
164 | } |
165 | *to = tmp; /* complete the circle */ |
166 | } while (--cycles); |
167 | } |
168 | |
169 | /* structure for gzip file read operations */ |
170 | typedef struct { |
171 | int fd; /* file descriptor */ |
172 | int size; /* 1 << size is bytes in buf */ |
173 | unsigned left; /* bytes available at next */ |
174 | unsigned char *buf; /* buffer */ |
175 | z_const unsigned char *next; /* next byte in buffer */ |
176 | char *name; /* file name for error messages */ |
177 | } file; |
178 | |
179 | /* reload buffer */ |
180 | local int readin(file *in) |
181 | { |
182 | int len; |
183 | |
184 | len = read(in->fd, in->buf, 1 << in->size); |
185 | if (len == -1) bye("error reading ", in->name); |
186 | in->left = (unsigned)len; |
187 | in->next = in->buf; |
188 | return len; |
189 | } |
190 | |
191 | /* read from file in, exit if end-of-file */ |
192 | local int readmore(file *in) |
193 | { |
194 | if (readin(in) == 0) bye("unexpected end of ", in->name); |
195 | return 0; |
196 | } |
197 | |
198 | #define read1(in) (in->left == 0 ? readmore(in) : 0, \ |
199 | in->left--, *(in->next)++) |
200 | |
201 | /* skip over n bytes of in */ |
202 | local void skip(file *in, unsigned n) |
203 | { |
204 | unsigned bypass; |
205 | |
206 | if (n > in->left) { |
207 | n -= in->left; |
208 | bypass = n & ~((1U << in->size) - 1); |
209 | if (bypass) { |
210 | if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1) |
211 | bye("seeking ", in->name); |
212 | n -= bypass; |
213 | } |
214 | readmore(in); |
215 | if (n > in->left) |
216 | bye("unexpected end of ", in->name); |
217 | } |
218 | in->left -= n; |
219 | in->next += n; |
220 | } |
221 | |
222 | /* read a four-byte unsigned integer, little-endian, from in */ |
223 | unsigned long read4(file *in) |
224 | { |
225 | unsigned long val; |
226 | |
227 | val = read1(in); |
228 | val += (unsigned)read1(in) << 8; |
229 | val += (unsigned long)read1(in) << 16; |
230 | val += (unsigned long)read1(in) << 24; |
231 | return val; |
232 | } |
233 | |
234 | /* skip over gzip header */ |
235 | local void gzheader(file *in) |
236 | { |
237 | int flags; |
238 | unsigned n; |
239 | |
240 | if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file"); |
241 | if (read1(in) != 8) bye("unknown compression method in", in->name); |
242 | flags = read1(in); |
243 | if (flags & 0xe0) bye("unknown header flags set in", in->name); |
244 | skip(in, 6); |
245 | if (flags & 4) { |
246 | n = read1(in); |
247 | n += (unsigned)(read1(in)) << 8; |
248 | skip(in, n); |
249 | } |
250 | if (flags & 8) while (read1(in) != 0) ; |
251 | if (flags & 16) while (read1(in) != 0) ; |
252 | if (flags & 2) skip(in, 2); |
253 | } |
254 | |
255 | /* decompress gzip file "name", return strm with a deflate stream ready to |
256 | continue compression of the data in the gzip file, and return a file |
257 | descriptor pointing to where to write the compressed data -- the deflate |
258 | stream is initialized to compress using level "level" */ |
259 | local int gzscan(char *name, z_stream *strm, int level) |
260 | { |
261 | int ret, lastbit, left, full; |
262 | unsigned have; |
263 | unsigned long crc, tot; |
264 | unsigned char *window; |
265 | off_t lastoff, end; |
266 | file gz; |
267 | |
268 | /* open gzip file */ |
269 | gz.name = name; |
270 | gz.fd = open(name, O_RDWR, 0); |
271 | if (gz.fd == -1) bye("cannot open ", name); |
272 | gz.buf = malloc(CHUNK); |
273 | if (gz.buf == NULL) bye("out of memory", ""); |
274 | gz.size = LGCHUNK; |
275 | gz.left = 0; |
276 | |
277 | /* skip gzip header */ |
278 | gzheader(&gz); |
279 | |
280 | /* prepare to decompress */ |
281 | window = malloc(DSIZE); |
282 | if (window == NULL) bye("out of memory", ""); |
283 | strm->zalloc = Z_NULL; |
284 | strm->zfree = Z_NULL; |
285 | strm->opaque = Z_NULL; |
286 | ret = inflateInit2(strm, -15); |
287 | if (ret != Z_OK) bye("out of memory", " or library mismatch"); |
288 | |
289 | /* decompress the deflate stream, saving append information */ |
290 | lastbit = 0; |
291 | lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
292 | left = 0; |
293 | strm->avail_in = gz.left; |
294 | strm->next_in = gz.next; |
295 | crc = crc32(0L, Z_NULL, 0); |
296 | have = full = 0; |
297 | do { |
298 | /* if needed, get more input */ |
299 | if (strm->avail_in == 0) { |
300 | readmore(&gz); |
301 | strm->avail_in = gz.left; |
302 | strm->next_in = gz.next; |
303 | } |
304 | |
305 | /* set up output to next available section of sliding window */ |
306 | strm->avail_out = DSIZE - have; |
307 | strm->next_out = window + have; |
308 | |
309 | /* inflate and check for errors */ |
310 | ret = inflate(strm, Z_BLOCK); |
311 | if (ret == Z_STREAM_ERROR) bye("internal stream error!", ""); |
312 | if (ret == Z_MEM_ERROR) bye("out of memory", ""); |
313 | if (ret == Z_DATA_ERROR) |
314 | bye("invalid compressed data--format violated in", name); |
315 | |
316 | /* update crc and sliding window pointer */ |
317 | crc = crc32(crc, window + have, DSIZE - have - strm->avail_out); |
318 | if (strm->avail_out) |
319 | have = DSIZE - strm->avail_out; |
320 | else { |
321 | have = 0; |
322 | full = 1; |
323 | } |
324 | |
325 | /* process end of block */ |
326 | if (strm->data_type & 128) { |
327 | if (strm->data_type & 64) |
328 | left = strm->data_type & 0x1f; |
329 | else { |
330 | lastbit = strm->data_type & 0x1f; |
331 | lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in; |
332 | } |
333 | } |
334 | } while (ret != Z_STREAM_END); |
335 | inflateEnd(strm); |
336 | gz.left = strm->avail_in; |
337 | gz.next = strm->next_in; |
338 | |
339 | /* save the location of the end of the compressed data */ |
340 | end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
341 | |
342 | /* check gzip trailer and save total for deflate */ |
343 | if (crc != read4(&gz)) |
344 | bye("invalid compressed data--crc mismatch in ", name); |
345 | tot = strm->total_out; |
346 | if ((tot & 0xffffffffUL) != read4(&gz)) |
347 | bye("invalid compressed data--length mismatch in", name); |
348 | |
349 | /* if not at end of file, warn */ |
350 | if (gz.left || readin(&gz)) |
351 | fprintf(stderr, |
352 | "gzappend warning: junk at end of gzip file overwritten\n"); |
353 | |
354 | /* clear last block bit */ |
355 | lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET); |
356 | if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
357 | *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7))); |
358 | lseek(gz.fd, -1L, SEEK_CUR); |
359 | if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name); |
360 | |
361 | /* if window wrapped, build dictionary from window by rotating */ |
362 | if (full) { |
363 | rotate(window, DSIZE, have); |
364 | have = DSIZE; |
365 | } |
366 | |
367 | /* set up deflate stream with window, crc, total_in, and leftover bits */ |
368 | ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); |
369 | if (ret != Z_OK) bye("out of memory", ""); |
370 | deflateSetDictionary(strm, window, have); |
371 | strm->adler = crc; |
372 | strm->total_in = tot; |
373 | if (left) { |
374 | lseek(gz.fd, --end, SEEK_SET); |
375 | if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
376 | deflatePrime(strm, 8 - left, *gz.buf); |
377 | } |
378 | lseek(gz.fd, end, SEEK_SET); |
379 | |
380 | /* clean up and return */ |
381 | free(window); |
382 | free(gz.buf); |
383 | return gz.fd; |
384 | } |
385 | |
386 | /* append file "name" to gzip file gd using deflate stream strm -- if last |
387 | is true, then finish off the deflate stream at the end */ |
388 | local void gztack(char *name, int gd, z_stream *strm, int last) |
389 | { |
390 | int fd, len, ret; |
391 | unsigned left; |
392 | unsigned char *in, *out; |
393 | |
394 | /* open file to compress and append */ |
395 | fd = 0; |
396 | if (name != NULL) { |
397 | fd = open(name, O_RDONLY, 0); |
398 | if (fd == -1) |
399 | fprintf(stderr, "gzappend warning: %s not found, skipping ...\n", |
400 | name); |
401 | } |
402 | |
403 | /* allocate buffers */ |
404 | in = malloc(CHUNK); |
405 | out = malloc(CHUNK); |
406 | if (in == NULL || out == NULL) bye("out of memory", ""); |
407 | |
408 | /* compress input file and append to gzip file */ |
409 | do { |
410 | /* get more input */ |
411 | len = read(fd, in, CHUNK); |
412 | if (len == -1) { |
413 | fprintf(stderr, |
414 | "gzappend warning: error reading %s, skipping rest ...\n", |
415 | name); |
416 | len = 0; |
417 | } |
418 | strm->avail_in = (unsigned)len; |
419 | strm->next_in = in; |
420 | if (len) strm->adler = crc32(strm->adler, in, (unsigned)len); |
421 | |
422 | /* compress and write all available output */ |
423 | do { |
424 | strm->avail_out = CHUNK; |
425 | strm->next_out = out; |
426 | ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH); |
427 | left = CHUNK - strm->avail_out; |
428 | while (left) { |
429 | len = write(gd, out + CHUNK - strm->avail_out - left, left); |
430 | if (len == -1) bye("writing gzip file", ""); |
431 | left -= (unsigned)len; |
432 | } |
433 | } while (strm->avail_out == 0 && ret != Z_STREAM_END); |
434 | } while (len != 0); |
435 | |
436 | /* write trailer after last entry */ |
437 | if (last) { |
438 | deflateEnd(strm); |
439 | out[0] = (unsigned char)(strm->adler); |
440 | out[1] = (unsigned char)(strm->adler >> 8); |
441 | out[2] = (unsigned char)(strm->adler >> 16); |
442 | out[3] = (unsigned char)(strm->adler >> 24); |
443 | out[4] = (unsigned char)(strm->total_in); |
444 | out[5] = (unsigned char)(strm->total_in >> 8); |
445 | out[6] = (unsigned char)(strm->total_in >> 16); |
446 | out[7] = (unsigned char)(strm->total_in >> 24); |
447 | len = 8; |
448 | do { |
449 | ret = write(gd, out + 8 - len, len); |
450 | if (ret == -1) bye("writing gzip file", ""); |
451 | len -= ret; |
452 | } while (len); |
453 | close(gd); |
454 | } |
455 | |
456 | /* clean up and return */ |
457 | free(out); |
458 | free(in); |
459 | if (fd > 0) close(fd); |
460 | } |
461 | |
462 | /* process the compression level option if present, scan the gzip file, and |
463 | append the specified files, or append the data from stdin if no other file |
464 | names are provided on the command line -- the gzip file must be writable |
465 | and seekable */ |
466 | int main(int argc, char **argv) |
467 | { |
468 | int gd, level; |
469 | z_stream strm; |
470 | |
471 | /* ignore command name */ |
472 | argc--; argv++; |
473 | |
474 | /* provide usage if no arguments */ |
475 | if (*argv == NULL) { |
476 | printf( |
477 | "gzappend 1.2 (11 Oct 2012) Copyright (C) 2003, 2012 Mark Adler\n" |
478 | ); |
479 | printf( |
480 | "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n"); |
481 | return 0; |
482 | } |
483 | |
484 | /* set compression level */ |
485 | level = Z_DEFAULT_COMPRESSION; |
486 | if (argv[0][0] == '-') { |
487 | if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0) |
488 | bye("invalid compression level", ""); |
489 | level = argv[0][1] - '0'; |
490 | if (*++argv == NULL) bye("no gzip file name after options", ""); |
491 | } |
492 | |
493 | /* prepare to append to gzip file */ |
494 | gd = gzscan(*argv++, &strm, level); |
495 | |
496 | /* append files on command line, or from stdin if none */ |
497 | if (*argv == NULL) |
498 | gztack(NULL, gd, &strm, 1); |
499 | else |
500 | do { |
501 | gztack(*argv, gd, &strm, argv[1] == NULL); |
502 | } while (*++argv != NULL); |
503 | return 0; |
504 | } |