9e052883 |
1 | /* gzjoin -- command to join gzip files into one gzip file |
2 | |
3 | Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
4 | version 1.2, 14 Aug 2012 |
5 | |
6 | This software is provided 'as-is', without any express or implied |
7 | warranty. In no event will the author be held liable for any damages |
8 | arising from the use of this software. |
9 | |
10 | Permission is granted to anyone to use this software for any purpose, |
11 | including commercial applications, and to alter it and redistribute it |
12 | freely, subject to the following restrictions: |
13 | |
14 | 1. The origin of this software must not be misrepresented; you must not |
15 | claim that you wrote the original software. If you use this software |
16 | in a product, an acknowledgment in the product documentation would be |
17 | appreciated but is not required. |
18 | 2. Altered source versions must be plainly marked as such, and must not be |
19 | misrepresented as being the original software. |
20 | 3. This notice may not be removed or altered from any source distribution. |
21 | |
22 | Mark Adler madler@alumni.caltech.edu |
23 | */ |
24 | |
25 | /* |
26 | * Change history: |
27 | * |
28 | * 1.0 11 Dec 2004 - First version |
29 | * 1.1 12 Jun 2005 - Changed ssize_t to long for portability |
30 | * 1.2 14 Aug 2012 - Clean up for z_const usage |
31 | */ |
32 | |
33 | /* |
34 | gzjoin takes one or more gzip files on the command line and writes out a |
35 | single gzip file that will uncompress to the concatenation of the |
36 | uncompressed data from the individual gzip files. gzjoin does this without |
37 | having to recompress any of the data and without having to calculate a new |
38 | crc32 for the concatenated uncompressed data. gzjoin does however have to |
39 | decompress all of the input data in order to find the bits in the compressed |
40 | data that need to be modified to concatenate the streams. |
41 | |
42 | gzjoin does not do an integrity check on the input gzip files other than |
43 | checking the gzip header and decompressing the compressed data. They are |
44 | otherwise assumed to be complete and correct. |
45 | |
46 | Each joint between gzip files removes at least 18 bytes of previous trailer |
47 | and subsequent header, and inserts an average of about three bytes to the |
48 | compressed data in order to connect the streams. The output gzip file |
49 | has a minimal ten-byte gzip header with no file name or modification time. |
50 | |
51 | This program was written to illustrate the use of the Z_BLOCK option of |
52 | inflate() and the crc32_combine() function. gzjoin will not compile with |
53 | versions of zlib earlier than 1.2.3. |
54 | */ |
55 | |
56 | #include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
57 | #include <stdlib.h> /* exit(), malloc(), free() */ |
58 | #include <fcntl.h> /* open() */ |
59 | #include <unistd.h> /* close(), read(), lseek() */ |
60 | #include "zlib.h" |
61 | /* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ |
62 | |
63 | #define local static |
64 | |
65 | /* exit with an error (return a value to allow use in an expression) */ |
66 | local int bail(char *why1, char *why2) |
67 | { |
68 | fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
69 | exit(1); |
70 | return 0; |
71 | } |
72 | |
73 | /* -- simple buffered file input with access to the buffer -- */ |
74 | |
75 | #define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
76 | |
77 | /* bin buffered input file type */ |
78 | typedef struct { |
79 | char *name; /* name of file for error messages */ |
80 | int fd; /* file descriptor */ |
81 | unsigned left; /* bytes remaining at next */ |
82 | unsigned char *next; /* next byte to read */ |
83 | unsigned char *buf; /* allocated buffer of length CHUNK */ |
84 | } bin; |
85 | |
86 | /* close a buffered file and free allocated memory */ |
87 | local void bclose(bin *in) |
88 | { |
89 | if (in != NULL) { |
90 | if (in->fd != -1) |
91 | close(in->fd); |
92 | if (in->buf != NULL) |
93 | free(in->buf); |
94 | free(in); |
95 | } |
96 | } |
97 | |
98 | /* open a buffered file for input, return a pointer to type bin, or NULL on |
99 | failure */ |
100 | local bin *bopen(char *name) |
101 | { |
102 | bin *in; |
103 | |
104 | in = malloc(sizeof(bin)); |
105 | if (in == NULL) |
106 | return NULL; |
107 | in->buf = malloc(CHUNK); |
108 | in->fd = open(name, O_RDONLY, 0); |
109 | if (in->buf == NULL || in->fd == -1) { |
110 | bclose(in); |
111 | return NULL; |
112 | } |
113 | in->left = 0; |
114 | in->next = in->buf; |
115 | in->name = name; |
116 | return in; |
117 | } |
118 | |
119 | /* load buffer from file, return -1 on read error, 0 or 1 on success, with |
120 | 1 indicating that end-of-file was reached */ |
121 | local int bload(bin *in) |
122 | { |
123 | long len; |
124 | |
125 | if (in == NULL) |
126 | return -1; |
127 | if (in->left != 0) |
128 | return 0; |
129 | in->next = in->buf; |
130 | do { |
131 | len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
132 | if (len < 0) |
133 | return -1; |
134 | in->left += (unsigned)len; |
135 | } while (len != 0 && in->left < CHUNK); |
136 | return len == 0 ? 1 : 0; |
137 | } |
138 | |
139 | /* get a byte from the file, bail if end of file */ |
140 | #define bget(in) (in->left ? 0 : bload(in), \ |
141 | in->left ? (in->left--, *(in->next)++) : \ |
142 | bail("unexpected end of file on ", in->name)) |
143 | |
144 | /* get a four-byte little-endian unsigned integer from file */ |
145 | local unsigned long bget4(bin *in) |
146 | { |
147 | unsigned long val; |
148 | |
149 | val = bget(in); |
150 | val += (unsigned long)(bget(in)) << 8; |
151 | val += (unsigned long)(bget(in)) << 16; |
152 | val += (unsigned long)(bget(in)) << 24; |
153 | return val; |
154 | } |
155 | |
156 | /* skip bytes in file */ |
157 | local void bskip(bin *in, unsigned skip) |
158 | { |
159 | /* check pointer */ |
160 | if (in == NULL) |
161 | return; |
162 | |
163 | /* easy case -- skip bytes in buffer */ |
164 | if (skip <= in->left) { |
165 | in->left -= skip; |
166 | in->next += skip; |
167 | return; |
168 | } |
169 | |
170 | /* skip what's in buffer, discard buffer contents */ |
171 | skip -= in->left; |
172 | in->left = 0; |
173 | |
174 | /* seek past multiples of CHUNK bytes */ |
175 | if (skip > CHUNK) { |
176 | unsigned left; |
177 | |
178 | left = skip & (CHUNK - 1); |
179 | if (left == 0) { |
180 | /* exact number of chunks: seek all the way minus one byte to check |
181 | for end-of-file with a read */ |
182 | lseek(in->fd, skip - 1, SEEK_CUR); |
183 | if (read(in->fd, in->buf, 1) != 1) |
184 | bail("unexpected end of file on ", in->name); |
185 | return; |
186 | } |
187 | |
188 | /* skip the integral chunks, update skip with remainder */ |
189 | lseek(in->fd, skip - left, SEEK_CUR); |
190 | skip = left; |
191 | } |
192 | |
193 | /* read more input and skip remainder */ |
194 | bload(in); |
195 | if (skip > in->left) |
196 | bail("unexpected end of file on ", in->name); |
197 | in->left -= skip; |
198 | in->next += skip; |
199 | } |
200 | |
201 | /* -- end of buffered input functions -- */ |
202 | |
203 | /* skip the gzip header from file in */ |
204 | local void gzhead(bin *in) |
205 | { |
206 | int flags; |
207 | |
208 | /* verify gzip magic header and compression method */ |
209 | if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
210 | bail(in->name, " is not a valid gzip file"); |
211 | |
212 | /* get and verify flags */ |
213 | flags = bget(in); |
214 | if ((flags & 0xe0) != 0) |
215 | bail("unknown reserved bits set in ", in->name); |
216 | |
217 | /* skip modification time, extra flags, and os */ |
218 | bskip(in, 6); |
219 | |
220 | /* skip extra field if present */ |
221 | if (flags & 4) { |
222 | unsigned len; |
223 | |
224 | len = bget(in); |
225 | len += (unsigned)(bget(in)) << 8; |
226 | bskip(in, len); |
227 | } |
228 | |
229 | /* skip file name if present */ |
230 | if (flags & 8) |
231 | while (bget(in) != 0) |
232 | ; |
233 | |
234 | /* skip comment if present */ |
235 | if (flags & 16) |
236 | while (bget(in) != 0) |
237 | ; |
238 | |
239 | /* skip header crc if present */ |
240 | if (flags & 2) |
241 | bskip(in, 2); |
242 | } |
243 | |
244 | /* write a four-byte little-endian unsigned integer to out */ |
245 | local void put4(unsigned long val, FILE *out) |
246 | { |
247 | putc(val & 0xff, out); |
248 | putc((val >> 8) & 0xff, out); |
249 | putc((val >> 16) & 0xff, out); |
250 | putc((val >> 24) & 0xff, out); |
251 | } |
252 | |
253 | /* Load up zlib stream from buffered input, bail if end of file */ |
254 | local void zpull(z_streamp strm, bin *in) |
255 | { |
256 | if (in->left == 0) |
257 | bload(in); |
258 | if (in->left == 0) |
259 | bail("unexpected end of file on ", in->name); |
260 | strm->avail_in = in->left; |
261 | strm->next_in = in->next; |
262 | } |
263 | |
264 | /* Write header for gzip file to out and initialize trailer. */ |
265 | local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
266 | { |
267 | fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
268 | *crc = crc32(0L, Z_NULL, 0); |
269 | *tot = 0; |
270 | } |
271 | |
272 | /* Copy the compressed data from name, zeroing the last block bit of the last |
273 | block if clr is true, and adding empty blocks as needed to get to a byte |
274 | boundary. If clr is false, then the last block becomes the last block of |
275 | the output, and the gzip trailer is written. crc and tot maintains the |
276 | crc and length (modulo 2^32) of the output for the trailer. The resulting |
277 | gzip file is written to out. gzinit() must be called before the first call |
278 | of gzcopy() to write the gzip header and to initialize crc and tot. */ |
279 | local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
280 | FILE *out) |
281 | { |
282 | int ret; /* return value from zlib functions */ |
283 | int pos; /* where the "last block" bit is in byte */ |
284 | int last; /* true if processing the last block */ |
285 | bin *in; /* buffered input file */ |
286 | unsigned char *start; /* start of compressed data in buffer */ |
287 | unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
288 | z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
289 | z_stream strm; /* zlib inflate stream */ |
290 | |
291 | /* open gzip file and skip header */ |
292 | in = bopen(name); |
293 | if (in == NULL) |
294 | bail("could not open ", name); |
295 | gzhead(in); |
296 | |
297 | /* allocate buffer for uncompressed data and initialize raw inflate |
298 | stream */ |
299 | junk = malloc(CHUNK); |
300 | strm.zalloc = Z_NULL; |
301 | strm.zfree = Z_NULL; |
302 | strm.opaque = Z_NULL; |
303 | strm.avail_in = 0; |
304 | strm.next_in = Z_NULL; |
305 | ret = inflateInit2(&strm, -15); |
306 | if (junk == NULL || ret != Z_OK) |
307 | bail("out of memory", ""); |
308 | |
309 | /* inflate and copy compressed data, clear last-block bit if requested */ |
310 | len = 0; |
311 | zpull(&strm, in); |
312 | start = in->next; |
313 | last = start[0] & 1; |
314 | if (last && clr) |
315 | start[0] &= ~1; |
316 | strm.avail_out = 0; |
317 | for (;;) { |
318 | /* if input used and output done, write used input and get more */ |
319 | if (strm.avail_in == 0 && strm.avail_out != 0) { |
320 | fwrite(start, 1, strm.next_in - start, out); |
321 | start = in->buf; |
322 | in->left = 0; |
323 | zpull(&strm, in); |
324 | } |
325 | |
326 | /* decompress -- return early when end-of-block reached */ |
327 | strm.avail_out = CHUNK; |
328 | strm.next_out = junk; |
329 | ret = inflate(&strm, Z_BLOCK); |
330 | switch (ret) { |
331 | case Z_MEM_ERROR: |
332 | bail("out of memory", ""); |
333 | case Z_DATA_ERROR: |
334 | bail("invalid compressed data in ", in->name); |
335 | } |
336 | |
337 | /* update length of uncompressed data */ |
338 | len += CHUNK - strm.avail_out; |
339 | |
340 | /* check for block boundary (only get this when block copied out) */ |
341 | if (strm.data_type & 128) { |
342 | /* if that was the last block, then done */ |
343 | if (last) |
344 | break; |
345 | |
346 | /* number of unused bits in last byte */ |
347 | pos = strm.data_type & 7; |
348 | |
349 | /* find the next last-block bit */ |
350 | if (pos != 0) { |
351 | /* next last-block bit is in last used byte */ |
352 | pos = 0x100 >> pos; |
353 | last = strm.next_in[-1] & pos; |
354 | if (last && clr) |
355 | in->buf[strm.next_in - in->buf - 1] &= ~pos; |
356 | } |
357 | else { |
358 | /* next last-block bit is in next unused byte */ |
359 | if (strm.avail_in == 0) { |
360 | /* don't have that byte yet -- get it */ |
361 | fwrite(start, 1, strm.next_in - start, out); |
362 | start = in->buf; |
363 | in->left = 0; |
364 | zpull(&strm, in); |
365 | } |
366 | last = strm.next_in[0] & 1; |
367 | if (last && clr) |
368 | in->buf[strm.next_in - in->buf] &= ~1; |
369 | } |
370 | } |
371 | } |
372 | |
373 | /* update buffer with unused input */ |
374 | in->left = strm.avail_in; |
375 | in->next = in->buf + (strm.next_in - in->buf); |
376 | |
377 | /* copy used input, write empty blocks to get to byte boundary */ |
378 | pos = strm.data_type & 7; |
379 | fwrite(start, 1, in->next - start - 1, out); |
380 | last = in->next[-1]; |
381 | if (pos == 0 || !clr) |
382 | /* already at byte boundary, or last file: write last byte */ |
383 | putc(last, out); |
384 | else { |
385 | /* append empty blocks to last byte */ |
386 | last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
387 | if (pos & 1) { |
388 | /* odd -- append an empty stored block */ |
389 | putc(last, out); |
390 | if (pos == 1) |
391 | putc(0, out); /* two more bits in block header */ |
392 | fwrite("\0\0\xff\xff", 1, 4, out); |
393 | } |
394 | else { |
395 | /* even -- append 1, 2, or 3 empty fixed blocks */ |
396 | switch (pos) { |
397 | case 6: |
398 | putc(last | 8, out); |
399 | last = 0; |
400 | case 4: |
401 | putc(last | 0x20, out); |
402 | last = 0; |
403 | case 2: |
404 | putc(last | 0x80, out); |
405 | putc(0, out); |
406 | } |
407 | } |
408 | } |
409 | |
410 | /* update crc and tot */ |
411 | *crc = crc32_combine(*crc, bget4(in), len); |
412 | *tot += (unsigned long)len; |
413 | |
414 | /* clean up */ |
415 | inflateEnd(&strm); |
416 | free(junk); |
417 | bclose(in); |
418 | |
419 | /* write trailer if this is the last gzip file */ |
420 | if (!clr) { |
421 | put4(*crc, out); |
422 | put4(*tot, out); |
423 | } |
424 | } |
425 | |
426 | /* join the gzip files on the command line, write result to stdout */ |
427 | int main(int argc, char **argv) |
428 | { |
429 | unsigned long crc, tot; /* running crc and total uncompressed length */ |
430 | |
431 | /* skip command name */ |
432 | argc--; |
433 | argv++; |
434 | |
435 | /* show usage if no arguments */ |
436 | if (argc == 0) { |
437 | fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
438 | stderr); |
439 | return 0; |
440 | } |
441 | |
442 | /* join gzip files on command line and write to stdout */ |
443 | gzinit(&crc, &tot, stdout); |
444 | while (argc--) |
445 | gzcopy(*argv++, argc, &crc, &tot, stdout); |
446 | |
447 | /* done */ |
448 | return 0; |
449 | } |