|
1
|
/* gzjoin -- command to join gzip files into one gzip file |
|
2
|
|
|
3
|
Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
|
4
|
version 1.2, 14 Aug 2012 |
|
5
|
|
|
6
|
This software is provided 'as-is', without any express or implied |
|
7
|
warranty. In no event will the author be held liable for any damages |
|
8
|
arising from the use of this software. |
|
9
|
|
|
10
|
Permission is granted to anyone to use this software for any purpose, |
|
11
|
including commercial applications, and to alter it and redistribute it |
|
12
|
freely, subject to the following restrictions: |
|
13
|
|
|
14
|
1. The origin of this software must not be misrepresented; you must not |
|
15
|
claim that you wrote the original software. If you use this software |
|
16
|
in a product, an acknowledgment in the product documentation would be |
|
17
|
appreciated but is not required. |
|
18
|
2. Altered source versions must be plainly marked as such, and must not be |
|
19
|
misrepresented as being the original software. |
|
20
|
3. This notice may not be removed or altered from any source distribution. |
|
21
|
|
|
22
|
Mark Adler [email protected] |
|
23
|
*/ |
|
24
|
|
|
25
|
/* |
|
26
|
* Change history: |
|
27
|
* |
|
28
|
* 1.0 11 Dec 2004 - First version |
|
29
|
* 1.1 12 Jun 2005 - Changed ssize_t to long for portability |
|
30
|
* 1.2 14 Aug 2012 - Clean up for z_const usage |
|
31
|
*/ |
|
32
|
|
|
33
|
/* |
|
34
|
gzjoin takes one or more gzip files on the command line and writes out a |
|
35
|
single gzip file that will uncompress to the concatenation of the |
|
36
|
uncompressed data from the individual gzip files. gzjoin does this without |
|
37
|
having to recompress any of the data and without having to calculate a new |
|
38
|
crc32 for the concatenated uncompressed data. gzjoin does however have to |
|
39
|
decompress all of the input data in order to find the bits in the compressed |
|
40
|
data that need to be modified to concatenate the streams. |
|
41
|
|
|
42
|
gzjoin does not do an integrity check on the input gzip files other than |
|
43
|
checking the gzip header and decompressing the compressed data. They are |
|
44
|
otherwise assumed to be complete and correct. |
|
45
|
|
|
46
|
Each joint between gzip files removes at least 18 bytes of previous trailer |
|
47
|
and subsequent header, and inserts an average of about three bytes to the |
|
48
|
compressed data in order to connect the streams. The output gzip file |
|
49
|
has a minimal ten-byte gzip header with no file name or modification time. |
|
50
|
|
|
51
|
This program was written to illustrate the use of the Z_BLOCK option of |
|
52
|
inflate() and the crc32_combine() function. gzjoin will not compile with |
|
53
|
versions of zlib earlier than 1.2.3. |
|
54
|
*/ |
|
55
|
|
|
56
|
#include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
|
57
|
#include <stdlib.h> /* exit(), malloc(), free() */ |
|
58
|
#include <fcntl.h> /* open() */ |
|
59
|
#include <unistd.h> /* close(), read(), lseek() */ |
|
60
|
#include "zlib.h" |
|
61
|
/* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ |
|
62
|
|
|
63
|
#define local static |
|
64
|
|
|
65
|
/* exit with an error (return a value to allow use in an expression) */ |
|
66
|
local int bail(char *why1, char *why2) |
|
67
|
{ |
|
68
|
fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
|
69
|
exit(1); |
|
70
|
return 0; |
|
71
|
} |
|
72
|
|
|
73
|
/* -- simple buffered file input with access to the buffer -- */ |
|
74
|
|
|
75
|
#define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
|
76
|
|
|
77
|
/* bin buffered input file type */ |
|
78
|
typedef struct { |
|
79
|
char *name; /* name of file for error messages */ |
|
80
|
int fd; /* file descriptor */ |
|
81
|
unsigned left; /* bytes remaining at next */ |
|
82
|
unsigned char *next; /* next byte to read */ |
|
83
|
unsigned char *buf; /* allocated buffer of length CHUNK */ |
|
84
|
} bin; |
|
85
|
|
|
86
|
/* close a buffered file and free allocated memory */ |
|
87
|
local void bclose(bin *in) |
|
88
|
{ |
|
89
|
if (in != NULL) { |
|
90
|
if (in->fd != -1) |
|
91
|
close(in->fd); |
|
92
|
if (in->buf != NULL) |
|
93
|
free(in->buf); |
|
94
|
free(in); |
|
95
|
} |
|
96
|
} |
|
97
|
|
|
98
|
/* open a buffered file for input, return a pointer to type bin, or NULL on |
|
99
|
failure */ |
|
100
|
local bin *bopen(char *name) |
|
101
|
{ |
|
102
|
bin *in; |
|
103
|
|
|
104
|
in = malloc(sizeof(bin)); |
|
105
|
if (in == NULL) |
|
106
|
return NULL; |
|
107
|
in->buf = malloc(CHUNK); |
|
108
|
in->fd = open(name, O_RDONLY, 0); |
|
109
|
if (in->buf == NULL || in->fd == -1) { |
|
110
|
bclose(in); |
|
111
|
return NULL; |
|
112
|
} |
|
113
|
in->left = 0; |
|
114
|
in->next = in->buf; |
|
115
|
in->name = name; |
|
116
|
return in; |
|
117
|
} |
|
118
|
|
|
119
|
/* load buffer from file, return -1 on read error, 0 or 1 on success, with |
|
120
|
1 indicating that end-of-file was reached */ |
|
121
|
local int bload(bin *in) |
|
122
|
{ |
|
123
|
long len; |
|
124
|
|
|
125
|
if (in == NULL) |
|
126
|
return -1; |
|
127
|
if (in->left != 0) |
|
128
|
return 0; |
|
129
|
in->next = in->buf; |
|
130
|
do { |
|
131
|
len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
|
132
|
if (len < 0) |
|
133
|
return -1; |
|
134
|
in->left += (unsigned)len; |
|
135
|
} while (len != 0 && in->left < CHUNK); |
|
136
|
return len == 0 ? 1 : 0; |
|
137
|
} |
|
138
|
|
|
139
|
/* get a byte from the file, bail if end of file */ |
|
140
|
#define bget(in) (in->left ? 0 : bload(in), \ |
|
141
|
in->left ? (in->left--, *(in->next)++) : \ |
|
142
|
bail("unexpected end of file on ", in->name)) |
|
143
|
|
|
144
|
/* get a four-byte little-endian unsigned integer from file */ |
|
145
|
local unsigned long bget4(bin *in) |
|
146
|
{ |
|
147
|
unsigned long val; |
|
148
|
|
|
149
|
val = bget(in); |
|
150
|
val += (unsigned long)(bget(in)) << 8; |
|
151
|
val += (unsigned long)(bget(in)) << 16; |
|
152
|
val += (unsigned long)(bget(in)) << 24; |
|
153
|
return val; |
|
154
|
} |
|
155
|
|
|
156
|
/* skip bytes in file */ |
|
157
|
local void bskip(bin *in, unsigned skip) |
|
158
|
{ |
|
159
|
/* check pointer */ |
|
160
|
if (in == NULL) |
|
161
|
return; |
|
162
|
|
|
163
|
/* easy case -- skip bytes in buffer */ |
|
164
|
if (skip <= in->left) { |
|
165
|
in->left -= skip; |
|
166
|
in->next += skip; |
|
167
|
return; |
|
168
|
} |
|
169
|
|
|
170
|
/* skip what's in buffer, discard buffer contents */ |
|
171
|
skip -= in->left; |
|
172
|
in->left = 0; |
|
173
|
|
|
174
|
/* seek past multiples of CHUNK bytes */ |
|
175
|
if (skip > CHUNK) { |
|
176
|
unsigned left; |
|
177
|
|
|
178
|
left = skip & (CHUNK - 1); |
|
179
|
if (left == 0) { |
|
180
|
/* exact number of chunks: seek all the way minus one byte to check |
|
181
|
for end-of-file with a read */ |
|
182
|
lseek(in->fd, skip - 1, SEEK_CUR); |
|
183
|
if (read(in->fd, in->buf, 1) != 1) |
|
184
|
bail("unexpected end of file on ", in->name); |
|
185
|
return; |
|
186
|
} |
|
187
|
|
|
188
|
/* skip the integral chunks, update skip with remainder */ |
|
189
|
lseek(in->fd, skip - left, SEEK_CUR); |
|
190
|
skip = left; |
|
191
|
} |
|
192
|
|
|
193
|
/* read more input and skip remainder */ |
|
194
|
bload(in); |
|
195
|
if (skip > in->left) |
|
196
|
bail("unexpected end of file on ", in->name); |
|
197
|
in->left -= skip; |
|
198
|
in->next += skip; |
|
199
|
} |
|
200
|
|
|
201
|
/* -- end of buffered input functions -- */ |
|
202
|
|
|
203
|
/* skip the gzip header from file in */ |
|
204
|
local void gzhead(bin *in) |
|
205
|
{ |
|
206
|
int flags; |
|
207
|
|
|
208
|
/* verify gzip magic header and compression method */ |
|
209
|
if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
|
210
|
bail(in->name, " is not a valid gzip file"); |
|
211
|
|
|
212
|
/* get and verify flags */ |
|
213
|
flags = bget(in); |
|
214
|
if ((flags & 0xe0) != 0) |
|
215
|
bail("unknown reserved bits set in ", in->name); |
|
216
|
|
|
217
|
/* skip modification time, extra flags, and os */ |
|
218
|
bskip(in, 6); |
|
219
|
|
|
220
|
/* skip extra field if present */ |
|
221
|
if (flags & 4) { |
|
222
|
unsigned len; |
|
223
|
|
|
224
|
len = bget(in); |
|
225
|
len += (unsigned)(bget(in)) << 8; |
|
226
|
bskip(in, len); |
|
227
|
} |
|
228
|
|
|
229
|
/* skip file name if present */ |
|
230
|
if (flags & 8) |
|
231
|
while (bget(in) != 0) |
|
232
|
; |
|
233
|
|
|
234
|
/* skip comment if present */ |
|
235
|
if (flags & 16) |
|
236
|
while (bget(in) != 0) |
|
237
|
; |
|
238
|
|
|
239
|
/* skip header crc if present */ |
|
240
|
if (flags & 2) |
|
241
|
bskip(in, 2); |
|
242
|
} |
|
243
|
|
|
244
|
/* write a four-byte little-endian unsigned integer to out */ |
|
245
|
local void put4(unsigned long val, FILE *out) |
|
246
|
{ |
|
247
|
putc(val & 0xff, out); |
|
248
|
putc((val >> 8) & 0xff, out); |
|
249
|
putc((val >> 16) & 0xff, out); |
|
250
|
putc((val >> 24) & 0xff, out); |
|
251
|
} |
|
252
|
|
|
253
|
/* Load up zlib stream from buffered input, bail if end of file */ |
|
254
|
local void zpull(z_streamp strm, bin *in) |
|
255
|
{ |
|
256
|
if (in->left == 0) |
|
257
|
bload(in); |
|
258
|
if (in->left == 0) |
|
259
|
bail("unexpected end of file on ", in->name); |
|
260
|
strm->avail_in = in->left; |
|
261
|
strm->next_in = in->next; |
|
262
|
} |
|
263
|
|
|
264
|
/* Write header for gzip file to out and initialize trailer. */ |
|
265
|
local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
|
266
|
{ |
|
267
|
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
|
268
|
*crc = crc32(0L, Z_NULL, 0); |
|
269
|
*tot = 0; |
|
270
|
} |
|
271
|
|
|
272
|
/* Copy the compressed data from name, zeroing the last block bit of the last |
|
273
|
block if clr is true, and adding empty blocks as needed to get to a byte |
|
274
|
boundary. If clr is false, then the last block becomes the last block of |
|
275
|
the output, and the gzip trailer is written. crc and tot maintains the |
|
276
|
crc and length (modulo 2^32) of the output for the trailer. The resulting |
|
277
|
gzip file is written to out. gzinit() must be called before the first call |
|
278
|
of gzcopy() to write the gzip header and to initialize crc and tot. */ |
|
279
|
local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
|
280
|
FILE *out) |
|
281
|
{ |
|
282
|
int ret; /* return value from zlib functions */ |
|
283
|
int pos; /* where the "last block" bit is in byte */ |
|
284
|
int last; /* true if processing the last block */ |
|
285
|
bin *in; /* buffered input file */ |
|
286
|
unsigned char *start; /* start of compressed data in buffer */ |
|
287
|
unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
|
288
|
z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
|
289
|
z_stream strm; /* zlib inflate stream */ |
|
290
|
|
|
291
|
/* open gzip file and skip header */ |
|
292
|
in = bopen(name); |
|
293
|
if (in == NULL) |
|
294
|
bail("could not open ", name); |
|
295
|
gzhead(in); |
|
296
|
|
|
297
|
/* allocate buffer for uncompressed data and initialize raw inflate |
|
298
|
stream */ |
|
299
|
junk = malloc(CHUNK); |
|
300
|
strm.zalloc = Z_NULL; |
|
301
|
strm.zfree = Z_NULL; |
|
302
|
strm.opaque = Z_NULL; |
|
303
|
strm.avail_in = 0; |
|
304
|
strm.next_in = Z_NULL; |
|
305
|
ret = inflateInit2(&strm, -15); |
|
306
|
if (junk == NULL || ret != Z_OK) |
|
307
|
bail("out of memory", ""); |
|
308
|
|
|
309
|
/* inflate and copy compressed data, clear last-block bit if requested */ |
|
310
|
len = 0; |
|
311
|
zpull(&strm, in); |
|
312
|
start = in->next; |
|
313
|
last = start[0] & 1; |
|
314
|
if (last && clr) |
|
315
|
start[0] &= ~1; |
|
316
|
strm.avail_out = 0; |
|
317
|
for (;;) { |
|
318
|
/* if input used and output done, write used input and get more */ |
|
319
|
if (strm.avail_in == 0 && strm.avail_out != 0) { |
|
320
|
fwrite(start, 1, strm.next_in - start, out); |
|
321
|
start = in->buf; |
|
322
|
in->left = 0; |
|
323
|
zpull(&strm, in); |
|
324
|
} |
|
325
|
|
|
326
|
/* decompress -- return early when end-of-block reached */ |
|
327
|
strm.avail_out = CHUNK; |
|
328
|
strm.next_out = junk; |
|
329
|
ret = inflate(&strm, Z_BLOCK); |
|
330
|
switch (ret) { |
|
331
|
case Z_MEM_ERROR: |
|
332
|
bail("out of memory", ""); |
|
333
|
case Z_DATA_ERROR: |
|
334
|
bail("invalid compressed data in ", in->name); |
|
335
|
} |
|
336
|
|
|
337
|
/* update length of uncompressed data */ |
|
338
|
len += CHUNK - strm.avail_out; |
|
339
|
|
|
340
|
/* check for block boundary (only get this when block copied out) */ |
|
341
|
if (strm.data_type & 128) { |
|
342
|
/* if that was the last block, then done */ |
|
343
|
if (last) |
|
344
|
break; |
|
345
|
|
|
346
|
/* number of unused bits in last byte */ |
|
347
|
pos = strm.data_type & 7; |
|
348
|
|
|
349
|
/* find the next last-block bit */ |
|
350
|
if (pos != 0) { |
|
351
|
/* next last-block bit is in last used byte */ |
|
352
|
pos = 0x100 >> pos; |
|
353
|
last = strm.next_in[-1] & pos; |
|
354
|
if (last && clr) |
|
355
|
in->buf[strm.next_in - in->buf - 1] &= ~pos; |
|
356
|
} |
|
357
|
else { |
|
358
|
/* next last-block bit is in next unused byte */ |
|
359
|
if (strm.avail_in == 0) { |
|
360
|
/* don't have that byte yet -- get it */ |
|
361
|
fwrite(start, 1, strm.next_in - start, out); |
|
362
|
start = in->buf; |
|
363
|
in->left = 0; |
|
364
|
zpull(&strm, in); |
|
365
|
} |
|
366
|
last = strm.next_in[0] & 1; |
|
367
|
if (last && clr) |
|
368
|
in->buf[strm.next_in - in->buf] &= ~1; |
|
369
|
} |
|
370
|
} |
|
371
|
} |
|
372
|
|
|
373
|
/* update buffer with unused input */ |
|
374
|
in->left = strm.avail_in; |
|
375
|
in->next = in->buf + (strm.next_in - in->buf); |
|
376
|
|
|
377
|
/* copy used input, write empty blocks to get to byte boundary */ |
|
378
|
pos = strm.data_type & 7; |
|
379
|
fwrite(start, 1, in->next - start - 1, out); |
|
380
|
last = in->next[-1]; |
|
381
|
if (pos == 0 || !clr) |
|
382
|
/* already at byte boundary, or last file: write last byte */ |
|
383
|
putc(last, out); |
|
384
|
else { |
|
385
|
/* append empty blocks to last byte */ |
|
386
|
last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
|
387
|
if (pos & 1) { |
|
388
|
/* odd -- append an empty stored block */ |
|
389
|
putc(last, out); |
|
390
|
if (pos == 1) |
|
391
|
putc(0, out); /* two more bits in block header */ |
|
392
|
fwrite("\0\0\xff\xff", 1, 4, out); |
|
393
|
} |
|
394
|
else { |
|
395
|
/* even -- append 1, 2, or 3 empty fixed blocks */ |
|
396
|
switch (pos) { |
|
397
|
case 6: |
|
398
|
putc(last | 8, out); |
|
399
|
last = 0; |
|
400
|
case 4: |
|
401
|
putc(last | 0x20, out); |
|
402
|
last = 0; |
|
403
|
case 2: |
|
404
|
putc(last | 0x80, out); |
|
405
|
putc(0, out); |
|
406
|
} |
|
407
|
} |
|
408
|
} |
|
409
|
|
|
410
|
/* update crc and tot */ |
|
411
|
*crc = crc32_combine(*crc, bget4(in), len); |
|
412
|
*tot += (unsigned long)len; |
|
413
|
|
|
414
|
/* clean up */ |
|
415
|
inflateEnd(&strm); |
|
416
|
free(junk); |
|
417
|
bclose(in); |
|
418
|
|
|
419
|
/* write trailer if this is the last gzip file */ |
|
420
|
if (!clr) { |
|
421
|
put4(*crc, out); |
|
422
|
put4(*tot, out); |
|
423
|
} |
|
424
|
} |
|
425
|
|
|
426
|
/* join the gzip files on the command line, write result to stdout */ |
|
427
|
int main(int argc, char **argv) |
|
428
|
{ |
|
429
|
unsigned long crc, tot; /* running crc and total uncompressed length */ |
|
430
|
|
|
431
|
/* skip command name */ |
|
432
|
argc--; |
|
433
|
argv++; |
|
434
|
|
|
435
|
/* show usage if no arguments */ |
|
436
|
if (argc == 0) { |
|
437
|
fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
|
438
|
stderr); |
|
439
|
return 0; |
|
440
|
} |
|
441
|
|
|
442
|
/* join gzip files on command line and write to stdout */ |
|
443
|
gzinit(&crc, &tot, stdout); |
|
444
|
while (argc--) |
|
445
|
gzcopy(*argv++, argc, &crc, &tot, stdout); |
|
446
|
|
|
447
|
/* done */ |
|
448
|
return 0; |
|
449
|
} |
|
450
|
|