1/**
2 * xzlib.c: front end for the transparent suport of lzma compression
3 *          at the I/O layer, based on an example file from lzma project
4 *
5 * See Copyright for the status of this software.
6 *
7 * Anders F Bjorklund <afb@users.sourceforge.net>
8 */
9#define IN_LIBXML
10#include "libxml.h"
11#ifdef HAVE_LZMA_H
12
13#include <string.h>
14#ifdef HAVE_ERRNO_H
15#include <errno.h>
16#endif
17
18
19#ifdef HAVE_SYS_TYPES_H
20#include <sys/types.h>
21#endif
22#ifdef HAVE_SYS_STAT_H
23#include <sys/stat.h>
24#endif
25#ifdef HAVE_FCNTL_H
26#include <fcntl.h>
27#endif
28#ifdef HAVE_UNISTD_H
29#include <unistd.h>
30#endif
31#ifdef HAVE_STDLIB_H
32#include <stdlib.h>
33#endif
34#ifdef HAVE_ZLIB_H
35#include <zlib.h>
36#endif
37#include <lzma.h>
38
39#include "xzlib.h"
40#include <libxml/xmlmemory.h>
41
42/* values for xz_state how */
43#define LOOK 0                  /* look for a gzip/lzma header */
44#define COPY 1                  /* copy input directly */
45#define GZIP 2                  /* decompress a gzip stream */
46#define LZMA 3                  /* decompress a lzma stream */
47
48/* internal lzma file state data structure */
49typedef struct {
50    int mode;                   /* see lzma modes above */
51    int fd;                     /* file descriptor */
52    char *path;                 /* path or fd for error messages */
53    uint64_t pos;               /* current position in uncompressed data */
54    unsigned int size;          /* buffer size, zero if not allocated yet */
55    unsigned int want;          /* requested buffer size, default is BUFSIZ */
56    unsigned char *in;          /* input buffer */
57    unsigned char *out;         /* output buffer (double-sized when reading) */
58    unsigned char *next;        /* next output data to deliver or write */
59    unsigned int have;          /* amount of output data unused at next */
60    int eof;                    /* true if end of input file reached */
61    uint64_t start;             /* where the lzma data started, for rewinding */
62    uint64_t raw;               /* where the raw data started, for seeking */
63    int how;                    /* 0: get header, 1: copy, 2: decompress */
64    int direct;                 /* true if last read direct, false if lzma */
65    /* seek request */
66    uint64_t skip;              /* amount to skip (already rewound if backwards) */
67    int seek;                   /* true if seek request pending */
68    /* error information */
69    int err;                    /* error code */
70    char *msg;                  /* error message */
71    /* lzma stream */
72    int init;                   /* is the iniflate stream initialized */
73    lzma_stream strm;           /* stream structure in-place (not a pointer) */
74    char padding1[32];          /* padding allowing to cope with possible
75                                   extensions of above structure without
76				   too much side effect */
77#ifdef HAVE_ZLIB_H
78    /* zlib inflate or deflate stream */
79    z_stream zstrm;             /* stream structure in-place (not a pointer) */
80#endif
81    char padding2[32];          /* padding allowing to cope with possible
82                                   extensions of above structure without
83				   too much side effect */
84} xz_state, *xz_statep;
85
86static void
87xz_error(xz_statep state, int err, const char *msg)
88{
89    /* free previously allocated message and clear */
90    if (state->msg != NULL) {
91        if (state->err != LZMA_MEM_ERROR)
92            xmlFree(state->msg);
93        state->msg = NULL;
94    }
95
96    /* set error code, and if no message, then done */
97    state->err = err;
98    if (msg == NULL)
99        return;
100
101    /* for an out of memory error, save as static string */
102    if (err == LZMA_MEM_ERROR) {
103        state->msg = (char *) msg;
104        return;
105    }
106
107    /* construct error message with path */
108    if ((state->msg =
109         xmlMalloc(strlen(state->path) + strlen(msg) + 3)) == NULL) {
110        state->err = LZMA_MEM_ERROR;
111        state->msg = (char *) "out of memory";
112        return;
113    }
114    strcpy(state->msg, state->path);
115    strcat(state->msg, ": ");
116    strcat(state->msg, msg);
117    return;
118}
119
120static void
121xz_reset(xz_statep state)
122{
123    state->have = 0;            /* no output data available */
124    state->eof = 0;             /* not at end of file */
125    state->how = LOOK;          /* look for gzip header */
126    state->direct = 1;          /* default for empty file */
127    state->seek = 0;            /* no seek request pending */
128    xz_error(state, LZMA_OK, NULL);     /* clear error */
129    state->pos = 0;             /* no uncompressed data yet */
130    state->strm.avail_in = 0;   /* no input data yet */
131#ifdef HAVE_ZLIB_H
132    state->zstrm.avail_in = 0;  /* no input data yet */
133#endif
134}
135
136static xzFile
137xz_open(const char *path, int fd, const char *mode ATTRIBUTE_UNUSED)
138{
139    xz_statep state;
140
141    /* allocate xzFile structure to return */
142    state = xmlMalloc(sizeof(xz_state));
143    if (state == NULL)
144        return NULL;
145    state->size = 0;            /* no buffers allocated yet */
146    state->want = BUFSIZ;       /* requested buffer size */
147    state->msg = NULL;          /* no error message yet */
148    state->init = 0;            /* initialization of zlib data */
149
150    /* save the path name for error messages */
151    state->path = xmlMalloc(strlen(path) + 1);
152    if (state->path == NULL) {
153        xmlFree(state);
154        return NULL;
155    }
156    strcpy(state->path, path);
157
158    /* open the file with the appropriate mode (or just use fd) */
159    state->fd = fd != -1 ? fd : open(path,
160#ifdef O_LARGEFILE
161                                     O_LARGEFILE |
162#endif
163#ifdef O_BINARY
164                                     O_BINARY |
165#endif
166                                     O_RDONLY, 0666);
167    if (state->fd == -1) {
168        xmlFree(state->path);
169        xmlFree(state);
170        return NULL;
171    }
172
173    /* save the current position for rewinding (only if reading) */
174    state->start = lseek(state->fd, 0, SEEK_CUR);
175    if (state->start == (uint64_t) - 1)
176        state->start = 0;
177
178    /* initialize stream */
179    xz_reset(state);
180
181    /* return stream */
182    return (xzFile) state;
183}
184
185xzFile
186__libxml2_xzopen(const char *path, const char *mode)
187{
188    return xz_open(path, -1, mode);
189}
190
191xzFile
192__libxml2_xzdopen(int fd, const char *mode)
193{
194    char *path;                 /* identifier for error messages */
195    xzFile xz;
196
197    if (fd == -1 || (path = xmlMalloc(7 + 3 * sizeof(int))) == NULL)
198        return NULL;
199    sprintf(path, "<fd:%d>", fd);       /* for debugging */
200    xz = xz_open(path, fd, mode);
201    xmlFree(path);
202    return xz;
203}
204
205static int
206xz_load(xz_statep state, unsigned char *buf, unsigned int len,
207        unsigned int *have)
208{
209    int ret;
210
211    *have = 0;
212    do {
213        ret = read(state->fd, buf + *have, len - *have);
214        if (ret <= 0)
215            break;
216        *have += ret;
217    } while (*have < len);
218    if (ret < 0) {
219        xz_error(state, -1, strerror(errno));
220        return -1;
221    }
222    if (ret == 0)
223        state->eof = 1;
224    return 0;
225}
226
227static int
228xz_avail(xz_statep state)
229{
230    lzma_stream *strm = &(state->strm);
231
232    if (state->err != LZMA_OK)
233        return -1;
234    if (state->eof == 0) {
235        /* avail_in is size_t, which is not necessary sizeof(unsigned) */
236        unsigned tmp = strm->avail_in;
237
238        if (xz_load(state, state->in, state->size, &tmp) == -1) {
239            strm->avail_in = tmp;
240            return -1;
241        }
242        strm->avail_in = tmp;
243        strm->next_in = state->in;
244    }
245    return 0;
246}
247
248static int
249is_format_xz(xz_statep state)
250{
251    lzma_stream *strm = &(state->strm);
252
253    return strm->avail_in >= 6 && memcmp(state->in, "\3757zXZ", 6) == 0;
254}
255
256static int
257is_format_lzma(xz_statep state)
258{
259    lzma_stream *strm = &(state->strm);
260
261    lzma_filter filter;
262    lzma_options_lzma *opt;
263    uint32_t dict_size;
264    uint64_t uncompressed_size;
265    size_t i;
266
267    if (strm->avail_in < 13)
268        return 0;
269
270    filter.id = LZMA_FILTER_LZMA1;
271    if (lzma_properties_decode(&filter, NULL, state->in, 5) != LZMA_OK)
272        return 0;
273
274    opt = filter.options;
275    dict_size = opt->dict_size;
276    free(opt); /* we can't use xmlFree on a string returned by zlib */
277
278    /* A hack to ditch tons of false positives: We allow only dictionary
279     * sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
280     * created only files with 2^n, but accepts any dictionary size.
281     * If someone complains, this will be reconsidered.
282     */
283    if (dict_size != UINT32_MAX) {
284        uint32_t d = dict_size - 1;
285
286        d |= d >> 2;
287        d |= d >> 3;
288        d |= d >> 4;
289        d |= d >> 8;
290        d |= d >> 16;
291        ++d;
292        if (d != dict_size || dict_size == 0)
293            return 0;
294    }
295
296    /* Another hack to ditch false positives: Assume that if the
297     * uncompressed size is known, it must be less than 256 GiB.
298     * Again, if someone complains, this will be reconsidered.
299     */
300    uncompressed_size = 0;
301    for (i = 0; i < 8; ++i)
302        uncompressed_size |= (uint64_t) (state->in[5 + i]) << (i * 8);
303
304    if (uncompressed_size != UINT64_MAX
305        && uncompressed_size > (UINT64_C(1) << 38))
306        return 0;
307
308    return 1;
309}
310
311#ifdef HAVE_ZLIB_H
312
313/* Get next byte from input, or -1 if end or error. */
314#define NEXT() ((strm->avail_in == 0 && xz_avail(state) == -1) ? -1 : \
315                (strm->avail_in == 0 ? -1 : \
316                 (strm->avail_in--, *(strm->next_in)++)))
317
318/* Get a four-byte little-endian integer and return 0 on success and the value
319   in *ret.  Otherwise -1 is returned and *ret is not modified. */
320static int
321gz_next4(xz_statep state, unsigned long *ret)
322{
323    int ch;
324    unsigned long val;
325    z_streamp strm = &(state->zstrm);
326
327    val = NEXT();
328    val += (unsigned) NEXT() << 8;
329    val += (unsigned long) NEXT() << 16;
330    ch = NEXT();
331    if (ch == -1)
332        return -1;
333    val += (unsigned long) ch << 24;
334    *ret = val;
335    return 0;
336}
337#endif
338
339static int
340xz_head(xz_statep state)
341{
342    lzma_stream *strm = &(state->strm);
343    lzma_stream init = LZMA_STREAM_INIT;
344    int flags;
345    unsigned len;
346
347    /* allocate read buffers and inflate memory */
348    if (state->size == 0) {
349        /* allocate buffers */
350        state->in = xmlMalloc(state->want);
351        state->out = xmlMalloc(state->want << 1);
352        if (state->in == NULL || state->out == NULL) {
353            if (state->out != NULL)
354                xmlFree(state->out);
355            if (state->in != NULL)
356                xmlFree(state->in);
357            xz_error(state, LZMA_MEM_ERROR, "out of memory");
358            return -1;
359        }
360        state->size = state->want;
361
362        /* allocate decoder memory */
363        state->strm = init;
364        state->strm.avail_in = 0;
365        state->strm.next_in = NULL;
366        if (lzma_auto_decoder(&state->strm, UINT64_MAX, 0) != LZMA_OK) {
367            xmlFree(state->out);
368            xmlFree(state->in);
369            state->size = 0;
370            xz_error(state, LZMA_MEM_ERROR, "out of memory");
371            return -1;
372        }
373#ifdef HAVE_ZLIB_H
374        /* allocate inflate memory */
375        state->zstrm.zalloc = Z_NULL;
376        state->zstrm.zfree = Z_NULL;
377        state->zstrm.opaque = Z_NULL;
378        state->zstrm.avail_in = 0;
379        state->zstrm.next_in = Z_NULL;
380        if (state->init == 0) {
381            if (inflateInit2(&(state->zstrm), -15) != Z_OK) {/* raw inflate */
382                xmlFree(state->out);
383                xmlFree(state->in);
384                state->size = 0;
385                xz_error(state, LZMA_MEM_ERROR, "out of memory");
386                return -1;
387            }
388            state->init = 1;
389        }
390#endif
391    }
392
393    /* get some data in the input buffer */
394    if (strm->avail_in == 0) {
395        if (xz_avail(state) == -1)
396            return -1;
397        if (strm->avail_in == 0)
398            return 0;
399    }
400
401    /* look for the xz magic header bytes */
402    if (is_format_xz(state) || is_format_lzma(state)) {
403        state->how = LZMA;
404        state->direct = 0;
405        return 0;
406    }
407#ifdef HAVE_ZLIB_H
408    /* look for the gzip magic header bytes 31 and 139 */
409    if (strm->next_in[0] == 31) {
410        strm->avail_in--;
411        strm->next_in++;
412        if (strm->avail_in == 0 && xz_avail(state) == -1)
413            return -1;
414        if (strm->avail_in && strm->next_in[0] == 139) {
415            /* we have a gzip header, woo hoo! */
416            strm->avail_in--;
417            strm->next_in++;
418
419            /* skip rest of header */
420            if (NEXT() != 8) {  /* compression method */
421                xz_error(state, LZMA_DATA_ERROR,
422                         "unknown compression method");
423                return -1;
424            }
425            flags = NEXT();
426            if (flags & 0xe0) { /* reserved flag bits */
427                xz_error(state, LZMA_DATA_ERROR,
428                         "unknown header flags set");
429                return -1;
430            }
431            NEXT();             /* modification time */
432            NEXT();
433            NEXT();
434            NEXT();
435            NEXT();             /* extra flags */
436            NEXT();             /* operating system */
437            if (flags & 4) {    /* extra field */
438                len = (unsigned) NEXT();
439                len += (unsigned) NEXT() << 8;
440                while (len--)
441                    if (NEXT() < 0)
442                        break;
443            }
444            if (flags & 8)      /* file name */
445                while (NEXT() > 0) ;
446            if (flags & 16)     /* comment */
447                while (NEXT() > 0) ;
448            if (flags & 2) {    /* header crc */
449                NEXT();
450                NEXT();
451            }
452            /* an unexpected end of file is not checked for here -- it will be
453             * noticed on the first request for uncompressed data */
454
455            /* set up for decompression */
456            inflateReset(&state->zstrm);
457            state->zstrm.adler = crc32(0L, Z_NULL, 0);
458            state->how = GZIP;
459            state->direct = 0;
460            return 0;
461        } else {
462            /* not a gzip file -- save first byte (31) and fall to raw i/o */
463            state->out[0] = 31;
464            state->have = 1;
465        }
466    }
467#endif
468
469    /* doing raw i/o, save start of raw data for seeking, copy any leftover
470     * input to output -- this assumes that the output buffer is larger than
471     * the input buffer, which also assures space for gzungetc() */
472    state->raw = state->pos;
473    state->next = state->out;
474    if (strm->avail_in) {
475        memcpy(state->next + state->have, strm->next_in, strm->avail_in);
476        state->have += strm->avail_in;
477        strm->avail_in = 0;
478    }
479    state->how = COPY;
480    state->direct = 1;
481    return 0;
482}
483
484static int
485xz_decomp(xz_statep state)
486{
487    int ret;
488    unsigned had;
489    unsigned long crc, len;
490    lzma_stream *strm = &(state->strm);
491
492    lzma_action action = LZMA_RUN;
493
494    /* fill output buffer up to end of deflate stream */
495    had = strm->avail_out;
496    do {
497        /* get more input for inflate() */
498        if (strm->avail_in == 0 && xz_avail(state) == -1)
499            return -1;
500        if (strm->avail_in == 0) {
501            xz_error(state, LZMA_DATA_ERROR, "unexpected end of file");
502            return -1;
503        }
504        if (state->eof)
505            action = LZMA_FINISH;
506
507        /* decompress and handle errors */
508#ifdef HAVE_ZLIB_H
509        if (state->how == GZIP) {
510            state->zstrm.avail_in = (uInt) state->strm.avail_in;
511            state->zstrm.next_in = (Bytef *) state->strm.next_in;
512            state->zstrm.avail_out = (uInt) state->strm.avail_out;
513            state->zstrm.next_out = (Bytef *) state->strm.next_out;
514            ret = inflate(&state->zstrm, Z_NO_FLUSH);
515            if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) {
516                xz_error(state, Z_STREAM_ERROR,
517                         "internal error: inflate stream corrupt");
518                return -1;
519            }
520            if (ret == Z_MEM_ERROR)
521                ret = LZMA_MEM_ERROR;
522            if (ret == Z_DATA_ERROR)
523                ret = LZMA_DATA_ERROR;
524            if (ret == Z_STREAM_END)
525                ret = LZMA_STREAM_END;
526            state->strm.avail_in = state->zstrm.avail_in;
527            state->strm.next_in = state->zstrm.next_in;
528            state->strm.avail_out = state->zstrm.avail_out;
529            state->strm.next_out = state->zstrm.next_out;
530        } else                  /* state->how == LZMA */
531#endif
532            ret = lzma_code(strm, action);
533        if (ret == LZMA_MEM_ERROR) {
534            xz_error(state, LZMA_MEM_ERROR, "out of memory");
535            return -1;
536        }
537        if (ret == LZMA_DATA_ERROR) {
538            xz_error(state, LZMA_DATA_ERROR, "compressed data error");
539            return -1;
540        }
541    } while (strm->avail_out && ret != LZMA_STREAM_END);
542
543    /* update available output and crc check value */
544    state->have = had - strm->avail_out;
545    state->next = strm->next_out - state->have;
546#ifdef HAVE_ZLIB_H
547    state->zstrm.adler =
548        crc32(state->zstrm.adler, state->next, state->have);
549#endif
550
551    if (ret == LZMA_STREAM_END) {
552#ifdef HAVE_ZLIB_H
553        if (state->how == GZIP) {
554            if (gz_next4(state, &crc) == -1 || gz_next4(state, &len) == -1) {
555                xz_error(state, LZMA_DATA_ERROR, "unexpected end of file");
556                return -1;
557            }
558            if (crc != state->zstrm.adler) {
559                xz_error(state, LZMA_DATA_ERROR, "incorrect data check");
560                return -1;
561            }
562            if (len != (state->zstrm.total_out & 0xffffffffL)) {
563                xz_error(state, LZMA_DATA_ERROR, "incorrect length check");
564                return -1;
565            }
566            state->strm.avail_in = 0;
567            state->strm.next_in = NULL;
568            state->strm.avail_out = 0;
569            state->strm.next_out = NULL;
570        } else
571#endif
572        if (strm->avail_in != 0 || !state->eof) {
573            xz_error(state, LZMA_DATA_ERROR, "trailing garbage");
574            return -1;
575        }
576        state->how = LOOK;      /* ready for next stream, once have is 0 (leave
577                                 * state->direct unchanged to remember how) */
578    }
579
580    /* good decompression */
581    return 0;
582}
583
584static int
585xz_make(xz_statep state)
586{
587    lzma_stream *strm = &(state->strm);
588
589    if (state->how == LOOK) {   /* look for lzma / gzip header */
590        if (xz_head(state) == -1)
591            return -1;
592        if (state->have)        /* got some data from xz_head() */
593            return 0;
594    }
595    if (state->how == COPY) {   /* straight copy */
596        if (xz_load(state, state->out, state->size << 1, &(state->have)) ==
597            -1)
598            return -1;
599        state->next = state->out;
600    } else if (state->how == LZMA || state->how == GZIP) {      /* decompress */
601        strm->avail_out = state->size << 1;
602        strm->next_out = state->out;
603        if (xz_decomp(state) == -1)
604            return -1;
605    }
606    return 0;
607}
608
609static int
610xz_skip(xz_statep state, uint64_t len)
611{
612    unsigned n;
613
614    /* skip over len bytes or reach end-of-file, whichever comes first */
615    while (len)
616        /* skip over whatever is in output buffer */
617        if (state->have) {
618            n = (uint64_t) state->have > len ?
619                (unsigned) len : state->have;
620            state->have -= n;
621            state->next += n;
622            state->pos += n;
623            len -= n;
624        }
625
626    /* output buffer empty -- return if we're at the end of the input */
627        else if (state->eof && state->strm.avail_in == 0)
628            break;
629
630    /* need more data to skip -- load up output buffer */
631        else {
632            /* get more output, looking for header if required */
633            if (xz_make(state) == -1)
634                return -1;
635        }
636    return 0;
637}
638
639int
640__libxml2_xzread(xzFile file, void *buf, unsigned len)
641{
642    unsigned got, n;
643    xz_statep state;
644    lzma_stream *strm;
645
646    /* get internal structure */
647    if (file == NULL)
648        return -1;
649    state = (xz_statep) file;
650    strm = &(state->strm);
651
652    /* check that we're reading and that there's no error */
653    if (state->err != LZMA_OK)
654        return -1;
655
656    /* since an int is returned, make sure len fits in one, otherwise return
657     * with an error (this avoids the flaw in the interface) */
658    if ((int) len < 0) {
659        xz_error(state, LZMA_BUF_ERROR,
660                 "requested length does not fit in int");
661        return -1;
662    }
663
664    /* if len is zero, avoid unnecessary operations */
665    if (len == 0)
666        return 0;
667
668    /* process a skip request */
669    if (state->seek) {
670        state->seek = 0;
671        if (xz_skip(state, state->skip) == -1)
672            return -1;
673    }
674
675    /* get len bytes to buf, or less than len if at the end */
676    got = 0;
677    do {
678        /* first just try copying data from the output buffer */
679        if (state->have) {
680            n = state->have > len ? len : state->have;
681            memcpy(buf, state->next, n);
682            state->next += n;
683            state->have -= n;
684        }
685
686        /* output buffer empty -- return if we're at the end of the input */
687        else if (state->eof && strm->avail_in == 0)
688            break;
689
690        /* need output data -- for small len or new stream load up our output
691         * buffer */
692        else if (state->how == LOOK || len < (state->size << 1)) {
693            /* get more output, looking for header if required */
694            if (xz_make(state) == -1)
695                return -1;
696            continue;           /* no progress yet -- go back to memcpy() above */
697            /* the copy above assures that we will leave with space in the
698             * output buffer, allowing at least one gzungetc() to succeed */
699        }
700
701        /* large len -- read directly into user buffer */
702        else if (state->how == COPY) {  /* read directly */
703            if (xz_load(state, buf, len, &n) == -1)
704                return -1;
705        }
706
707        /* large len -- decompress directly into user buffer */
708        else {                  /* state->how == LZMA */
709            strm->avail_out = len;
710            strm->next_out = buf;
711            if (xz_decomp(state) == -1)
712                return -1;
713            n = state->have;
714            state->have = 0;
715        }
716
717        /* update progress */
718        len -= n;
719        buf = (char *) buf + n;
720        got += n;
721        state->pos += n;
722    } while (len);
723
724    /* return number of bytes read into user buffer (will fit in int) */
725    return (int) got;
726}
727
728int
729__libxml2_xzclose(xzFile file)
730{
731    int ret;
732    xz_statep state;
733
734    /* get internal structure */
735    if (file == NULL)
736        return LZMA_DATA_ERROR;
737    state = (xz_statep) file;
738
739    /* free memory and close file */
740    if (state->size) {
741        lzma_end(&(state->strm));
742#ifdef HAVE_ZLIB_H
743        if (state->init == 1)
744            inflateEnd(&(state->zstrm));
745        state->init = 0;
746#endif
747        xmlFree(state->out);
748        xmlFree(state->in);
749    }
750    xmlFree(state->path);
751    ret = close(state->fd);
752    xmlFree(state);
753    return ret ? ret : LZMA_OK;
754}
755#endif /* HAVE_LZMA_H */
756