1/** 2 * xzlib.c: front end for the transparent suport of lzma compression 3 * at the I/O layer, based on an example file from lzma project 4 * 5 * See Copyright for the status of this software. 6 * 7 * Anders F Bjorklund <afb@users.sourceforge.net> 8 */ 9#define IN_LIBXML 10#include "libxml.h" 11#ifdef HAVE_LZMA_H 12 13#include <string.h> 14#ifdef HAVE_ERRNO_H 15#include <errno.h> 16#endif 17 18 19#ifdef HAVE_SYS_TYPES_H 20#include <sys/types.h> 21#endif 22#ifdef HAVE_SYS_STAT_H 23#include <sys/stat.h> 24#endif 25#ifdef HAVE_FCNTL_H 26#include <fcntl.h> 27#endif 28#ifdef HAVE_UNISTD_H 29#include <unistd.h> 30#endif 31#ifdef HAVE_STDLIB_H 32#include <stdlib.h> 33#endif 34#ifdef HAVE_ZLIB_H 35#include <zlib.h> 36#endif 37#include <lzma.h> 38 39#include "xzlib.h" 40#include <libxml/xmlmemory.h> 41 42/* values for xz_state how */ 43#define LOOK 0 /* look for a gzip/lzma header */ 44#define COPY 1 /* copy input directly */ 45#define GZIP 2 /* decompress a gzip stream */ 46#define LZMA 3 /* decompress a lzma stream */ 47 48/* internal lzma file state data structure */ 49typedef struct { 50 int mode; /* see lzma modes above */ 51 int fd; /* file descriptor */ 52 char *path; /* path or fd for error messages */ 53 uint64_t pos; /* current position in uncompressed data */ 54 unsigned int size; /* buffer size, zero if not allocated yet */ 55 unsigned int want; /* requested buffer size, default is BUFSIZ */ 56 unsigned char *in; /* input buffer */ 57 unsigned char *out; /* output buffer (double-sized when reading) */ 58 unsigned char *next; /* next output data to deliver or write */ 59 unsigned int have; /* amount of output data unused at next */ 60 int eof; /* true if end of input file reached */ 61 uint64_t start; /* where the lzma data started, for rewinding */ 62 uint64_t raw; /* where the raw data started, for seeking */ 63 int how; /* 0: get header, 1: copy, 2: decompress */ 64 int direct; /* true if last read direct, false if lzma */ 65 /* seek request */ 66 uint64_t skip; /* amount to skip (already rewound if backwards) */ 67 int seek; /* true if seek request pending */ 68 /* error information */ 69 int err; /* error code */ 70 char *msg; /* error message */ 71 /* lzma stream */ 72 int init; /* is the iniflate stream initialized */ 73 lzma_stream strm; /* stream structure in-place (not a pointer) */ 74 char padding1[32]; /* padding allowing to cope with possible 75 extensions of above structure without 76 too much side effect */ 77#ifdef HAVE_ZLIB_H 78 /* zlib inflate or deflate stream */ 79 z_stream zstrm; /* stream structure in-place (not a pointer) */ 80#endif 81 char padding2[32]; /* padding allowing to cope with possible 82 extensions of above structure without 83 too much side effect */ 84} xz_state, *xz_statep; 85 86static void 87xz_error(xz_statep state, int err, const char *msg) 88{ 89 /* free previously allocated message and clear */ 90 if (state->msg != NULL) { 91 if (state->err != LZMA_MEM_ERROR) 92 xmlFree(state->msg); 93 state->msg = NULL; 94 } 95 96 /* set error code, and if no message, then done */ 97 state->err = err; 98 if (msg == NULL) 99 return; 100 101 /* for an out of memory error, save as static string */ 102 if (err == LZMA_MEM_ERROR) { 103 state->msg = (char *) msg; 104 return; 105 } 106 107 /* construct error message with path */ 108 if ((state->msg = 109 xmlMalloc(strlen(state->path) + strlen(msg) + 3)) == NULL) { 110 state->err = LZMA_MEM_ERROR; 111 state->msg = (char *) "out of memory"; 112 return; 113 } 114 strcpy(state->msg, state->path); 115 strcat(state->msg, ": "); 116 strcat(state->msg, msg); 117 return; 118} 119 120static void 121xz_reset(xz_statep state) 122{ 123 state->have = 0; /* no output data available */ 124 state->eof = 0; /* not at end of file */ 125 state->how = LOOK; /* look for gzip header */ 126 state->direct = 1; /* default for empty file */ 127 state->seek = 0; /* no seek request pending */ 128 xz_error(state, LZMA_OK, NULL); /* clear error */ 129 state->pos = 0; /* no uncompressed data yet */ 130 state->strm.avail_in = 0; /* no input data yet */ 131#ifdef HAVE_ZLIB_H 132 state->zstrm.avail_in = 0; /* no input data yet */ 133#endif 134} 135 136static xzFile 137xz_open(const char *path, int fd, const char *mode ATTRIBUTE_UNUSED) 138{ 139 xz_statep state; 140 141 /* allocate xzFile structure to return */ 142 state = xmlMalloc(sizeof(xz_state)); 143 if (state == NULL) 144 return NULL; 145 state->size = 0; /* no buffers allocated yet */ 146 state->want = BUFSIZ; /* requested buffer size */ 147 state->msg = NULL; /* no error message yet */ 148 state->init = 0; /* initialization of zlib data */ 149 150 /* save the path name for error messages */ 151 state->path = xmlMalloc(strlen(path) + 1); 152 if (state->path == NULL) { 153 xmlFree(state); 154 return NULL; 155 } 156 strcpy(state->path, path); 157 158 /* open the file with the appropriate mode (or just use fd) */ 159 state->fd = fd != -1 ? fd : open(path, 160#ifdef O_LARGEFILE 161 O_LARGEFILE | 162#endif 163#ifdef O_BINARY 164 O_BINARY | 165#endif 166 O_RDONLY, 0666); 167 if (state->fd == -1) { 168 xmlFree(state->path); 169 xmlFree(state); 170 return NULL; 171 } 172 173 /* save the current position for rewinding (only if reading) */ 174 state->start = lseek(state->fd, 0, SEEK_CUR); 175 if (state->start == (uint64_t) - 1) 176 state->start = 0; 177 178 /* initialize stream */ 179 xz_reset(state); 180 181 /* return stream */ 182 return (xzFile) state; 183} 184 185xzFile 186__libxml2_xzopen(const char *path, const char *mode) 187{ 188 return xz_open(path, -1, mode); 189} 190 191xzFile 192__libxml2_xzdopen(int fd, const char *mode) 193{ 194 char *path; /* identifier for error messages */ 195 xzFile xz; 196 197 if (fd == -1 || (path = xmlMalloc(7 + 3 * sizeof(int))) == NULL) 198 return NULL; 199 sprintf(path, "<fd:%d>", fd); /* for debugging */ 200 xz = xz_open(path, fd, mode); 201 xmlFree(path); 202 return xz; 203} 204 205static int 206xz_load(xz_statep state, unsigned char *buf, unsigned int len, 207 unsigned int *have) 208{ 209 int ret; 210 211 *have = 0; 212 do { 213 ret = read(state->fd, buf + *have, len - *have); 214 if (ret <= 0) 215 break; 216 *have += ret; 217 } while (*have < len); 218 if (ret < 0) { 219 xz_error(state, -1, strerror(errno)); 220 return -1; 221 } 222 if (ret == 0) 223 state->eof = 1; 224 return 0; 225} 226 227static int 228xz_avail(xz_statep state) 229{ 230 lzma_stream *strm = &(state->strm); 231 232 if (state->err != LZMA_OK) 233 return -1; 234 if (state->eof == 0) { 235 /* avail_in is size_t, which is not necessary sizeof(unsigned) */ 236 unsigned tmp = strm->avail_in; 237 238 if (xz_load(state, state->in, state->size, &tmp) == -1) { 239 strm->avail_in = tmp; 240 return -1; 241 } 242 strm->avail_in = tmp; 243 strm->next_in = state->in; 244 } 245 return 0; 246} 247 248static int 249is_format_xz(xz_statep state) 250{ 251 lzma_stream *strm = &(state->strm); 252 253 return strm->avail_in >= 6 && memcmp(state->in, "\3757zXZ", 6) == 0; 254} 255 256static int 257is_format_lzma(xz_statep state) 258{ 259 lzma_stream *strm = &(state->strm); 260 261 lzma_filter filter; 262 lzma_options_lzma *opt; 263 uint32_t dict_size; 264 uint64_t uncompressed_size; 265 size_t i; 266 267 if (strm->avail_in < 13) 268 return 0; 269 270 filter.id = LZMA_FILTER_LZMA1; 271 if (lzma_properties_decode(&filter, NULL, state->in, 5) != LZMA_OK) 272 return 0; 273 274 opt = filter.options; 275 dict_size = opt->dict_size; 276 free(opt); /* we can't use xmlFree on a string returned by zlib */ 277 278 /* A hack to ditch tons of false positives: We allow only dictionary 279 * sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone 280 * created only files with 2^n, but accepts any dictionary size. 281 * If someone complains, this will be reconsidered. 282 */ 283 if (dict_size != UINT32_MAX) { 284 uint32_t d = dict_size - 1; 285 286 d |= d >> 2; 287 d |= d >> 3; 288 d |= d >> 4; 289 d |= d >> 8; 290 d |= d >> 16; 291 ++d; 292 if (d != dict_size || dict_size == 0) 293 return 0; 294 } 295 296 /* Another hack to ditch false positives: Assume that if the 297 * uncompressed size is known, it must be less than 256 GiB. 298 * Again, if someone complains, this will be reconsidered. 299 */ 300 uncompressed_size = 0; 301 for (i = 0; i < 8; ++i) 302 uncompressed_size |= (uint64_t) (state->in[5 + i]) << (i * 8); 303 304 if (uncompressed_size != UINT64_MAX 305 && uncompressed_size > (UINT64_C(1) << 38)) 306 return 0; 307 308 return 1; 309} 310 311#ifdef HAVE_ZLIB_H 312 313/* Get next byte from input, or -1 if end or error. */ 314#define NEXT() ((strm->avail_in == 0 && xz_avail(state) == -1) ? -1 : \ 315 (strm->avail_in == 0 ? -1 : \ 316 (strm->avail_in--, *(strm->next_in)++))) 317 318/* Get a four-byte little-endian integer and return 0 on success and the value 319 in *ret. Otherwise -1 is returned and *ret is not modified. */ 320static int 321gz_next4(xz_statep state, unsigned long *ret) 322{ 323 int ch; 324 unsigned long val; 325 z_streamp strm = &(state->zstrm); 326 327 val = NEXT(); 328 val += (unsigned) NEXT() << 8; 329 val += (unsigned long) NEXT() << 16; 330 ch = NEXT(); 331 if (ch == -1) 332 return -1; 333 val += (unsigned long) ch << 24; 334 *ret = val; 335 return 0; 336} 337#endif 338 339static int 340xz_head(xz_statep state) 341{ 342 lzma_stream *strm = &(state->strm); 343 lzma_stream init = LZMA_STREAM_INIT; 344 int flags; 345 unsigned len; 346 347 /* allocate read buffers and inflate memory */ 348 if (state->size == 0) { 349 /* allocate buffers */ 350 state->in = xmlMalloc(state->want); 351 state->out = xmlMalloc(state->want << 1); 352 if (state->in == NULL || state->out == NULL) { 353 if (state->out != NULL) 354 xmlFree(state->out); 355 if (state->in != NULL) 356 xmlFree(state->in); 357 xz_error(state, LZMA_MEM_ERROR, "out of memory"); 358 return -1; 359 } 360 state->size = state->want; 361 362 /* allocate decoder memory */ 363 state->strm = init; 364 state->strm.avail_in = 0; 365 state->strm.next_in = NULL; 366 if (lzma_auto_decoder(&state->strm, UINT64_MAX, 0) != LZMA_OK) { 367 xmlFree(state->out); 368 xmlFree(state->in); 369 state->size = 0; 370 xz_error(state, LZMA_MEM_ERROR, "out of memory"); 371 return -1; 372 } 373#ifdef HAVE_ZLIB_H 374 /* allocate inflate memory */ 375 state->zstrm.zalloc = Z_NULL; 376 state->zstrm.zfree = Z_NULL; 377 state->zstrm.opaque = Z_NULL; 378 state->zstrm.avail_in = 0; 379 state->zstrm.next_in = Z_NULL; 380 if (state->init == 0) { 381 if (inflateInit2(&(state->zstrm), -15) != Z_OK) {/* raw inflate */ 382 xmlFree(state->out); 383 xmlFree(state->in); 384 state->size = 0; 385 xz_error(state, LZMA_MEM_ERROR, "out of memory"); 386 return -1; 387 } 388 state->init = 1; 389 } 390#endif 391 } 392 393 /* get some data in the input buffer */ 394 if (strm->avail_in == 0) { 395 if (xz_avail(state) == -1) 396 return -1; 397 if (strm->avail_in == 0) 398 return 0; 399 } 400 401 /* look for the xz magic header bytes */ 402 if (is_format_xz(state) || is_format_lzma(state)) { 403 state->how = LZMA; 404 state->direct = 0; 405 return 0; 406 } 407#ifdef HAVE_ZLIB_H 408 /* look for the gzip magic header bytes 31 and 139 */ 409 if (strm->next_in[0] == 31) { 410 strm->avail_in--; 411 strm->next_in++; 412 if (strm->avail_in == 0 && xz_avail(state) == -1) 413 return -1; 414 if (strm->avail_in && strm->next_in[0] == 139) { 415 /* we have a gzip header, woo hoo! */ 416 strm->avail_in--; 417 strm->next_in++; 418 419 /* skip rest of header */ 420 if (NEXT() != 8) { /* compression method */ 421 xz_error(state, LZMA_DATA_ERROR, 422 "unknown compression method"); 423 return -1; 424 } 425 flags = NEXT(); 426 if (flags & 0xe0) { /* reserved flag bits */ 427 xz_error(state, LZMA_DATA_ERROR, 428 "unknown header flags set"); 429 return -1; 430 } 431 NEXT(); /* modification time */ 432 NEXT(); 433 NEXT(); 434 NEXT(); 435 NEXT(); /* extra flags */ 436 NEXT(); /* operating system */ 437 if (flags & 4) { /* extra field */ 438 len = (unsigned) NEXT(); 439 len += (unsigned) NEXT() << 8; 440 while (len--) 441 if (NEXT() < 0) 442 break; 443 } 444 if (flags & 8) /* file name */ 445 while (NEXT() > 0) ; 446 if (flags & 16) /* comment */ 447 while (NEXT() > 0) ; 448 if (flags & 2) { /* header crc */ 449 NEXT(); 450 NEXT(); 451 } 452 /* an unexpected end of file is not checked for here -- it will be 453 * noticed on the first request for uncompressed data */ 454 455 /* set up for decompression */ 456 inflateReset(&state->zstrm); 457 state->zstrm.adler = crc32(0L, Z_NULL, 0); 458 state->how = GZIP; 459 state->direct = 0; 460 return 0; 461 } else { 462 /* not a gzip file -- save first byte (31) and fall to raw i/o */ 463 state->out[0] = 31; 464 state->have = 1; 465 } 466 } 467#endif 468 469 /* doing raw i/o, save start of raw data for seeking, copy any leftover 470 * input to output -- this assumes that the output buffer is larger than 471 * the input buffer, which also assures space for gzungetc() */ 472 state->raw = state->pos; 473 state->next = state->out; 474 if (strm->avail_in) { 475 memcpy(state->next + state->have, strm->next_in, strm->avail_in); 476 state->have += strm->avail_in; 477 strm->avail_in = 0; 478 } 479 state->how = COPY; 480 state->direct = 1; 481 return 0; 482} 483 484static int 485xz_decomp(xz_statep state) 486{ 487 int ret; 488 unsigned had; 489 unsigned long crc, len; 490 lzma_stream *strm = &(state->strm); 491 492 lzma_action action = LZMA_RUN; 493 494 /* fill output buffer up to end of deflate stream */ 495 had = strm->avail_out; 496 do { 497 /* get more input for inflate() */ 498 if (strm->avail_in == 0 && xz_avail(state) == -1) 499 return -1; 500 if (strm->avail_in == 0) { 501 xz_error(state, LZMA_DATA_ERROR, "unexpected end of file"); 502 return -1; 503 } 504 if (state->eof) 505 action = LZMA_FINISH; 506 507 /* decompress and handle errors */ 508#ifdef HAVE_ZLIB_H 509 if (state->how == GZIP) { 510 state->zstrm.avail_in = (uInt) state->strm.avail_in; 511 state->zstrm.next_in = (Bytef *) state->strm.next_in; 512 state->zstrm.avail_out = (uInt) state->strm.avail_out; 513 state->zstrm.next_out = (Bytef *) state->strm.next_out; 514 ret = inflate(&state->zstrm, Z_NO_FLUSH); 515 if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { 516 xz_error(state, Z_STREAM_ERROR, 517 "internal error: inflate stream corrupt"); 518 return -1; 519 } 520 if (ret == Z_MEM_ERROR) 521 ret = LZMA_MEM_ERROR; 522 if (ret == Z_DATA_ERROR) 523 ret = LZMA_DATA_ERROR; 524 if (ret == Z_STREAM_END) 525 ret = LZMA_STREAM_END; 526 state->strm.avail_in = state->zstrm.avail_in; 527 state->strm.next_in = state->zstrm.next_in; 528 state->strm.avail_out = state->zstrm.avail_out; 529 state->strm.next_out = state->zstrm.next_out; 530 } else /* state->how == LZMA */ 531#endif 532 ret = lzma_code(strm, action); 533 if (ret == LZMA_MEM_ERROR) { 534 xz_error(state, LZMA_MEM_ERROR, "out of memory"); 535 return -1; 536 } 537 if (ret == LZMA_DATA_ERROR) { 538 xz_error(state, LZMA_DATA_ERROR, "compressed data error"); 539 return -1; 540 } 541 } while (strm->avail_out && ret != LZMA_STREAM_END); 542 543 /* update available output and crc check value */ 544 state->have = had - strm->avail_out; 545 state->next = strm->next_out - state->have; 546#ifdef HAVE_ZLIB_H 547 state->zstrm.adler = 548 crc32(state->zstrm.adler, state->next, state->have); 549#endif 550 551 if (ret == LZMA_STREAM_END) { 552#ifdef HAVE_ZLIB_H 553 if (state->how == GZIP) { 554 if (gz_next4(state, &crc) == -1 || gz_next4(state, &len) == -1) { 555 xz_error(state, LZMA_DATA_ERROR, "unexpected end of file"); 556 return -1; 557 } 558 if (crc != state->zstrm.adler) { 559 xz_error(state, LZMA_DATA_ERROR, "incorrect data check"); 560 return -1; 561 } 562 if (len != (state->zstrm.total_out & 0xffffffffL)) { 563 xz_error(state, LZMA_DATA_ERROR, "incorrect length check"); 564 return -1; 565 } 566 state->strm.avail_in = 0; 567 state->strm.next_in = NULL; 568 state->strm.avail_out = 0; 569 state->strm.next_out = NULL; 570 } else 571#endif 572 if (strm->avail_in != 0 || !state->eof) { 573 xz_error(state, LZMA_DATA_ERROR, "trailing garbage"); 574 return -1; 575 } 576 state->how = LOOK; /* ready for next stream, once have is 0 (leave 577 * state->direct unchanged to remember how) */ 578 } 579 580 /* good decompression */ 581 return 0; 582} 583 584static int 585xz_make(xz_statep state) 586{ 587 lzma_stream *strm = &(state->strm); 588 589 if (state->how == LOOK) { /* look for lzma / gzip header */ 590 if (xz_head(state) == -1) 591 return -1; 592 if (state->have) /* got some data from xz_head() */ 593 return 0; 594 } 595 if (state->how == COPY) { /* straight copy */ 596 if (xz_load(state, state->out, state->size << 1, &(state->have)) == 597 -1) 598 return -1; 599 state->next = state->out; 600 } else if (state->how == LZMA || state->how == GZIP) { /* decompress */ 601 strm->avail_out = state->size << 1; 602 strm->next_out = state->out; 603 if (xz_decomp(state) == -1) 604 return -1; 605 } 606 return 0; 607} 608 609static int 610xz_skip(xz_statep state, uint64_t len) 611{ 612 unsigned n; 613 614 /* skip over len bytes or reach end-of-file, whichever comes first */ 615 while (len) 616 /* skip over whatever is in output buffer */ 617 if (state->have) { 618 n = (uint64_t) state->have > len ? 619 (unsigned) len : state->have; 620 state->have -= n; 621 state->next += n; 622 state->pos += n; 623 len -= n; 624 } 625 626 /* output buffer empty -- return if we're at the end of the input */ 627 else if (state->eof && state->strm.avail_in == 0) 628 break; 629 630 /* need more data to skip -- load up output buffer */ 631 else { 632 /* get more output, looking for header if required */ 633 if (xz_make(state) == -1) 634 return -1; 635 } 636 return 0; 637} 638 639int 640__libxml2_xzread(xzFile file, void *buf, unsigned len) 641{ 642 unsigned got, n; 643 xz_statep state; 644 lzma_stream *strm; 645 646 /* get internal structure */ 647 if (file == NULL) 648 return -1; 649 state = (xz_statep) file; 650 strm = &(state->strm); 651 652 /* check that we're reading and that there's no error */ 653 if (state->err != LZMA_OK) 654 return -1; 655 656 /* since an int is returned, make sure len fits in one, otherwise return 657 * with an error (this avoids the flaw in the interface) */ 658 if ((int) len < 0) { 659 xz_error(state, LZMA_BUF_ERROR, 660 "requested length does not fit in int"); 661 return -1; 662 } 663 664 /* if len is zero, avoid unnecessary operations */ 665 if (len == 0) 666 return 0; 667 668 /* process a skip request */ 669 if (state->seek) { 670 state->seek = 0; 671 if (xz_skip(state, state->skip) == -1) 672 return -1; 673 } 674 675 /* get len bytes to buf, or less than len if at the end */ 676 got = 0; 677 do { 678 /* first just try copying data from the output buffer */ 679 if (state->have) { 680 n = state->have > len ? len : state->have; 681 memcpy(buf, state->next, n); 682 state->next += n; 683 state->have -= n; 684 } 685 686 /* output buffer empty -- return if we're at the end of the input */ 687 else if (state->eof && strm->avail_in == 0) 688 break; 689 690 /* need output data -- for small len or new stream load up our output 691 * buffer */ 692 else if (state->how == LOOK || len < (state->size << 1)) { 693 /* get more output, looking for header if required */ 694 if (xz_make(state) == -1) 695 return -1; 696 continue; /* no progress yet -- go back to memcpy() above */ 697 /* the copy above assures that we will leave with space in the 698 * output buffer, allowing at least one gzungetc() to succeed */ 699 } 700 701 /* large len -- read directly into user buffer */ 702 else if (state->how == COPY) { /* read directly */ 703 if (xz_load(state, buf, len, &n) == -1) 704 return -1; 705 } 706 707 /* large len -- decompress directly into user buffer */ 708 else { /* state->how == LZMA */ 709 strm->avail_out = len; 710 strm->next_out = buf; 711 if (xz_decomp(state) == -1) 712 return -1; 713 n = state->have; 714 state->have = 0; 715 } 716 717 /* update progress */ 718 len -= n; 719 buf = (char *) buf + n; 720 got += n; 721 state->pos += n; 722 } while (len); 723 724 /* return number of bytes read into user buffer (will fit in int) */ 725 return (int) got; 726} 727 728int 729__libxml2_xzclose(xzFile file) 730{ 731 int ret; 732 xz_statep state; 733 734 /* get internal structure */ 735 if (file == NULL) 736 return LZMA_DATA_ERROR; 737 state = (xz_statep) file; 738 739 /* free memory and close file */ 740 if (state->size) { 741 lzma_end(&(state->strm)); 742#ifdef HAVE_ZLIB_H 743 if (state->init == 1) 744 inflateEnd(&(state->zstrm)); 745 state->init = 0; 746#endif 747 xmlFree(state->out); 748 xmlFree(state->in); 749 } 750 xmlFree(state->path); 751 ret = close(state->fd); 752 xmlFree(state); 753 return ret ? ret : LZMA_OK; 754} 755#endif /* HAVE_LZMA_H */ 756