1/*- 2 * Copyright (c) 2003-2007,2013 Tim Kientzle 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26 27#include <errno.h> 28#include <stdlib.h> 29#include <string.h> 30 31/* 32 * This is a somewhat tricky test that verifies the ability to 33 * write and read very large entries to zip archives. 34 * 35 * See test_tar_large.c for more information about the machinery 36 * being used here. 37 */ 38 39static size_t nullsize; 40static void *nulldata; 41 42struct fileblock { 43 struct fileblock *next; 44 int size; 45 void *buff; 46 int64_t gap_size; /* Size of following gap */ 47}; 48 49struct fileblocks { 50 int64_t filesize; 51 int64_t fileposition; 52 int64_t gap_remaining; 53 void *buff; 54 struct fileblock *first; 55 struct fileblock *current; 56 struct fileblock *last; 57}; 58 59/* The following size definitions simplify things below. */ 60#define KB ((int64_t)1024) 61#define MB ((int64_t)1024 * KB) 62#define GB ((int64_t)1024 * MB) 63#define TB ((int64_t)1024 * GB) 64 65static int64_t memory_read_skip(struct archive *, void *, int64_t request); 66static ssize_t memory_read(struct archive *, void *, const void **buff); 67static ssize_t memory_write(struct archive *, void *, const void *, size_t); 68 69static uint16_t le16(const void *_p) { 70 const uint8_t *p = _p; 71 return p[0] | (p[1] << 8); 72} 73 74static uint32_t le32(const void *_p) { 75 const uint8_t *p = _p; 76 return le16(p) | ((uint32_t)le16(p + 2) << 16); 77} 78 79static uint64_t le64(const void *_p) { 80 const uint8_t *p = _p; 81 return le32(p) | ((uint64_t)le32(p + 4) << 32); 82} 83 84static ssize_t 85memory_write(struct archive *a, void *_private, const void *buff, size_t size) 86{ 87 struct fileblocks *private = _private; 88 struct fileblock *block; 89 90 (void)a; 91 92 if ((const char *)nulldata <= (const char *)buff 93 && (const char *)buff < (const char *)nulldata + nullsize) { 94 /* We don't need to store a block of gap data. */ 95 private->last->gap_size += (int64_t)size; 96 } else { 97 /* Yes, we're assuming the very first write is metadata. */ 98 /* It's header or metadata, copy and save it. */ 99 block = (struct fileblock *)malloc(sizeof(*block)); 100 memset(block, 0, sizeof(*block)); 101 block->size = (int)size; 102 block->buff = malloc(size); 103 memcpy(block->buff, buff, size); 104 if (private->last == NULL) { 105 private->first = private->last = block; 106 } else { 107 private->last->next = block; 108 private->last = block; 109 } 110 block->next = NULL; 111 } 112 private->filesize += size; 113 return ((long)size); 114} 115 116static ssize_t 117memory_read(struct archive *a, void *_private, const void **buff) 118{ 119 struct fileblocks *private = _private; 120 ssize_t size; 121 122 (void)a; 123 124 while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) { 125 private->current = private->current->next; 126 if (private->current != NULL) { 127 private->buff = private->current->buff; 128 private->gap_remaining = private->current->gap_size; 129 } 130 } 131 132 if (private->current == NULL) 133 return (0); 134 135 /* If there's real data, return that. */ 136 if (private->buff != NULL) { 137 *buff = private->buff; 138 size = ((char *)private->current->buff + private->current->size) 139 - (char *)private->buff; 140 private->buff = NULL; 141 private->fileposition += size; 142 return (size); 143 } 144 145 /* Big gap: too big to return all at once, so just return some. */ 146 if (private->gap_remaining > (int64_t)nullsize) { 147 private->gap_remaining -= nullsize; 148 *buff = nulldata; 149 private->fileposition += nullsize; 150 return (nullsize); 151 } 152 153 /* Small gap: finish the gap and prep for next block. */ 154 if (private->gap_remaining > 0) { 155 size = (ssize_t)private->gap_remaining; 156 *buff = nulldata; 157 private->gap_remaining = 0; 158 private->fileposition += size; 159 160 private->current = private->current->next; 161 if (private->current != NULL) { 162 private->buff = private->current->buff; 163 private->gap_remaining = private->current->gap_size; 164 } 165 166 return (size); 167 } 168 fprintf(stderr, "\n\n\nInternal failure\n\n\n"); 169 exit(1); 170} 171 172static int 173memory_read_open(struct archive *a, void *_private) 174{ 175 struct fileblocks *private = _private; 176 177 (void)a; /* UNUSED */ 178 179 private->current = private->first; 180 private->fileposition = 0; 181 if (private->current != NULL) { 182 private->buff = private->current->buff; 183 private->gap_remaining = private->current->gap_size; 184 } 185 return (ARCHIVE_OK); 186} 187 188static int64_t 189memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence) 190{ 191 struct fileblocks *private = _private; 192 193 (void)a; 194 if (whence == SEEK_END) { 195 offset = private->filesize + offset; 196 } else if (whence == SEEK_CUR) { 197 offset = private->fileposition + offset; 198 } 199 200 if (offset < 0) { 201 fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n"); 202 exit(1); 203 } 204 205 /* We've converted the request into a SEEK_SET. */ 206 private->fileposition = offset; 207 208 /* Walk the block list to find the new position. */ 209 offset = 0; 210 private->current = private->first; 211 while (private->current != NULL) { 212 if (offset + private->current->size > private->fileposition) { 213 /* Position is in this block. */ 214 private->buff = (char *)private->current->buff 215 + private->fileposition - offset; 216 private->gap_remaining = private->current->gap_size; 217 return private->fileposition; 218 } 219 offset += private->current->size; 220 if (offset + private->current->gap_size > private->fileposition) { 221 /* Position is in this gap. */ 222 private->buff = NULL; 223 private->gap_remaining = private->current->gap_size 224 - (private->fileposition - offset); 225 return private->fileposition; 226 } 227 offset += private->current->gap_size; 228 /* Skip to next block. */ 229 private->current = private->current->next; 230 } 231 if (private->fileposition == private->filesize) { 232 return private->fileposition; 233 } 234 fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n"); 235 exit(1); 236} 237 238static int64_t 239memory_read_skip(struct archive *a, void *_private, int64_t skip) 240{ 241 struct fileblocks *private = _private; 242 int64_t old_position = private->fileposition; 243 int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR); 244 return (new_position - old_position); 245} 246 247static struct fileblocks * 248fileblocks_new(void) 249{ 250 struct fileblocks *fileblocks; 251 252 fileblocks = calloc(1, sizeof(struct fileblocks)); 253 return fileblocks; 254} 255 256static void 257fileblocks_free(struct fileblocks *fileblocks) 258{ 259 while (fileblocks->first != NULL) { 260 struct fileblock *b = fileblocks->first; 261 fileblocks->first = fileblocks->first->next; 262 free(b->buff); 263 free(b); 264 } 265 free(fileblocks); 266} 267 268 269/* The sizes of the entries we're going to generate. */ 270static int64_t test_sizes[] = { 271 /* Test for 32-bit signed overflow. */ 272 2 * GB - 1, 2 * GB, 2 * GB + 1, 273 /* Test for 32-bit unsigned overflow. */ 274 4 * GB - 1, 4 * GB, 4 * GB + 1, 275 /* And one larger sample */ 276 5 * GB, 277 0 278}; 279 280 281static void 282verify_large_zip(struct archive *a, struct fileblocks *fileblocks, int seeking) 283{ 284 char namebuff[64]; 285 struct archive_entry *ae; 286 int i; 287 288 (void)seeking; /* UNUSED */ 289 290 assertEqualIntA(a, ARCHIVE_OK, 291 archive_read_set_options(a, "zip:ignorecrc32")); 292 assertEqualIntA(a, ARCHIVE_OK, 293 archive_read_set_open_callback(a, memory_read_open)); 294 assertEqualIntA(a, ARCHIVE_OK, 295 archive_read_set_read_callback(a, memory_read)); 296 assertEqualIntA(a, ARCHIVE_OK, 297 archive_read_set_skip_callback(a, memory_read_skip)); 298 assertEqualIntA(a, ARCHIVE_OK, 299 archive_read_set_seek_callback(a, memory_read_seek)); 300 assertEqualIntA(a, ARCHIVE_OK, 301 archive_read_set_callback_data(a, fileblocks)); 302 assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a)); 303 304 /* 305 * Read entries back. 306 */ 307 for (i = 0; test_sizes[i] > 0; i++) { 308 assertEqualIntA(a, ARCHIVE_OK, 309 archive_read_next_header(a, &ae)); 310 snprintf(namebuff, sizeof(namebuff), "file_%d", i); 311 assertEqualString(namebuff, archive_entry_pathname(ae)); 312 if (seeking) { 313 assert(archive_entry_size_is_set(ae)); 314 assertEqualInt(test_sizes[i], archive_entry_size(ae)); 315 } else { 316 assertEqualInt(0, archive_entry_size_is_set(ae)); 317 } 318 /* TODO: Read to end of data, verify length */ 319 } 320 assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 321 assertEqualString("lastfile", archive_entry_pathname(ae)); 322 323 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 324 325 /* Close out the archive. */ 326 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 327} 328 329DEFINE_TEST(test_write_format_zip_large) 330{ 331 int i; 332 char namebuff[64]; 333 struct fileblocks *fileblocks = fileblocks_new(); 334 struct archive_entry *ae; 335 struct archive *a; 336 const char *p; 337 const char *cd_start, *zip64_eocd, *zip64_locator, *eocd; 338 int64_t cd_size; 339 char *buff; 340 int64_t filesize; 341 size_t writesize, buffsize, s; 342 343 nullsize = (size_t)(1 * MB); 344 nulldata = malloc(nullsize); 345 memset(nulldata, 0xAA, nullsize); 346 347 /* 348 * Open an archive for writing. 349 */ 350 a = archive_write_new(); 351 archive_write_set_format_zip(a); 352 /* TODO: Repeat this entire test suite with default compression */ 353 assertEqualIntA(a, ARCHIVE_OK, 354 archive_write_set_options(a, "zip:compression=store")); 355 assertEqualIntA(a, ARCHIVE_OK, 356 archive_write_set_options(a, "zip:fakecrc32")); 357 assertEqualIntA(a, ARCHIVE_OK, 358 archive_write_set_bytes_per_block(a, 0)); /* No buffering. */ 359 assertEqualIntA(a, ARCHIVE_OK, 360 archive_write_open(a, fileblocks, NULL, memory_write, NULL)); 361 362 /* 363 * Write a series of large files to it. 364 */ 365 for (i = 0; test_sizes[i] != 0; i++) { 366 assert((ae = archive_entry_new()) != NULL); 367 snprintf(namebuff, sizeof(namebuff), "file_%d", i); 368 archive_entry_copy_pathname(ae, namebuff); 369 archive_entry_set_mode(ae, S_IFREG | 0755); 370 filesize = test_sizes[i]; 371 archive_entry_set_size(ae, filesize); 372 373 assertEqualIntA(a, ARCHIVE_OK, 374 archive_write_header(a, ae)); 375 archive_entry_free(ae); 376 377 /* 378 * Write the actual data to the archive. 379 */ 380 while (filesize > 0) { 381 writesize = nullsize; 382 if ((int64_t)writesize > filesize) 383 writesize = (size_t)filesize; 384 assertEqualIntA(a, (int)writesize, 385 (int)archive_write_data(a, nulldata, writesize)); 386 filesize -= writesize; 387 } 388 } 389 390 assert((ae = archive_entry_new()) != NULL); 391 archive_entry_copy_pathname(ae, "lastfile"); 392 archive_entry_set_mode(ae, S_IFREG | 0755); 393 assertA(0 == archive_write_header(a, ae)); 394 archive_entry_free(ae); 395 396 /* Close out the archive. */ 397 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 398 assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 399 400 /* 401 * Read back with seeking reader: 402 */ 403 a = archive_read_new(); 404 assertEqualIntA(a, ARCHIVE_OK, 405 archive_read_support_format_zip_seekable(a)); 406 verify_large_zip(a, fileblocks, 1); 407 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 408 409 /* 410 * Read back with streaming reader: 411 */ 412 a = archive_read_new(); 413 assertEqualIntA(a, ARCHIVE_OK, 414 archive_read_support_format_zip_streamable(a)); 415 verify_large_zip(a, fileblocks, 0); 416 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 417 418 /* 419 * Manually verify some of the final bytes of the archives. 420 */ 421 /* Collect the final bytes together */ 422#define FINAL_SIZE 8192 423 buff = malloc(FINAL_SIZE); 424 buffsize = 0; 425 memory_read_open(NULL, fileblocks); 426 memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END); 427 while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) { 428 memcpy(buff + buffsize, p, s); 429 buffsize += s; 430 } 431 assertEqualInt(buffsize, FINAL_SIZE); 432 433 p = buff + buffsize; 434 435 /* Verify regular end-of-central-directory record */ 436 eocd = p - 22; 437 assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8); 438 assertEqualMem(eocd + 8, "\010\0\010\0", 4); /* 8 entries total */ 439 cd_size = le32(eocd + 12); 440 /* Start of CD offset should be 0xffffffff */ 441 assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4); 442 assertEqualMem(eocd + 20, "\0\0", 2); /* No Zip comment */ 443 444 /* Verify Zip64 locator */ 445 zip64_locator = p - 42; 446 assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8); 447 zip64_eocd = p - (fileblocks->filesize - le64(zip64_locator + 8)); 448 assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4); 449 450 /* Verify Zip64 end-of-cd record. */ 451 assert(zip64_eocd == p - 98); 452 assertEqualMem(zip64_eocd, "PK\006\006", 4); 453 assertEqualInt(44, le64(zip64_eocd + 4)); // Size of EoCD record - 12 454 assertEqualMem(zip64_eocd + 12, "\055\0", 2); // Made by version: 45 455 assertEqualMem(zip64_eocd + 14, "\055\0", 2); // Requires version: 45 456 assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk 457 assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks 458 assertEqualInt(8, le64(zip64_eocd + 24)); // Entries on this disk 459 assertEqualInt(8, le64(zip64_eocd + 32)); // Total entries 460 cd_size = le64(zip64_eocd + 40); 461 cd_start = p - (fileblocks->filesize - le64(zip64_eocd + 48)); 462 463 assert(cd_start + cd_size == zip64_eocd); 464 465 assertEqualInt(le64(zip64_eocd + 48) // Start of CD 466 + cd_size 467 + 56 // Size of Zip64 EOCD 468 + 20 // Size of Zip64 locator 469 + 22, // Size of EOCD 470 fileblocks->filesize); 471 472 // TODO: Scan entire Central Directory, sanity-check all data 473 assertEqualMem(cd_start, "PK\001\002", 4); 474 475 fileblocks_free(fileblocks); 476 free(buff); 477 free(nulldata); 478} 479