test_tar_large.c revision 272461
1193323Sed/*- 2193323Sed * Copyright (c) 2003-2007 Tim Kientzle 3193323Sed * All rights reserved. 4193323Sed * 5193323Sed * Redistribution and use in source and binary forms, with or without 6193323Sed * modification, are permitted provided that the following conditions 7193323Sed * are met: 8193323Sed * 1. Redistributions of source code must retain the above copyright 9193323Sed * notice, this list of conditions and the following disclaimer. 10193323Sed * 2. Redistributions in binary form must reproduce the above copyright 11193323Sed * notice, this list of conditions and the following disclaimer in the 12193323Sed * documentation and/or other materials provided with the distribution. 13193323Sed * 14193323Sed * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15193323Sed * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16193323Sed * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17193323Sed * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18193323Sed * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19193323Sed * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20193323Sed * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21193323Sed * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22193323Sed * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23193323Sed * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24193323Sed */ 25193323Sed#include "test.h" 26198090Srdivacky__FBSDID("$FreeBSD: releng/10.1/contrib/libarchive/libarchive/test/test_tar_large.c 232153 2012-02-25 10:58:02Z mm $"); 27193323Sed 28193323Sed#include <errno.h> 29193323Sed#include <stdlib.h> 30193323Sed#include <string.h> 31193323Sed 32193323Sed/* 33193323Sed * This is a somewhat tricky test that verifies the ability to 34198090Srdivacky * write and read very large entries to tar archives. It 35193323Sed * writes entries from 2GB up to 1TB to an archive in memory. 36193323Sed * The memory storage here carefully avoids actually storing 37193323Sed * any part of the file bodies, so it runs very quickly and requires 38193323Sed * very little memory. If you're willing to wait a few minutes, 39193323Sed * you should be able to exercise petabyte entries with this code. 40193323Sed */ 41193323Sed 42193323Sed/* 43193323Sed * Each file is built up by duplicating the following block. 44193323Sed */ 45193323Sedstatic size_t filedatasize; 46193323Sedstatic void *filedata; 47193323Sed 48193323Sed/* 49193323Sed * We store the archive as blocks of data generated by libarchive, 50193323Sed * each possibly followed by bytes of file data. 51193323Sed */ 52193323Sedstruct memblock { 53193323Sed struct memblock *next; 54193323Sed size_t size; 55193323Sed void *buff; 56193323Sed int64_t filebytes; 57193323Sed}; 58193323Sed 59193323Sed/* 60193323Sed * The total memory store is just a list of memblocks plus 61193323Sed * some accounting overhead. 62193323Sed */ 63193323Sedstruct memdata { 64193323Sed int64_t filebytes; 65193323Sed void *buff; 66193323Sed struct memblock *first; 67193323Sed struct memblock *last; 68193323Sed}; 69193323Sed 70198090Srdivacky/* The following size definitions simplify things below. */ 71193323Sed#define KB ((int64_t)1024) 72193323Sed#define MB ((int64_t)1024 * KB) 73193323Sed#define GB ((int64_t)1024 * MB) 74193323Sed#define TB ((int64_t)1024 * GB) 75193323Sed 76193323Sedstatic int64_t memory_read_skip(struct archive *, void *, int64_t request); 77193323Sedstatic ssize_t memory_read(struct archive *, void *, const void **buff); 78193323Sedstatic ssize_t memory_write(struct archive *, void *, const void *, size_t); 79193323Sed 80193323Sed 81193323Sedstatic ssize_t 82193323Sedmemory_write(struct archive *a, void *_private, const void *buff, size_t size) 83193323Sed{ 84193323Sed struct memdata *private = _private; 85193323Sed struct memblock *block; 86193323Sed 87193323Sed (void)a; 88193323Sed 89193323Sed /* 90193323Sed * Since libarchive tries to behave in a zero-copy manner, if 91193323Sed * you give a pointer to filedata to the library, a pointer 92193323Sed * into that data will (usually) pop out here. This way, we 93193323Sed * can tell the difference between filedata and library header 94193323Sed * and metadata. 95193323Sed */ 96193323Sed if ((const char *)filedata <= (const char *)buff 97193323Sed && (const char *)buff < (const char *)filedata + filedatasize) { 98193323Sed /* We don't need to store a block of file data. */ 99193323Sed private->last->filebytes += (int64_t)size; 100193323Sed } else { 101193323Sed /* Yes, we're assuming the very first write is metadata. */ 102193323Sed /* It's header or metadata, copy and save it. */ 103193323Sed block = (struct memblock *)malloc(sizeof(*block)); 104193323Sed memset(block, 0, sizeof(*block)); 105193323Sed block->size = size; 106193323Sed block->buff = malloc(size); 107193323Sed memcpy(block->buff, buff, size); 108193323Sed if (private->last == NULL) { 109193323Sed private->first = private->last = block; 110193323Sed } else { 111193323Sed private->last->next = block; 112193323Sed private->last = block; 113193323Sed } 114193323Sed block->next = NULL; 115193323Sed } 116193323Sed return ((long)size); 117193323Sed} 118193323Sed 119193323Sedstatic ssize_t 120193323Sedmemory_read(struct archive *a, void *_private, const void **buff) 121193323Sed{ 122193323Sed struct memdata *private = _private; 123193323Sed struct memblock *block; 124193323Sed ssize_t size; 125193323Sed 126193323Sed (void)a; 127193323Sed 128193323Sed free(private->buff); 129193323Sed private->buff = NULL; 130193323Sed if (private->first == NULL) { 131193323Sed private->last = NULL; 132193323Sed return (ARCHIVE_EOF); 133193323Sed } 134193323Sed if (private->filebytes > 0) { 135193323Sed /* 136193323Sed * We're returning file bytes, simulate it by 137193323Sed * passing blocks from the template data. 138193323Sed */ 139193323Sed if (private->filebytes > (int64_t)filedatasize) 140193323Sed size = (ssize_t)filedatasize; 141193323Sed else 142193323Sed size = (ssize_t)private->filebytes; 143193323Sed private->filebytes -= size; 144193323Sed *buff = filedata; 145198090Srdivacky } else { 146198090Srdivacky /* 147193323Sed * We need to get some real data to return. 148193323Sed */ 149193323Sed block = private->first; 150193323Sed private->first = block->next; 151193323Sed size = (ssize_t)block->size; 152193323Sed if (block->buff != NULL) { 153193323Sed private->buff = block->buff; 154193323Sed *buff = block->buff; 155193323Sed } else { 156193323Sed private->buff = NULL; 157193323Sed *buff = filedata; 158193323Sed } 159193323Sed private->filebytes = block->filebytes; 160193323Sed free(block); 161193323Sed } 162193323Sed return (size); 163193323Sed} 164193323Sed 165198090Srdivacky 166198090Srdivackystatic int64_t 167193323Sedmemory_read_skip(struct archive *a, void *_private, int64_t skip) 168193323Sed{ 169193323Sed struct memdata *private = _private; 170193323Sed 171193323Sed (void)a; 172193323Sed 173193323Sed if (private->first == NULL) { 174193323Sed private->last = NULL; 175193323Sed return (0); 176193323Sed } 177193323Sed if (private->filebytes > 0) { 178193323Sed if (private->filebytes < skip) 179193323Sed skip = (off_t)private->filebytes; 180193323Sed private->filebytes -= skip; 181193323Sed } else { 182193323Sed skip = 0; 183193323Sed } 184193323Sed return (skip); 185193323Sed} 186193323Sed 187193323SedDEFINE_TEST(test_tar_large) 188193323Sed{ 189193323Sed /* The sizes of the entries we're going to generate. */ 190193323Sed static int64_t tests[] = { 191193323Sed /* Test for 32-bit signed overflow. */ 192193323Sed 2 * GB - 1, 2 * GB, 2 * GB + 1, 193193323Sed /* Test for 32-bit unsigned overflow. */ 194193323Sed 4 * GB - 1, 4 * GB, 4 * GB + 1, 195193323Sed /* 8GB is the "official" max for ustar. */ 196193323Sed 8 * GB - 1, 8 * GB, 8 * GB + 1, 197193323Sed /* Bend ustar a tad and you can get 64GB (12 octal digits). */ 198193323Sed 64 * GB - 1, 64 * GB, 199193323Sed /* And larger entries that require non-ustar extensions. */ 200198090Srdivacky 256 * GB, 1 * TB, 0 }; 201193323Sed int i; 202193323Sed char namebuff[64]; 203193323Sed struct memdata memdata; 204193323Sed struct archive_entry *ae; 205193323Sed struct archive *a; 206193323Sed int64_t filesize; 207193323Sed size_t writesize; 208193323Sed 209193323Sed filedatasize = (size_t)(1 * MB); 210193323Sed filedata = malloc(filedatasize); 211193323Sed memset(filedata, 0xAA, filedatasize); 212193323Sed memset(&memdata, 0, sizeof(memdata)); 213193323Sed 214193323Sed /* 215193323Sed * Open an archive for writing. 216193323Sed */ 217198090Srdivacky a = archive_write_new(); 218193323Sed archive_write_set_format_pax_restricted(a); 219193323Sed archive_write_set_bytes_per_block(a, 0); /* No buffering. */ 220193323Sed archive_write_open(a, &memdata, NULL, memory_write, NULL); 221193323Sed 222193323Sed /* 223193323Sed * Write a series of large files to it. 224193323Sed */ 225193323Sed for (i = 0; tests[i] != 0; i++) { 226193323Sed assert((ae = archive_entry_new()) != NULL); 227193323Sed sprintf(namebuff, "file_%d", i); 228193323Sed archive_entry_copy_pathname(ae, namebuff); 229193323Sed archive_entry_set_mode(ae, S_IFREG | 0755); 230193323Sed filesize = tests[i]; 231193323Sed 232193323Sed archive_entry_set_size(ae, filesize); 233198090Srdivacky 234193323Sed assertA(0 == archive_write_header(a, ae)); 235198090Srdivacky archive_entry_free(ae); 236193323Sed 237198090Srdivacky /* 238198090Srdivacky * Write the actual data to the archive. 239193323Sed */ 240193323Sed while (filesize > 0) { 241198090Srdivacky writesize = filedatasize; 242193323Sed if ((int64_t)writesize > filesize) 243193323Sed writesize = (size_t)filesize; 244193323Sed assertA((int)writesize 245193323Sed == archive_write_data(a, filedata, writesize)); 246193323Sed filesize -= writesize; 247193323Sed } 248193323Sed } 249193323Sed 250193323Sed assert((ae = archive_entry_new()) != NULL); 251193323Sed archive_entry_copy_pathname(ae, "lastfile"); 252193323Sed archive_entry_set_mode(ae, S_IFREG | 0755); 253193323Sed assertA(0 == archive_write_header(a, ae)); 254193323Sed archive_entry_free(ae); 255193323Sed 256193323Sed 257193323Sed /* Close out the archive. */ 258193323Sed assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 259193323Sed assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 260193323Sed 261193323Sed /* 262193323Sed * Open the same archive for reading. 263193323Sed */ 264193323Sed a = archive_read_new(); 265193323Sed archive_read_support_format_tar(a); 266193323Sed archive_read_open2(a, &memdata, NULL, 267193323Sed memory_read, memory_read_skip, NULL); 268193323Sed 269193323Sed /* 270193323Sed * Read entries back. 271193323Sed */ 272193323Sed for (i = 0; tests[i] > 0; i++) { 273193323Sed assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 274193323Sed sprintf(namebuff, "file_%d", i); 275193323Sed assertEqualString(namebuff, archive_entry_pathname(ae)); 276193323Sed assert(tests[i] == archive_entry_size(ae)); 277193323Sed } 278193323Sed assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 279193323Sed assertEqualString("lastfile", archive_entry_pathname(ae)); 280193323Sed 281193323Sed assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 282193323Sed 283193323Sed /* Close out the archive. */ 284193323Sed assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 285193323Sed assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 286193323Sed 287193323Sed free(memdata.buff); 288193323Sed free(filedata); 289193323Sed} 290193323Sed