1/*- 2 * Copyright (c) 2003-2007 Tim Kientzle 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26__FBSDID("$FreeBSD$"); 27 28#include <errno.h> 29#include <stdlib.h> 30#include <string.h> 31 32/* 33 * This is a somewhat tricky test that verifies the ability to 34 * write and read very large entries to tar archives. It 35 * writes entries from 2GB up to 1TB to an archive in memory. 36 * The memory storage here carefully avoids actually storing 37 * any part of the file bodies, so it runs very quickly and requires 38 * very little memory. If you're willing to wait a few minutes, 39 * you should be able to exercise petabyte entries with this code. 40 */ 41 42/* 43 * Each file is built up by duplicating the following block. 44 */ 45static size_t filedatasize; 46static void *filedata; 47 48/* 49 * We store the archive as blocks of data generated by libarchive, 50 * each possibly followed by bytes of file data. 51 */ 52struct memblock { 53 struct memblock *next; 54 size_t size; 55 void *buff; 56 int64_t filebytes; 57}; 58 59/* 60 * The total memory store is just a list of memblocks plus 61 * some accounting overhead. 62 */ 63struct memdata { 64 int64_t filebytes; 65 void *buff; 66 struct memblock *first; 67 struct memblock *last; 68}; 69 70/* The following size definitions simplify things below. */ 71#define KB ((int64_t)1024) 72#define MB ((int64_t)1024 * KB) 73#define GB ((int64_t)1024 * MB) 74#define TB ((int64_t)1024 * GB) 75 76static int64_t memory_read_skip(struct archive *, void *, int64_t request); 77static ssize_t memory_read(struct archive *, void *, const void **buff); 78static ssize_t memory_write(struct archive *, void *, const void *, size_t); 79 80 81static ssize_t 82memory_write(struct archive *a, void *_private, const void *buff, size_t size) 83{ 84 struct memdata *private = _private; 85 struct memblock *block; 86 87 (void)a; 88 89 /* 90 * Since libarchive tries to behave in a zero-copy manner, if 91 * you give a pointer to filedata to the library, a pointer 92 * into that data will (usually) pop out here. This way, we 93 * can tell the difference between filedata and library header 94 * and metadata. 95 */ 96 if ((const char *)filedata <= (const char *)buff 97 && (const char *)buff < (const char *)filedata + filedatasize) { 98 /* We don't need to store a block of file data. */ 99 private->last->filebytes += (int64_t)size; 100 } else { 101 /* Yes, we're assuming the very first write is metadata. */ 102 /* It's header or metadata, copy and save it. */ 103 block = (struct memblock *)malloc(sizeof(*block)); 104 memset(block, 0, sizeof(*block)); 105 block->size = size; 106 block->buff = malloc(size); 107 memcpy(block->buff, buff, size); 108 if (private->last == NULL) { 109 private->first = private->last = block; 110 } else { 111 private->last->next = block; 112 private->last = block; 113 } 114 block->next = NULL; 115 } 116 return ((long)size); 117} 118 119static ssize_t 120memory_read(struct archive *a, void *_private, const void **buff) 121{ 122 struct memdata *private = _private; 123 struct memblock *block; 124 ssize_t size; 125 126 (void)a; 127 128 free(private->buff); 129 private->buff = NULL; 130 if (private->first == NULL) { 131 private->last = NULL; 132 return (ARCHIVE_EOF); 133 } 134 if (private->filebytes > 0) { 135 /* 136 * We're returning file bytes, simulate it by 137 * passing blocks from the template data. 138 */ 139 if (private->filebytes > (int64_t)filedatasize) 140 size = (ssize_t)filedatasize; 141 else 142 size = (ssize_t)private->filebytes; 143 private->filebytes -= size; 144 *buff = filedata; 145 } else { 146 /* 147 * We need to get some real data to return. 148 */ 149 block = private->first; 150 private->first = block->next; 151 size = (ssize_t)block->size; 152 if (block->buff != NULL) { 153 private->buff = block->buff; 154 *buff = block->buff; 155 } else { 156 private->buff = NULL; 157 *buff = filedata; 158 } 159 private->filebytes = block->filebytes; 160 free(block); 161 } 162 return (size); 163} 164 165 166static int64_t 167memory_read_skip(struct archive *a, void *_private, int64_t skip) 168{ 169 struct memdata *private = _private; 170 171 (void)a; 172 173 if (private->first == NULL) { 174 private->last = NULL; 175 return (0); 176 } 177 if (private->filebytes > 0) { 178 if (private->filebytes < skip) 179 skip = (off_t)private->filebytes; 180 private->filebytes -= skip; 181 } else { 182 skip = 0; 183 } 184 return (skip); 185} 186 187DEFINE_TEST(test_tar_large) 188{ 189 /* The sizes of the entries we're going to generate. */ 190 static int64_t tests[] = { 191 /* Test for 32-bit signed overflow. */ 192 2 * GB - 1, 2 * GB, 2 * GB + 1, 193 /* Test for 32-bit unsigned overflow. */ 194 4 * GB - 1, 4 * GB, 4 * GB + 1, 195 /* 8GB is the "official" max for ustar. */ 196 8 * GB - 1, 8 * GB, 8 * GB + 1, 197 /* Bend ustar a tad and you can get 64GB (12 octal digits). */ 198 64 * GB - 1, 64 * GB, 199 /* And larger entries that require non-ustar extensions. */ 200 256 * GB, 1 * TB, 0 }; 201 int i; 202 char namebuff[64]; 203 struct memdata memdata; 204 struct archive_entry *ae; 205 struct archive *a; 206 int64_t filesize; 207 size_t writesize; 208 209 filedatasize = (size_t)(1 * MB); 210 filedata = malloc(filedatasize); 211 memset(filedata, 0xAA, filedatasize); 212 memset(&memdata, 0, sizeof(memdata)); 213 214 /* 215 * Open an archive for writing. 216 */ 217 a = archive_write_new(); 218 archive_write_set_format_pax_restricted(a); 219 archive_write_set_bytes_per_block(a, 0); /* No buffering. */ 220 archive_write_open(a, &memdata, NULL, memory_write, NULL); 221 222 /* 223 * Write a series of large files to it. 224 */ 225 for (i = 0; tests[i] != 0; i++) { 226 assert((ae = archive_entry_new()) != NULL); 227 sprintf(namebuff, "file_%d", i); 228 archive_entry_copy_pathname(ae, namebuff); 229 archive_entry_set_mode(ae, S_IFREG | 0755); 230 filesize = tests[i]; 231 232 archive_entry_set_size(ae, filesize); 233 234 assertA(0 == archive_write_header(a, ae)); 235 archive_entry_free(ae); 236 237 /* 238 * Write the actual data to the archive. 239 */ 240 while (filesize > 0) { 241 writesize = filedatasize; 242 if ((int64_t)writesize > filesize) 243 writesize = (size_t)filesize; 244 assertA((int)writesize 245 == archive_write_data(a, filedata, writesize)); 246 filesize -= writesize; 247 } 248 } 249 250 assert((ae = archive_entry_new()) != NULL); 251 archive_entry_copy_pathname(ae, "lastfile"); 252 archive_entry_set_mode(ae, S_IFREG | 0755); 253 assertA(0 == archive_write_header(a, ae)); 254 archive_entry_free(ae); 255 256 257 /* Close out the archive. */ 258 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 259 assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 260 261 /* 262 * Open the same archive for reading. 263 */ 264 a = archive_read_new(); 265 archive_read_support_format_tar(a); 266 archive_read_open2(a, &memdata, NULL, 267 memory_read, memory_read_skip, NULL); 268 269 /* 270 * Read entries back. 271 */ 272 for (i = 0; tests[i] > 0; i++) { 273 assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 274 sprintf(namebuff, "file_%d", i); 275 assertEqualString(namebuff, archive_entry_pathname(ae)); 276 assert(tests[i] == archive_entry_size(ae)); 277 } 278 assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 279 assertEqualString("lastfile", archive_entry_pathname(ae)); 280 281 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 282 283 /* Close out the archive. */ 284 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 285 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 286 287 free(memdata.buff); 288 free(filedata); 289} 290