1/* 2 * Copyright (c) Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12/* ************************************* 13* Compiler Options 14***************************************/ 15#ifdef _MSC_VER /* Visual */ 16# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 17# pragma warning(disable : 4204) /* non-constant aggregate initializer */ 18#endif 19#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 20# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 21#endif 22 23/*-************************************* 24* Includes 25***************************************/ 26#include "platform.h" /* Large Files support, SET_BINARY_MODE */ 27#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ 28#include <stdio.h> /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ 29#include <stdlib.h> /* malloc, free */ 30#include <string.h> /* strcmp, strlen */ 31#include <fcntl.h> /* O_WRONLY */ 32#include <assert.h> 33#include <errno.h> /* errno */ 34#include <limits.h> /* INT_MAX */ 35#include <signal.h> 36#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ 37 38#if defined (_MSC_VER) 39# include <sys/stat.h> 40# include <io.h> 41#endif 42 43#include "../lib/common/mem.h" /* U32, U64 */ 44#include "fileio.h" 45 46#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ 47#include "../lib/zstd.h" 48#include "../lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ 49 50#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) 51# include <zlib.h> 52# if !defined(z_const) 53# define z_const 54# endif 55#endif 56 57#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) 58# include <lzma.h> 59#endif 60 61#define LZ4_MAGICNUMBER 0x184D2204 62#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) 63# define LZ4F_ENABLE_OBSOLETE_ENUMS 64# include <lz4frame.h> 65# include <lz4.h> 66#endif 67 68 69/*-************************************* 70* Constants 71***************************************/ 72#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ 73#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ 74 75#define FNSPACE 30 76 77/* Default file permissions 0666 (modulated by umask) */ 78#if !defined(_WIN32) 79/* These macros aren't defined on windows. */ 80#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) 81#else 82#define DEFAULT_FILE_PERMISSIONS (0666) 83#endif 84 85/*-************************************* 86* Macros 87***************************************/ 88#define KB *(1 <<10) 89#define MB *(1 <<20) 90#define GB *(1U<<30) 91#undef MAX 92#define MAX(a,b) ((a)>(b) ? (a) : (b)) 93 94struct FIO_display_prefs_s { 95 int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ 96 FIO_progressSetting_e progressSetting; 97}; 98 99static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; 100 101#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 102#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) 103#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 104 105static const U64 g_refreshRate = SEC_TO_MICRO / 6; 106static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 107 108#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) 109#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } 110#define DISPLAYUPDATE(l, ...) { \ 111 if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ 112 if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ 113 DELAY_NEXT_UPDATE(); \ 114 DISPLAY(__VA_ARGS__); \ 115 if (g_display_prefs.displayLevel>=4) fflush(stderr); \ 116 } } } 117 118#undef MIN /* in case it would be already defined */ 119#define MIN(a,b) ((a) < (b) ? (a) : (b)) 120 121 122#define EXM_THROW(error, ...) \ 123{ \ 124 DISPLAYLEVEL(1, "zstd: "); \ 125 DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 126 DISPLAYLEVEL(1, "error %i : ", error); \ 127 DISPLAYLEVEL(1, __VA_ARGS__); \ 128 DISPLAYLEVEL(1, " \n"); \ 129 exit(error); \ 130} 131 132#define CHECK_V(v, f) \ 133 v = f; \ 134 if (ZSTD_isError(v)) { \ 135 DISPLAYLEVEL(5, "%s \n", #f); \ 136 EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ 137 } 138#define CHECK(f) { size_t err; CHECK_V(err, f); } 139 140 141/*-************************************ 142* Signal (Ctrl-C trapping) 143**************************************/ 144static const char* g_artefact = NULL; 145static void INThandler(int sig) 146{ 147 assert(sig==SIGINT); (void)sig; 148#if !defined(_MSC_VER) 149 signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ 150#endif 151 if (g_artefact) { 152 assert(UTIL_isRegularFile(g_artefact)); 153 remove(g_artefact); 154 } 155 DISPLAY("\n"); 156 exit(2); 157} 158static void addHandler(char const* dstFileName) 159{ 160 if (UTIL_isRegularFile(dstFileName)) { 161 g_artefact = dstFileName; 162 signal(SIGINT, INThandler); 163 } else { 164 g_artefact = NULL; 165 } 166} 167/* Idempotent */ 168static void clearHandler(void) 169{ 170 if (g_artefact) signal(SIGINT, SIG_DFL); 171 g_artefact = NULL; 172} 173 174 175/*-********************************************************* 176* Termination signal trapping (Print debug stack trace) 177***********************************************************/ 178#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */ 179# if (__has_feature(address_sanitizer)) 180# define BACKTRACE_ENABLE 0 181# endif /* __has_feature(address_sanitizer) */ 182#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */ 183# define BACKTRACE_ENABLE 0 184#endif 185 186#if !defined(BACKTRACE_ENABLE) 187/* automatic detector : backtrace enabled by default on linux+glibc and osx */ 188# if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \ 189 || (defined(__APPLE__) && defined(__MACH__)) 190# define BACKTRACE_ENABLE 1 191# else 192# define BACKTRACE_ENABLE 0 193# endif 194#endif 195 196/* note : after this point, BACKTRACE_ENABLE is necessarily defined */ 197 198 199#if BACKTRACE_ENABLE 200 201#include <execinfo.h> /* backtrace, backtrace_symbols */ 202 203#define MAX_STACK_FRAMES 50 204 205static void ABRThandler(int sig) { 206 const char* name; 207 void* addrlist[MAX_STACK_FRAMES]; 208 char** symbollist; 209 int addrlen, i; 210 211 switch (sig) { 212 case SIGABRT: name = "SIGABRT"; break; 213 case SIGFPE: name = "SIGFPE"; break; 214 case SIGILL: name = "SIGILL"; break; 215 case SIGINT: name = "SIGINT"; break; 216 case SIGSEGV: name = "SIGSEGV"; break; 217 default: name = "UNKNOWN"; 218 } 219 220 DISPLAY("Caught %s signal, printing stack:\n", name); 221 /* Retrieve current stack addresses. */ 222 addrlen = backtrace(addrlist, MAX_STACK_FRAMES); 223 if (addrlen == 0) { 224 DISPLAY("\n"); 225 return; 226 } 227 /* Create readable strings to each frame. */ 228 symbollist = backtrace_symbols(addrlist, addrlen); 229 /* Print the stack trace, excluding calls handling the signal. */ 230 for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) { 231 DISPLAY("%s\n", symbollist[i]); 232 } 233 free(symbollist); 234 /* Reset and raise the signal so default handler runs. */ 235 signal(sig, SIG_DFL); 236 raise(sig); 237} 238#endif 239 240void FIO_addAbortHandler() 241{ 242#if BACKTRACE_ENABLE 243 signal(SIGABRT, ABRThandler); 244 signal(SIGFPE, ABRThandler); 245 signal(SIGILL, ABRThandler); 246 signal(SIGSEGV, ABRThandler); 247 signal(SIGBUS, ABRThandler); 248#endif 249} 250 251 252/*-************************************************************ 253* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW 254***************************************************************/ 255#if defined(_MSC_VER) && _MSC_VER >= 1400 256# define LONG_SEEK _fseeki64 257# define LONG_TELL _ftelli64 258#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 259# define LONG_SEEK fseeko 260# define LONG_TELL ftello 261#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) 262# define LONG_SEEK fseeko64 263# define LONG_TELL ftello64 264#elif defined(_WIN32) && !defined(__DJGPP__) 265# include <windows.h> 266 static int LONG_SEEK(FILE* file, __int64 offset, int origin) { 267 LARGE_INTEGER off; 268 DWORD method; 269 off.QuadPart = offset; 270 if (origin == SEEK_END) 271 method = FILE_END; 272 else if (origin == SEEK_CUR) 273 method = FILE_CURRENT; 274 else 275 method = FILE_BEGIN; 276 277 if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) 278 return 0; 279 else 280 return -1; 281 } 282 static __int64 LONG_TELL(FILE* file) { 283 LARGE_INTEGER off, newOff; 284 off.QuadPart = 0; 285 newOff.QuadPart = 0; 286 SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); 287 return newOff.QuadPart; 288 } 289#else 290# define LONG_SEEK fseek 291# define LONG_TELL ftell 292#endif 293 294 295/*-************************************* 296* Parameters: FIO_prefs_t 297***************************************/ 298 299/* typedef'd to FIO_prefs_t within fileio.h */ 300struct FIO_prefs_s { 301 302 /* Algorithm preferences */ 303 FIO_compressionType_t compressionType; 304 U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ 305 int dictIDFlag; 306 int checksumFlag; 307 int blockSize; 308 int overlapLog; 309 U32 adaptiveMode; 310 U32 useRowMatchFinder; 311 int rsyncable; 312 int minAdaptLevel; 313 int maxAdaptLevel; 314 int ldmFlag; 315 int ldmHashLog; 316 int ldmMinMatch; 317 int ldmBucketSizeLog; 318 int ldmHashRateLog; 319 size_t streamSrcSize; 320 size_t targetCBlockSize; 321 int srcSizeHint; 322 int testMode; 323 ZSTD_paramSwitch_e literalCompressionMode; 324 325 /* IO preferences */ 326 U32 removeSrcFile; 327 U32 overwrite; 328 329 /* Computation resources preferences */ 330 unsigned memLimit; 331 int nbWorkers; 332 333 int excludeCompressedFiles; 334 int patchFromMode; 335 int contentSize; 336 int allowBlockDevices; 337}; 338 339/*-************************************* 340* Parameters: FIO_ctx_t 341***************************************/ 342 343/* typedef'd to FIO_ctx_t within fileio.h */ 344struct FIO_ctx_s { 345 346 /* file i/o info */ 347 int nbFilesTotal; 348 int hasStdinInput; 349 int hasStdoutOutput; 350 351 /* file i/o state */ 352 int currFileIdx; 353 int nbFilesProcessed; 354 size_t totalBytesInput; 355 size_t totalBytesOutput; 356}; 357 358 359/*-************************************* 360* Parameters: Initialization 361***************************************/ 362 363#define FIO_OVERLAP_LOG_NOTSET 9999 364#define FIO_LDM_PARAM_NOTSET 9999 365 366 367FIO_prefs_t* FIO_createPreferences(void) 368{ 369 FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); 370 if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); 371 372 ret->compressionType = FIO_zstdCompression; 373 ret->overwrite = 0; 374 ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; 375 ret->dictIDFlag = 1; 376 ret->checksumFlag = 1; 377 ret->removeSrcFile = 0; 378 ret->memLimit = 0; 379 ret->nbWorkers = 1; 380 ret->blockSize = 0; 381 ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; 382 ret->adaptiveMode = 0; 383 ret->rsyncable = 0; 384 ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ 385 ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ 386 ret->ldmFlag = 0; 387 ret->ldmHashLog = 0; 388 ret->ldmMinMatch = 0; 389 ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; 390 ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; 391 ret->streamSrcSize = 0; 392 ret->targetCBlockSize = 0; 393 ret->srcSizeHint = 0; 394 ret->testMode = 0; 395 ret->literalCompressionMode = ZSTD_ps_auto; 396 ret->excludeCompressedFiles = 0; 397 ret->allowBlockDevices = 0; 398 return ret; 399} 400 401FIO_ctx_t* FIO_createContext(void) 402{ 403 FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); 404 if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); 405 406 ret->currFileIdx = 0; 407 ret->hasStdinInput = 0; 408 ret->hasStdoutOutput = 0; 409 ret->nbFilesTotal = 1; 410 ret->nbFilesProcessed = 0; 411 ret->totalBytesInput = 0; 412 ret->totalBytesOutput = 0; 413 return ret; 414} 415 416void FIO_freePreferences(FIO_prefs_t* const prefs) 417{ 418 free(prefs); 419} 420 421void FIO_freeContext(FIO_ctx_t* const fCtx) 422{ 423 free(fCtx); 424} 425 426 427/*-************************************* 428* Parameters: Display Options 429***************************************/ 430 431void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } 432 433void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } 434 435 436/*-************************************* 437* Parameters: Setters 438***************************************/ 439 440/* FIO_prefs_t functions */ 441 442void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } 443 444void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } 445 446void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } 447 448void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } 449 450void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } 451 452void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } 453 454void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } 455 456void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { 457#ifndef ZSTD_MULTITHREAD 458 if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); 459#endif 460 prefs->nbWorkers = nbWorkers; 461} 462 463void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } 464 465void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } 466 467void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { 468 if (blockSize && prefs->nbWorkers==0) 469 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); 470 prefs->blockSize = blockSize; 471} 472 473void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ 474 if (overlapLog && prefs->nbWorkers==0) 475 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); 476 prefs->overlapLog = overlapLog; 477} 478 479void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { 480 if ((adapt>0) && (prefs->nbWorkers==0)) 481 EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); 482 prefs->adaptiveMode = adapt; 483} 484 485void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) { 486 prefs->useRowMatchFinder = useRowMatchFinder; 487} 488 489void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { 490 if ((rsyncable>0) && (prefs->nbWorkers==0)) 491 EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); 492 prefs->rsyncable = rsyncable; 493} 494 495void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { 496 prefs->streamSrcSize = streamSrcSize; 497} 498 499void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { 500 prefs->targetCBlockSize = targetCBlockSize; 501} 502 503void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { 504 prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); 505} 506 507void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { 508 prefs->testMode = (testMode!=0); 509} 510 511void FIO_setLiteralCompressionMode( 512 FIO_prefs_t* const prefs, 513 ZSTD_paramSwitch_e mode) { 514 prefs->literalCompressionMode = mode; 515} 516 517void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) 518{ 519#ifndef ZSTD_NOCOMPRESS 520 assert(minCLevel >= ZSTD_minCLevel()); 521#endif 522 prefs->minAdaptLevel = minCLevel; 523} 524 525void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) 526{ 527 prefs->maxAdaptLevel = maxCLevel; 528} 529 530void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { 531 prefs->ldmFlag = (ldmFlag>0); 532} 533 534void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { 535 prefs->ldmHashLog = ldmHashLog; 536} 537 538void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { 539 prefs->ldmMinMatch = ldmMinMatch; 540} 541 542void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { 543 prefs->ldmBucketSizeLog = ldmBucketSizeLog; 544} 545 546 547void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { 548 prefs->ldmHashRateLog = ldmHashRateLog; 549} 550 551void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value) 552{ 553 prefs->patchFromMode = value != 0; 554} 555 556void FIO_setContentSize(FIO_prefs_t* const prefs, int value) 557{ 558 prefs->contentSize = value != 0; 559} 560 561/* FIO_ctx_t functions */ 562 563void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { 564 fCtx->hasStdoutOutput = value; 565} 566 567void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) 568{ 569 fCtx->nbFilesTotal = value; 570} 571 572void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) { 573 size_t i = 0; 574 for ( ; i < filenames->tableSize; ++i) { 575 if (!strcmp(stdinmark, filenames->fileNames[i])) { 576 fCtx->hasStdinInput = 1; 577 return; 578 } 579 } 580} 581 582/*-************************************* 583* Functions 584***************************************/ 585/** FIO_removeFile() : 586 * @result : Unlink `fileName`, even if it's read-only */ 587static int FIO_removeFile(const char* path) 588{ 589 stat_t statbuf; 590 if (!UTIL_stat(path, &statbuf)) { 591 DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path); 592 return 0; 593 } 594 if (!UTIL_isRegularFileStat(&statbuf)) { 595 DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); 596 return 0; 597 } 598#if defined(_WIN32) || defined(WIN32) 599 /* windows doesn't allow remove read-only files, 600 * so try to make it writable first */ 601 if (!(statbuf.st_mode & _S_IWRITE)) { 602 UTIL_chmod(path, &statbuf, _S_IWRITE); 603 } 604#endif 605 return remove(path); 606} 607 608/** FIO_openSrcFile() : 609 * condition : `srcFileName` must be non-NULL. `prefs` may be NULL. 610 * @result : FILE* to `srcFileName`, or NULL if it fails */ 611static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName) 612{ 613 stat_t statbuf; 614 int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; 615 assert(srcFileName != NULL); 616 if (!strcmp (srcFileName, stdinmark)) { 617 DISPLAYLEVEL(4,"Using stdin for input \n"); 618 SET_BINARY_MODE(stdin); 619 return stdin; 620 } 621 622 if (!UTIL_stat(srcFileName, &statbuf)) { 623 DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", 624 srcFileName, strerror(errno)); 625 return NULL; 626 } 627 628 if (!UTIL_isRegularFileStat(&statbuf) 629 && !UTIL_isFIFOStat(&statbuf) 630 && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf)) 631 ) { 632 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", 633 srcFileName); 634 return NULL; 635 } 636 637 { FILE* const f = fopen(srcFileName, "rb"); 638 if (f == NULL) 639 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 640 return f; 641 } 642} 643 644/** FIO_openDstFile() : 645 * condition : `dstFileName` must be non-NULL. 646 * @result : FILE* to `dstFileName`, or NULL if it fails */ 647static FILE* 648FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, 649 const char* srcFileName, const char* dstFileName, 650 const int mode) 651{ 652 if (prefs->testMode) return NULL; /* do not open file in test mode */ 653 654 assert(dstFileName != NULL); 655 if (!strcmp (dstFileName, stdoutmark)) { 656 DISPLAYLEVEL(4,"Using stdout for output \n"); 657 SET_BINARY_MODE(stdout); 658 if (prefs->sparseFileSupport == 1) { 659 prefs->sparseFileSupport = 0; 660 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 661 } 662 return stdout; 663 } 664 665 /* ensure dst is not the same as src */ 666 if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { 667 DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); 668 return NULL; 669 } 670 671 if (prefs->sparseFileSupport == 1) { 672 prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; 673 } 674 675 if (UTIL_isRegularFile(dstFileName)) { 676 /* Check if destination file already exists */ 677#if !defined(_WIN32) 678 /* this test does not work on Windows : 679 * `NUL` and `nul` are detected as regular files */ 680 if (!strcmp(dstFileName, nulmark)) { 681 EXM_THROW(40, "%s is unexpectedly categorized as a regular file", 682 dstFileName); 683 } 684#endif 685 if (!prefs->overwrite) { 686 if (g_display_prefs.displayLevel <= 1) { 687 /* No interaction possible */ 688 DISPLAY("zstd: %s already exists; not overwritten \n", 689 dstFileName); 690 return NULL; 691 } 692 DISPLAY("zstd: %s already exists; ", dstFileName); 693 if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) 694 return NULL; 695 } 696 /* need to unlink */ 697 FIO_removeFile(dstFileName); 698 } 699 700 { 701#if defined(_WIN32) 702 /* Windows requires opening the file as a "binary" file to avoid 703 * mangling. This macro doesn't exist on unix. */ 704 const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; 705 const int fd = _open(dstFileName, openflags, mode); 706 FILE* f = NULL; 707 if (fd != -1) { 708 f = _fdopen(fd, "wb"); 709 } 710#else 711 const int openflags = O_WRONLY|O_CREAT|O_TRUNC; 712 const int fd = open(dstFileName, openflags, mode); 713 FILE* f = NULL; 714 if (fd != -1) { 715 f = fdopen(fd, "wb"); 716 } 717#endif 718 if (f == NULL) { 719 DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); 720 } 721 return f; 722 } 723} 724 725/*! FIO_createDictBuffer() : 726 * creates a buffer, pointed by `*bufferPtr`, 727 * loads `filename` content into it, up to DICTSIZE_MAX bytes. 728 * @return : loaded size 729 * if fileName==NULL, returns 0 and a NULL pointer 730 */ 731static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs) 732{ 733 FILE* fileHandle; 734 U64 fileSize; 735 stat_t statbuf; 736 737 assert(bufferPtr != NULL); 738 *bufferPtr = NULL; 739 if (fileName == NULL) return 0; 740 741 DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); 742 743 if (!UTIL_stat(fileName, &statbuf)) { 744 EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno)); 745 } 746 747 if (!UTIL_isRegularFileStat(&statbuf)) { 748 EXM_THROW(32, "Dictionary %s must be a regular file.", fileName); 749 } 750 751 fileHandle = fopen(fileName, "rb"); 752 753 if (fileHandle == NULL) { 754 EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno)); 755 } 756 757 fileSize = UTIL_getFileSizeStat(&statbuf); 758 { 759 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; 760 if (fileSize > dictSizeMax) { 761 EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", 762 fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ 763 } 764 } 765 *bufferPtr = malloc((size_t)fileSize); 766 if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); 767 { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); 768 if (readSize != fileSize) { 769 EXM_THROW(35, "Error reading dictionary file %s : %s", 770 fileName, strerror(errno)); 771 } 772 } 773 fclose(fileHandle); 774 return (size_t)fileSize; 775} 776 777 778 779/* FIO_checkFilenameCollisions() : 780 * Checks for and warns if there are any files that would have the same output path 781 */ 782int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { 783 const char **filenameTableSorted, *prevElem, *filename; 784 unsigned u; 785 786 filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); 787 if (!filenameTableSorted) { 788 DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); 789 return 1; 790 } 791 792 for (u = 0; u < nbFiles; ++u) { 793 filename = strrchr(filenameTable[u], PATH_SEP); 794 if (filename == NULL) { 795 filenameTableSorted[u] = filenameTable[u]; 796 } else { 797 filenameTableSorted[u] = filename+1; 798 } 799 } 800 801 qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); 802 prevElem = filenameTableSorted[0]; 803 for (u = 1; u < nbFiles; ++u) { 804 if (strcmp(prevElem, filenameTableSorted[u]) == 0) { 805 DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); 806 } 807 prevElem = filenameTableSorted[u]; 808 } 809 810 free((void*)filenameTableSorted); 811 return 0; 812} 813 814static const char* 815extractFilename(const char* path, char separator) 816{ 817 const char* search = strrchr(path, separator); 818 if (search == NULL) return path; 819 return search+1; 820} 821 822/* FIO_createFilename_fromOutDir() : 823 * Takes a source file name and specified output directory, and 824 * allocates memory for and returns a pointer to final path. 825 * This function never returns an error (it may abort() in case of pb) 826 */ 827static char* 828FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen) 829{ 830 const char* filenameStart; 831 char separator; 832 char* result; 833 834#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ 835 separator = '\\'; 836#else 837 separator = '/'; 838#endif 839 840 filenameStart = extractFilename(path, separator); 841#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ 842 filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */ 843#endif 844 845 result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1); 846 if (!result) { 847 EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno)); 848 } 849 850 memcpy(result, outDirName, strlen(outDirName)); 851 if (outDirName[strlen(outDirName)-1] == separator) { 852 memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart)); 853 } else { 854 memcpy(result + strlen(outDirName), &separator, 1); 855 memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart)); 856 } 857 858 return result; 859} 860 861/* FIO_highbit64() : 862 * gives position of highest bit. 863 * note : only works for v > 0 ! 864 */ 865static unsigned FIO_highbit64(unsigned long long v) 866{ 867 unsigned count = 0; 868 assert(v != 0); 869 v >>= 1; 870 while (v) { v >>= 1; count++; } 871 return count; 872} 873 874static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, 875 unsigned long long const dictSize, 876 unsigned long long const maxSrcFileSize) 877{ 878 unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); 879 unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX); 880 if (maxSize == UTIL_FILESIZE_UNKNOWN) 881 EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); 882 assert(maxSize != UTIL_FILESIZE_UNKNOWN); 883 if (maxSize > maxWindowSize) 884 EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB)); 885 FIO_setMemLimit(prefs, (unsigned)maxSize); 886} 887 888/* FIO_removeMultiFilesWarning() : 889 * Returns 1 if the console should abort, 0 if console should proceed. 890 * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts. 891 * 892 * If -f is specified, or there is just 1 file, zstd will always proceed as usual. 893 * If --rm is specified, there will be a prompt asking for user confirmation. 894 * If -f is specified with --rm, zstd will proceed as usual 895 * If -q is specified with --rm, zstd will abort pre-emptively 896 * If neither flag is specified, zstd will prompt the user for confirmation to proceed. 897 * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). 898 * However, if the output is stdout, we will always abort rather than displaying the warning prompt. 899 */ 900static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) 901{ 902 int error = 0; 903 if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { 904 if (g_display_prefs.displayLevel <= displayLevelCutoff) { 905 if (prefs->removeSrcFile) { 906 DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName); 907 error = 1; 908 } 909 } else { 910 if (!strcmp(outFileName, stdoutmark)) { 911 DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n"); 912 } else { 913 DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName); 914 } 915 DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n") 916 if (prefs->removeSrcFile) { 917 if (fCtx->hasStdoutOutput) { 918 DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n"); 919 error = 1; 920 } else { 921 error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); 922 } 923 } 924 } 925 } 926 return error; 927} 928 929#ifndef ZSTD_NOCOMPRESS 930 931/* ********************************************************************** 932 * Compression 933 ************************************************************************/ 934typedef struct { 935 FILE* srcFile; 936 FILE* dstFile; 937 void* srcBuffer; 938 size_t srcBufferSize; 939 void* dstBuffer; 940 size_t dstBufferSize; 941 void* dictBuffer; 942 size_t dictBufferSize; 943 const char* dictFileName; 944 ZSTD_CStream* cctx; 945} cRess_t; 946 947/** ZSTD_cycleLog() : 948 * condition for correct operation : hashLog > 1 */ 949static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) 950{ 951 U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); 952 assert(hashLog > 1); 953 return hashLog - btScale; 954} 955 956static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, 957 ZSTD_compressionParameters* comprParams, 958 unsigned long long const dictSize, 959 unsigned long long const maxSrcFileSize, 960 int cLevel) 961{ 962 unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1; 963 ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize); 964 FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize); 965 if (fileWindowLog > ZSTD_WINDOWLOG_MAX) 966 DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n"); 967 comprParams->windowLog = MAX(ZSTD_WINDOWLOG_MIN, MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog)); 968 if (fileWindowLog > ZSTD_cycleLog(cParams.chainLog, cParams.strategy)) { 969 if (!prefs->ldmFlag) 970 DISPLAYLEVEL(1, "long mode automatically triggered\n"); 971 FIO_setLdmFlag(prefs, 1); 972 } 973 if (cParams.strategy >= ZSTD_btopt) { 974 DISPLAYLEVEL(1, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); 975 DISPLAYLEVEL(1, "- Use --single-thread mode in the zstd cli\n"); 976 DISPLAYLEVEL(1, "- Set a larger targetLength (eg. --zstd=targetLength=4096)\n"); 977 DISPLAYLEVEL(1, "- Set a larger chainLog (eg. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); 978 DISPLAYLEVEL(1, "Also consider playing around with searchLog and hashLog\n"); 979 } 980} 981 982static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, 983 const char* dictFileName, unsigned long long const maxSrcFileSize, 984 int cLevel, ZSTD_compressionParameters comprParams) { 985 cRess_t ress; 986 memset(&ress, 0, sizeof(ress)); 987 988 DISPLAYLEVEL(6, "FIO_createCResources \n"); 989 ress.cctx = ZSTD_createCCtx(); 990 if (ress.cctx == NULL) 991 EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", 992 strerror(errno)); 993 ress.srcBufferSize = ZSTD_CStreamInSize(); 994 ress.srcBuffer = malloc(ress.srcBufferSize); 995 ress.dstBufferSize = ZSTD_CStreamOutSize(); 996 997 /* need to update memLimit before calling createDictBuffer 998 * because of memLimit check inside it */ 999 if (prefs->patchFromMode) { 1000 unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; 1001 FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); 1002 } 1003 ress.dstBuffer = malloc(ress.dstBufferSize); 1004 ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ 1005 if (!ress.srcBuffer || !ress.dstBuffer) 1006 EXM_THROW(31, "allocation error : not enough memory"); 1007 1008 /* Advanced parameters, including dictionary */ 1009 if (dictFileName && (ress.dictBuffer==NULL)) 1010 EXM_THROW(32, "allocation error : can't create dictBuffer"); 1011 ress.dictFileName = dictFileName; 1012 1013 if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) 1014 comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; 1015 1016 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */ 1017 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); 1018 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); 1019 /* compression level */ 1020 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); 1021 /* max compressed block size */ 1022 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) ); 1023 /* source size hint */ 1024 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) ); 1025 /* long distance matching */ 1026 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); 1027 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); 1028 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); 1029 if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { 1030 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); 1031 } 1032 if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { 1033 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); 1034 } 1035 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder)); 1036 /* compression parameters */ 1037 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) ); 1038 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) ); 1039 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) ); 1040 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) ); 1041 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); 1042 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); 1043 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, (int)comprParams.strategy) ); 1044 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); 1045 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) ); 1046 /* multi-threading */ 1047#ifdef ZSTD_MULTITHREAD 1048 DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); 1049 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); 1050 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); 1051 if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { 1052 DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); 1053 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); 1054 } 1055 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); 1056#endif 1057 /* dictionary */ 1058 if (prefs->patchFromMode) { 1059 CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); 1060 } else { 1061 CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); 1062 } 1063 1064 return ress; 1065} 1066 1067static void FIO_freeCResources(const cRess_t* const ress) 1068{ 1069 free(ress->srcBuffer); 1070 free(ress->dstBuffer); 1071 free(ress->dictBuffer); 1072 ZSTD_freeCStream(ress->cctx); /* never fails */ 1073} 1074 1075 1076#ifdef ZSTD_GZCOMPRESS 1077static unsigned long long 1078FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */ 1079 const char* srcFileName, U64 const srcFileSize, 1080 int compressionLevel, U64* readsize) 1081{ 1082 unsigned long long inFileSize = 0, outFileSize = 0; 1083 z_stream strm; 1084 1085 if (compressionLevel > Z_BEST_COMPRESSION) 1086 compressionLevel = Z_BEST_COMPRESSION; 1087 1088 strm.zalloc = Z_NULL; 1089 strm.zfree = Z_NULL; 1090 strm.opaque = Z_NULL; 1091 1092 { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 1093 15 /* maxWindowLogSize */ + 16 /* gzip only */, 1094 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 1095 if (ret != Z_OK) { 1096 EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); 1097 } } 1098 1099 strm.next_in = 0; 1100 strm.avail_in = 0; 1101 strm.next_out = (Bytef*)ress->dstBuffer; 1102 strm.avail_out = (uInt)ress->dstBufferSize; 1103 1104 while (1) { 1105 int ret; 1106 if (strm.avail_in == 0) { 1107 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 1108 if (inSize == 0) break; 1109 inFileSize += inSize; 1110 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1111 strm.avail_in = (uInt)inSize; 1112 } 1113 ret = deflate(&strm, Z_NO_FLUSH); 1114 if (ret != Z_OK) 1115 EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); 1116 { size_t const cSize = ress->dstBufferSize - strm.avail_out; 1117 if (cSize) { 1118 if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) 1119 EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); 1120 outFileSize += cSize; 1121 strm.next_out = (Bytef*)ress->dstBuffer; 1122 strm.avail_out = (uInt)ress->dstBufferSize; 1123 } } 1124 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { 1125 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", 1126 (unsigned)(inFileSize>>20), 1127 (double)outFileSize/inFileSize*100) 1128 } else { 1129 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ", 1130 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 1131 (double)outFileSize/inFileSize*100); 1132 } } 1133 1134 while (1) { 1135 int const ret = deflate(&strm, Z_FINISH); 1136 { size_t const cSize = ress->dstBufferSize - strm.avail_out; 1137 if (cSize) { 1138 if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) 1139 EXM_THROW(75, "Write error : %s ", strerror(errno)); 1140 outFileSize += cSize; 1141 strm.next_out = (Bytef*)ress->dstBuffer; 1142 strm.avail_out = (uInt)ress->dstBufferSize; 1143 } } 1144 if (ret == Z_STREAM_END) break; 1145 if (ret != Z_BUF_ERROR) 1146 EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); 1147 } 1148 1149 { int const ret = deflateEnd(&strm); 1150 if (ret != Z_OK) { 1151 EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); 1152 } } 1153 *readsize = inFileSize; 1154 return outFileSize; 1155} 1156#endif 1157 1158 1159#ifdef ZSTD_LZMACOMPRESS 1160static unsigned long long 1161FIO_compressLzmaFrame(cRess_t* ress, 1162 const char* srcFileName, U64 const srcFileSize, 1163 int compressionLevel, U64* readsize, int plain_lzma) 1164{ 1165 unsigned long long inFileSize = 0, outFileSize = 0; 1166 lzma_stream strm = LZMA_STREAM_INIT; 1167 lzma_action action = LZMA_RUN; 1168 lzma_ret ret; 1169 1170 if (compressionLevel < 0) compressionLevel = 0; 1171 if (compressionLevel > 9) compressionLevel = 9; 1172 1173 if (plain_lzma) { 1174 lzma_options_lzma opt_lzma; 1175 if (lzma_lzma_preset(&opt_lzma, compressionLevel)) 1176 EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName); 1177 ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ 1178 if (ret != LZMA_OK) 1179 EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); 1180 } else { 1181 ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ 1182 if (ret != LZMA_OK) 1183 EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); 1184 } 1185 1186 strm.next_in = 0; 1187 strm.avail_in = 0; 1188 strm.next_out = (BYTE*)ress->dstBuffer; 1189 strm.avail_out = ress->dstBufferSize; 1190 1191 while (1) { 1192 if (strm.avail_in == 0) { 1193 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 1194 if (inSize == 0) action = LZMA_FINISH; 1195 inFileSize += inSize; 1196 strm.next_in = (BYTE const*)ress->srcBuffer; 1197 strm.avail_in = inSize; 1198 } 1199 1200 ret = lzma_code(&strm, action); 1201 1202 if (ret != LZMA_OK && ret != LZMA_STREAM_END) 1203 EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); 1204 { size_t const compBytes = ress->dstBufferSize - strm.avail_out; 1205 if (compBytes) { 1206 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) 1207 EXM_THROW(85, "Write error : %s", strerror(errno)); 1208 outFileSize += compBytes; 1209 strm.next_out = (BYTE*)ress->dstBuffer; 1210 strm.avail_out = ress->dstBufferSize; 1211 } } 1212 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 1213 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 1214 (unsigned)(inFileSize>>20), 1215 (double)outFileSize/inFileSize*100) 1216 else 1217 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 1218 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 1219 (double)outFileSize/inFileSize*100); 1220 if (ret == LZMA_STREAM_END) break; 1221 } 1222 1223 lzma_end(&strm); 1224 *readsize = inFileSize; 1225 1226 return outFileSize; 1227} 1228#endif 1229 1230#ifdef ZSTD_LZ4COMPRESS 1231 1232#if LZ4_VERSION_NUMBER <= 10600 1233#define LZ4F_blockLinked blockLinked 1234#define LZ4F_max64KB max64KB 1235#endif 1236 1237static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 1238 1239static unsigned long long 1240FIO_compressLz4Frame(cRess_t* ress, 1241 const char* srcFileName, U64 const srcFileSize, 1242 int compressionLevel, int checksumFlag, 1243 U64* readsize) 1244{ 1245 const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); 1246 unsigned long long inFileSize = 0, outFileSize = 0; 1247 1248 LZ4F_preferences_t prefs; 1249 LZ4F_compressionContext_t ctx; 1250 1251 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 1252 if (LZ4F_isError(errorCode)) 1253 EXM_THROW(31, "zstd: failed to create lz4 compression context"); 1254 1255 memset(&prefs, 0, sizeof(prefs)); 1256 1257 assert(blockSize <= ress->srcBufferSize); 1258 1259 prefs.autoFlush = 1; 1260 prefs.compressionLevel = compressionLevel; 1261 prefs.frameInfo.blockMode = LZ4F_blockLinked; 1262 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 1263 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag; 1264#if LZ4_VERSION_NUMBER >= 10600 1265 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; 1266#endif 1267 assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); 1268 1269 { 1270 size_t readSize; 1271 size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); 1272 if (LZ4F_isError(headerSize)) 1273 EXM_THROW(33, "File header generation failed : %s", 1274 LZ4F_getErrorName(headerSize)); 1275 if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) 1276 EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); 1277 outFileSize += headerSize; 1278 1279 /* Read first block */ 1280 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 1281 inFileSize += readSize; 1282 1283 /* Main Loop */ 1284 while (readSize>0) { 1285 size_t const outSize = LZ4F_compressUpdate(ctx, 1286 ress->dstBuffer, ress->dstBufferSize, 1287 ress->srcBuffer, readSize, NULL); 1288 if (LZ4F_isError(outSize)) 1289 EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", 1290 srcFileName, LZ4F_getErrorName(outSize)); 1291 outFileSize += outSize; 1292 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { 1293 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 1294 (unsigned)(inFileSize>>20), 1295 (double)outFileSize/inFileSize*100) 1296 } else { 1297 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 1298 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 1299 (double)outFileSize/inFileSize*100); 1300 } 1301 1302 /* Write Block */ 1303 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); 1304 if (sizeCheck != outSize) 1305 EXM_THROW(36, "Write error : %s", strerror(errno)); 1306 } 1307 1308 /* Read next block */ 1309 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 1310 inFileSize += readSize; 1311 } 1312 if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 1313 1314 /* End of Stream mark */ 1315 headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); 1316 if (LZ4F_isError(headerSize)) 1317 EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", 1318 srcFileName, LZ4F_getErrorName(headerSize)); 1319 1320 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 1321 if (sizeCheck != headerSize) 1322 EXM_THROW(39, "Write error : %s (cannot write end of stream)", 1323 strerror(errno)); 1324 } 1325 outFileSize += headerSize; 1326 } 1327 1328 *readsize = inFileSize; 1329 LZ4F_freeCompressionContext(ctx); 1330 1331 return outFileSize; 1332} 1333#endif 1334 1335 1336static unsigned long long 1337FIO_compressZstdFrame(FIO_ctx_t* const fCtx, 1338 FIO_prefs_t* const prefs, 1339 const cRess_t* ressPtr, 1340 const char* srcFileName, U64 fileSize, 1341 int compressionLevel, U64* readsize) 1342{ 1343 cRess_t const ress = *ressPtr; 1344 FILE* const srcFile = ress.srcFile; 1345 FILE* const dstFile = ress.dstFile; 1346 U64 compressedfilesize = 0; 1347 ZSTD_EndDirective directive = ZSTD_e_continue; 1348 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; 1349 1350 /* stats */ 1351 ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 }; 1352 ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 }; 1353 typedef enum { noChange, slower, faster } speedChange_e; 1354 speedChange_e speedChange = noChange; 1355 unsigned flushWaiting = 0; 1356 unsigned inputPresented = 0; 1357 unsigned inputBlocked = 0; 1358 unsigned lastJobID = 0; 1359 UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize); 1360 1361 DISPLAYLEVEL(6, "compression using zstd format \n"); 1362 1363 /* init */ 1364 if (fileSize != UTIL_FILESIZE_UNKNOWN) { 1365 pledgedSrcSize = fileSize; 1366 CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); 1367 } else if (prefs->streamSrcSize > 0) { 1368 /* unknown source size; use the declared stream size */ 1369 pledgedSrcSize = prefs->streamSrcSize; 1370 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); 1371 } 1372 1373 { 1374 int windowLog; 1375 UTIL_HumanReadableSize_t windowSize; 1376 CHECK(ZSTD_CCtx_getParameter(ress.cctx, ZSTD_c_windowLog, &windowLog)); 1377 if (windowLog == 0) { 1378 const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0); 1379 windowLog = cParams.windowLog; 1380 } 1381 windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize))); 1382 DISPLAYLEVEL(4, "Decompression will require %.*f%s of memory\n", windowSize.precision, windowSize.value, windowSize.suffix); 1383 } 1384 (void)srcFileName; 1385 1386 /* Main compression loop */ 1387 do { 1388 size_t stillToFlush; 1389 /* Fill input Buffer */ 1390 size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); 1391 ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; 1392 DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); 1393 *readsize += inSize; 1394 1395 if ((inSize == 0) || (*readsize == fileSize)) 1396 directive = ZSTD_e_end; 1397 1398 stillToFlush = 1; 1399 while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ 1400 || (directive == ZSTD_e_end && stillToFlush != 0) ) { 1401 1402 size_t const oldIPos = inBuff.pos; 1403 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 1404 size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); 1405 CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); 1406 1407 /* count stats */ 1408 inputPresented++; 1409 if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */ 1410 if (!toFlushNow) flushWaiting = 1; 1411 1412 /* Write compressed stream */ 1413 DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", 1414 (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); 1415 if (outBuff.pos) { 1416 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 1417 if (sizeCheck != outBuff.pos) 1418 EXM_THROW(25, "Write error : %s (cannot write compressed block)", 1419 strerror(errno)); 1420 compressedfilesize += outBuff.pos; 1421 } 1422 1423 /* display notification; and adapt compression level */ 1424 if (READY_FOR_UPDATE()) { 1425 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); 1426 double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; 1427 UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed); 1428 UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed); 1429 UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced); 1430 1431 /* display progress notifications */ 1432 if (g_display_prefs.displayLevel >= 3) { 1433 DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%4s => %.2f%% ", 1434 compressionLevel, 1435 buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix, 1436 consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix, 1437 produced_hrs.precision, produced_hrs.value, produced_hrs.suffix, 1438 cShare ); 1439 } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { 1440 /* Require level 2 or forcibly displayed progress counter for summarized updates */ 1441 DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */ 1442 if (fCtx->nbFilesTotal > 1) { 1443 size_t srcFileNameSize = strlen(srcFileName); 1444 /* Ensure that the string we print is roughly the same size each time */ 1445 if (srcFileNameSize > 18) { 1446 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; 1447 DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ", 1448 fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); 1449 } else { 1450 DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ", 1451 fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); 1452 } 1453 } 1454 DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix); 1455 if (fileSize != UTIL_FILESIZE_UNKNOWN) 1456 DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix); 1457 DISPLAYLEVEL(1, " ==> %2.f%%", cShare); 1458 DELAY_NEXT_UPDATE(); 1459 } 1460 1461 /* adaptive mode : statistics measurement and speed correction */ 1462 if (prefs->adaptiveMode) { 1463 1464 /* check output speed */ 1465 if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ 1466 1467 unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; 1468 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; 1469 assert(zfp.produced >= previous_zfp_update.produced); 1470 assert(prefs->nbWorkers >= 1); 1471 1472 /* test if compression is blocked 1473 * either because output is slow and all buffers are full 1474 * or because input is slow and no job can start while waiting for at least one buffer to be filled. 1475 * note : exclude starting part, since currentJobID > 1 */ 1476 if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ 1477 && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ 1478 ) { 1479 DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") 1480 speedChange = slower; 1481 } 1482 1483 previous_zfp_update = zfp; 1484 1485 if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ 1486 && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ 1487 ) { 1488 DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed); 1489 speedChange = slower; 1490 } 1491 flushWaiting = 0; 1492 } 1493 1494 /* course correct only if there is at least one new job completed */ 1495 if (zfp.currentJobID > lastJobID) { 1496 DISPLAYLEVEL(6, "compression level adaptation check \n") 1497 1498 /* check input speed */ 1499 if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */ 1500 if (inputBlocked <= 0) { 1501 DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); 1502 speedChange = slower; 1503 } else if (speedChange == noChange) { 1504 unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; 1505 unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; 1506 unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; 1507 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; 1508 previous_zfp_correction = zfp; 1509 assert(inputPresented > 0); 1510 DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", 1511 inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, 1512 (unsigned)newlyIngested, (unsigned)newlyConsumed, 1513 (unsigned)newlyFlushed, (unsigned)newlyProduced); 1514 if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ 1515 && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ 1516 && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ 1517 ) { 1518 DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", 1519 newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); 1520 speedChange = faster; 1521 } 1522 } 1523 inputBlocked = 0; 1524 inputPresented = 0; 1525 } 1526 1527 if (speedChange == slower) { 1528 DISPLAYLEVEL(6, "slower speed , higher compression \n") 1529 compressionLevel ++; 1530 if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); 1531 if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; 1532 compressionLevel += (compressionLevel == 0); /* skip 0 */ 1533 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1534 } 1535 if (speedChange == faster) { 1536 DISPLAYLEVEL(6, "faster speed , lighter compression \n") 1537 compressionLevel --; 1538 if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; 1539 compressionLevel -= (compressionLevel == 0); /* skip 0 */ 1540 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1541 } 1542 speedChange = noChange; 1543 1544 lastJobID = zfp.currentJobID; 1545 } /* if (zfp.currentJobID > lastJobID) */ 1546 } /* if (g_adaptiveMode) */ 1547 } /* if (READY_FOR_UPDATE()) */ 1548 } /* while ((inBuff.pos != inBuff.size) */ 1549 } while (directive != ZSTD_e_end); 1550 1551 if (ferror(srcFile)) { 1552 EXM_THROW(26, "Read error : I/O error"); 1553 } 1554 if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { 1555 EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", 1556 (unsigned long long)*readsize, (unsigned long long)fileSize); 1557 } 1558 1559 return compressedfilesize; 1560} 1561 1562/*! FIO_compressFilename_internal() : 1563 * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. 1564 * @return : 0 : compression completed correctly, 1565 * 1 : missing or pb opening srcFileName 1566 */ 1567static int 1568FIO_compressFilename_internal(FIO_ctx_t* const fCtx, 1569 FIO_prefs_t* const prefs, 1570 cRess_t ress, 1571 const char* dstFileName, const char* srcFileName, 1572 int compressionLevel) 1573{ 1574 UTIL_time_t const timeStart = UTIL_getTime(); 1575 clock_t const cpuStart = clock(); 1576 U64 readsize = 0; 1577 U64 compressedfilesize = 0; 1578 U64 const fileSize = UTIL_getFileSize(srcFileName); 1579 DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); 1580 1581 /* compression format selection */ 1582 switch (prefs->compressionType) { 1583 default: 1584 case FIO_zstdCompression: 1585 compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); 1586 break; 1587 1588 case FIO_gzipCompression: 1589#ifdef ZSTD_GZCOMPRESS 1590 compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 1591#else 1592 (void)compressionLevel; 1593 EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", 1594 srcFileName); 1595#endif 1596 break; 1597 1598 case FIO_xzCompression: 1599 case FIO_lzmaCompression: 1600#ifdef ZSTD_LZMACOMPRESS 1601 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression); 1602#else 1603 (void)compressionLevel; 1604 EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", 1605 srcFileName); 1606#endif 1607 break; 1608 1609 case FIO_lz4Compression: 1610#ifdef ZSTD_LZ4COMPRESS 1611 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize); 1612#else 1613 (void)compressionLevel; 1614 EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", 1615 srcFileName); 1616#endif 1617 break; 1618 } 1619 1620 /* Status */ 1621 fCtx->totalBytesInput += (size_t)readsize; 1622 fCtx->totalBytesOutput += (size_t)compressedfilesize; 1623 DISPLAYLEVEL(2, "\r%79s\r", ""); 1624 if (g_display_prefs.displayLevel >= 2 && 1625 !fCtx->hasStdoutOutput && 1626 (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { 1627 UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); 1628 UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); 1629 if (readsize == 0) { 1630 DISPLAYLEVEL(2,"%-20s : (%6.*f%4s => %6.*f%4s, %s) \n", 1631 srcFileName, 1632 hr_isize.precision, hr_isize.value, hr_isize.suffix, 1633 hr_osize.precision, hr_osize.value, hr_osize.suffix, 1634 dstFileName); 1635 } else { 1636 DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%4s => %6.*f%4s, %s) \n", 1637 srcFileName, 1638 (double)compressedfilesize / (double)readsize * 100, 1639 hr_isize.precision, hr_isize.value, hr_isize.suffix, 1640 hr_osize.precision, hr_osize.value, hr_osize.suffix, 1641 dstFileName); 1642 } 1643 } 1644 1645 /* Elapsed Time and CPU Load */ 1646 { clock_t const cpuEnd = clock(); 1647 double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; 1648 U64 const timeLength_ns = UTIL_clockSpanNano(timeStart); 1649 double const timeLength_s = (double)timeLength_ns / 1000000000; 1650 double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100; 1651 DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n", 1652 srcFileName, timeLength_s, cpuLoad_pct); 1653 } 1654 return 0; 1655} 1656 1657 1658/*! FIO_compressFilename_dstFile() : 1659 * open dstFileName, or pass-through if ress.dstFile != NULL, 1660 * then start compression with FIO_compressFilename_internal(). 1661 * Manages source removal (--rm) and file permissions transfer. 1662 * note : ress.srcFile must be != NULL, 1663 * so reach this function through FIO_compressFilename_srcFile(). 1664 * @return : 0 : compression completed correctly, 1665 * 1 : pb 1666 */ 1667static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, 1668 FIO_prefs_t* const prefs, 1669 cRess_t ress, 1670 const char* dstFileName, 1671 const char* srcFileName, 1672 int compressionLevel) 1673{ 1674 int closeDstFile = 0; 1675 int result; 1676 stat_t statbuf; 1677 int transferMTime = 0; 1678 assert(ress.srcFile != NULL); 1679 if (ress.dstFile == NULL) { 1680 int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; 1681 if ( strcmp (srcFileName, stdinmark) 1682 && strcmp (dstFileName, stdoutmark) 1683 && UTIL_stat(srcFileName, &statbuf) 1684 && UTIL_isRegularFileStat(&statbuf) ) { 1685 dstFilePermissions = statbuf.st_mode; 1686 transferMTime = 1; 1687 } 1688 1689 closeDstFile = 1; 1690 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); 1691 ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); 1692 if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ 1693 /* Must only be added after FIO_openDstFile() succeeds. 1694 * Otherwise we may delete the destination file if it already exists, 1695 * and the user presses Ctrl-C when asked if they wish to overwrite. 1696 */ 1697 addHandler(dstFileName); 1698 } 1699 1700 result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); 1701 1702 if (closeDstFile) { 1703 FILE* const dstFile = ress.dstFile; 1704 ress.dstFile = NULL; 1705 1706 clearHandler(); 1707 1708 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); 1709 if (fclose(dstFile)) { /* error closing dstFile */ 1710 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1711 result=1; 1712 } 1713 if (transferMTime) { 1714 UTIL_utime(dstFileName, &statbuf); 1715 } 1716 if ( (result != 0) /* operation failure */ 1717 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 1718 ) { 1719 FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ 1720 } 1721 } 1722 1723 return result; 1724} 1725 1726/* List used to compare file extensions (used with --exclude-compressed flag) 1727* Different from the suffixList and should only apply to ZSTD compress operationResult 1728*/ 1729static const char *compressedFileExtensions[] = { 1730 ZSTD_EXTENSION, 1731 TZSTD_EXTENSION, 1732 GZ_EXTENSION, 1733 TGZ_EXTENSION, 1734 LZMA_EXTENSION, 1735 XZ_EXTENSION, 1736 TXZ_EXTENSION, 1737 LZ4_EXTENSION, 1738 TLZ4_EXTENSION, 1739 NULL 1740}; 1741 1742/*! FIO_compressFilename_srcFile() : 1743 * @return : 0 : compression completed correctly, 1744 * 1 : missing or pb opening srcFileName 1745 */ 1746static int 1747FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, 1748 FIO_prefs_t* const prefs, 1749 cRess_t ress, 1750 const char* dstFileName, 1751 const char* srcFileName, 1752 int compressionLevel) 1753{ 1754 int result; 1755 DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName); 1756 1757 /* ensure src is not a directory */ 1758 if (UTIL_isDirectory(srcFileName)) { 1759 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1760 return 1; 1761 } 1762 1763 /* ensure src is not the same as dict (if present) */ 1764 if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { 1765 DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); 1766 return 1; 1767 } 1768 1769 /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used 1770 * YES => ZSTD will skip compression of the file and will return 0. 1771 * NO => ZSTD will resume with compress operation. 1772 */ 1773 if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) { 1774 DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); 1775 return 0; 1776 } 1777 1778 ress.srcFile = FIO_openSrcFile(prefs, srcFileName); 1779 if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ 1780 1781 result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); 1782 1783 fclose(ress.srcFile); 1784 ress.srcFile = NULL; 1785 if ( prefs->removeSrcFile /* --rm */ 1786 && result == 0 /* success */ 1787 && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ 1788 ) { 1789 /* We must clear the handler, since after this point calling it would 1790 * delete both the source and destination files. 1791 */ 1792 clearHandler(); 1793 if (FIO_removeFile(srcFileName)) 1794 EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); 1795 } 1796 return result; 1797} 1798 1799static const char* checked_index(const char* options[], size_t length, size_t index) { 1800 assert(index < length); 1801 // Necessary to avoid warnings since -O3 will omit the above `assert` 1802 (void) length; 1803 return options[index]; 1804} 1805 1806#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index)) 1807 1808void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) { 1809 static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION, 1810 LZMA_EXTENSION, LZ4_EXTENSION}; 1811 static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"}; 1812 static const char* checkSumOptions[3] = {" --no-check", "", " --check"}; 1813 static const char* rowMatchFinderOptions[3] = {"", " --no-row-match-finder", " --row-match-finder"}; 1814 static const char* compressLiteralsOptions[3] = {"", " --compress-literals", " --no-compress-literals"}; 1815 1816 assert(g_display_prefs.displayLevel >= 4); 1817 1818 DISPLAY("--format=%s", formatOptions[prefs->compressionType]); 1819 DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport)); 1820 DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID"); 1821 DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag)); 1822 DISPLAY(" --block-size=%d", prefs->blockSize); 1823 if (prefs->adaptiveMode) 1824 DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel); 1825 DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); 1826 DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : ""); 1827 if (prefs->streamSrcSize) 1828 DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize); 1829 if (prefs->srcSizeHint) 1830 DISPLAY(" --size-hint=%d", prefs->srcSizeHint); 1831 if (prefs->targetCBlockSize) 1832 DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize); 1833 DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode)); 1834 DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB); 1835 DISPLAY(" --threads=%d", prefs->nbWorkers); 1836 DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : ""); 1837 DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-"); 1838 DISPLAY("\n"); 1839} 1840 1841#undef INDEX 1842 1843int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, 1844 const char* srcFileName, const char* dictFileName, 1845 int compressionLevel, ZSTD_compressionParameters comprParams) 1846{ 1847 cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); 1848 int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); 1849 1850#define DISPLAY_LEVEL_DEFAULT 2 1851 1852 FIO_freeCResources(&ress); 1853 return result; 1854} 1855 1856/* FIO_determineCompressedName() : 1857 * create a destination filename for compressed srcFileName. 1858 * @return a pointer to it. 1859 * This function never returns an error (it may abort() in case of pb) 1860 */ 1861static const char* 1862FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) 1863{ 1864 static size_t dfnbCapacity = 0; 1865 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 1866 char* outDirFilename = NULL; 1867 size_t sfnSize = strlen(srcFileName); 1868 size_t const srcSuffixLen = strlen(suffix); 1869 if (outDirName) { 1870 outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen); 1871 sfnSize = strlen(outDirFilename); 1872 assert(outDirFilename != NULL); 1873 } 1874 1875 if (dfnbCapacity <= sfnSize+srcSuffixLen+1) { 1876 /* resize buffer for dstName */ 1877 free(dstFileNameBuffer); 1878 dfnbCapacity = sfnSize + srcSuffixLen + 30; 1879 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 1880 if (!dstFileNameBuffer) { 1881 EXM_THROW(30, "zstd: %s", strerror(errno)); 1882 } 1883 } 1884 assert(dstFileNameBuffer != NULL); 1885 1886 if (outDirFilename) { 1887 memcpy(dstFileNameBuffer, outDirFilename, sfnSize); 1888 free(outDirFilename); 1889 } else { 1890 memcpy(dstFileNameBuffer, srcFileName, sfnSize); 1891 } 1892 memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */); 1893 return dstFileNameBuffer; 1894} 1895 1896static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles) 1897{ 1898 size_t i; 1899 unsigned long long fileSize, maxFileSize = 0; 1900 for (i = 0; i < nbFiles; i++) { 1901 fileSize = UTIL_getFileSize(inFileNames[i]); 1902 maxFileSize = fileSize > maxFileSize ? fileSize : maxFileSize; 1903 } 1904 return maxFileSize; 1905} 1906 1907/* FIO_compressMultipleFilenames() : 1908 * compress nbFiles files 1909 * into either one destination (outFileName), 1910 * or into one file each (outFileName == NULL, but suffix != NULL), 1911 * or into a destination folder (specified with -O) 1912 */ 1913int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, 1914 FIO_prefs_t* const prefs, 1915 const char** inFileNamesTable, 1916 const char* outMirroredRootDirName, 1917 const char* outDirName, 1918 const char* outFileName, const char* suffix, 1919 const char* dictFileName, int compressionLevel, 1920 ZSTD_compressionParameters comprParams) 1921{ 1922 int status; 1923 int error = 0; 1924 cRess_t ress = FIO_createCResources(prefs, dictFileName, 1925 FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), 1926 compressionLevel, comprParams); 1927 1928 /* init */ 1929 assert(outFileName != NULL || suffix != NULL); 1930 if (outFileName != NULL) { /* output into a single destination (stdout typically) */ 1931 if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { 1932 FIO_freeCResources(&ress); 1933 return 1; 1934 } 1935 ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); 1936 if (ress.dstFile == NULL) { /* could not open outFileName */ 1937 error = 1; 1938 } else { 1939 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { 1940 status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); 1941 if (!status) fCtx->nbFilesProcessed++; 1942 error |= status; 1943 } 1944 if (fclose(ress.dstFile)) 1945 EXM_THROW(29, "Write error (%s) : cannot properly close %s", 1946 strerror(errno), outFileName); 1947 ress.dstFile = NULL; 1948 } 1949 } else { 1950 if (outMirroredRootDirName) 1951 UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); 1952 1953 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { 1954 const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; 1955 const char* dstFileName = NULL; 1956 if (outMirroredRootDirName) { 1957 char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); 1958 if (validMirroredDirName) { 1959 dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix); 1960 free(validMirroredDirName); 1961 } else { 1962 DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName); 1963 error=1; 1964 continue; 1965 } 1966 } else { 1967 dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ 1968 } 1969 status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); 1970 if (!status) fCtx->nbFilesProcessed++; 1971 error |= status; 1972 } 1973 1974 if (outDirName) 1975 FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); 1976 } 1977 1978 if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { 1979 UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); 1980 UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); 1981 1982 DISPLAYLEVEL(2, "\r%79s\r", ""); 1983 DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", 1984 fCtx->nbFilesProcessed, 1985 (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, 1986 hr_isize.precision, hr_isize.value, hr_isize.suffix, 1987 hr_osize.precision, hr_osize.value, hr_osize.suffix); 1988 } 1989 1990 FIO_freeCResources(&ress); 1991 return error; 1992} 1993 1994#endif /* #ifndef ZSTD_NOCOMPRESS */ 1995 1996 1997 1998#ifndef ZSTD_NODECOMPRESS 1999 2000/* ************************************************************************** 2001 * Decompression 2002 ***************************************************************************/ 2003typedef struct { 2004 void* srcBuffer; 2005 size_t srcBufferSize; 2006 size_t srcBufferLoaded; 2007 void* dstBuffer; 2008 size_t dstBufferSize; 2009 ZSTD_DStream* dctx; 2010 FILE* dstFile; 2011} dRess_t; 2012 2013static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) 2014{ 2015 dRess_t ress; 2016 memset(&ress, 0, sizeof(ress)); 2017 2018 if (prefs->patchFromMode) 2019 FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */); 2020 2021 /* Allocation */ 2022 ress.dctx = ZSTD_createDStream(); 2023 if (ress.dctx==NULL) 2024 EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); 2025 CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); 2026 CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); 2027 2028 ress.srcBufferSize = ZSTD_DStreamInSize(); 2029 ress.srcBuffer = malloc(ress.srcBufferSize); 2030 ress.dstBufferSize = ZSTD_DStreamOutSize(); 2031 ress.dstBuffer = malloc(ress.dstBufferSize); 2032 if (!ress.srcBuffer || !ress.dstBuffer) 2033 EXM_THROW(61, "Allocation error : not enough memory"); 2034 2035 /* dictionary */ 2036 { void* dictBuffer; 2037 size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs); 2038 CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); 2039 free(dictBuffer); 2040 } 2041 2042 return ress; 2043} 2044 2045static void FIO_freeDResources(dRess_t ress) 2046{ 2047 CHECK( ZSTD_freeDStream(ress.dctx) ); 2048 free(ress.srcBuffer); 2049 free(ress.dstBuffer); 2050} 2051 2052 2053/** FIO_fwriteSparse() : 2054* @return : storedSkips, 2055* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ 2056static unsigned 2057FIO_fwriteSparse(FILE* file, 2058 const void* buffer, size_t bufferSize, 2059 const FIO_prefs_t* const prefs, 2060 unsigned storedSkips) 2061{ 2062 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 2063 size_t bufferSizeT = bufferSize / sizeof(size_t); 2064 const size_t* const bufferTEnd = bufferT + bufferSizeT; 2065 const size_t* ptrT = bufferT; 2066 static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ 2067 2068 if (prefs->testMode) return 0; /* do not output anything in test mode */ 2069 2070 if (!prefs->sparseFileSupport) { /* normal write */ 2071 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 2072 if (sizeCheck != bufferSize) 2073 EXM_THROW(70, "Write error : cannot write decoded block : %s", 2074 strerror(errno)); 2075 return 0; 2076 } 2077 2078 /* avoid int overflow */ 2079 if (storedSkips > 1 GB) { 2080 if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) 2081 EXM_THROW(91, "1 GB skip error (sparse file support)"); 2082 storedSkips -= 1 GB; 2083 } 2084 2085 while (ptrT < bufferTEnd) { 2086 size_t nb0T; 2087 2088 /* adjust last segment if < 32 KB */ 2089 size_t seg0SizeT = segmentSizeT; 2090 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 2091 bufferSizeT -= seg0SizeT; 2092 2093 /* count leading zeroes */ 2094 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 2095 storedSkips += (unsigned)(nb0T * sizeof(size_t)); 2096 2097 if (nb0T != seg0SizeT) { /* not all 0s */ 2098 size_t const nbNon0ST = seg0SizeT - nb0T; 2099 /* skip leading zeros */ 2100 if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) 2101 EXM_THROW(92, "Sparse skip error ; try --no-sparse"); 2102 storedSkips = 0; 2103 /* write the rest */ 2104 if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) 2105 EXM_THROW(93, "Write error : cannot write decoded block : %s", 2106 strerror(errno)); 2107 } 2108 ptrT += seg0SizeT; 2109 } 2110 2111 { static size_t const maskT = sizeof(size_t)-1; 2112 if (bufferSize & maskT) { 2113 /* size not multiple of sizeof(size_t) : implies end of block */ 2114 const char* const restStart = (const char*)bufferTEnd; 2115 const char* restPtr = restStart; 2116 const char* const restEnd = (const char*)buffer + bufferSize; 2117 assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); 2118 for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 2119 storedSkips += (unsigned) (restPtr - restStart); 2120 if (restPtr != restEnd) { 2121 /* not all remaining bytes are 0 */ 2122 size_t const restSize = (size_t)(restEnd - restPtr); 2123 if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) 2124 EXM_THROW(92, "Sparse skip error ; try --no-sparse"); 2125 if (fwrite(restPtr, 1, restSize, file) != restSize) 2126 EXM_THROW(95, "Write error : cannot write end of decoded block : %s", 2127 strerror(errno)); 2128 storedSkips = 0; 2129 } } } 2130 2131 return storedSkips; 2132} 2133 2134static void 2135FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) 2136{ 2137 if (prefs->testMode) assert(storedSkips == 0); 2138 if (storedSkips>0) { 2139 assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ 2140 (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ 2141 if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) 2142 EXM_THROW(69, "Final skip error (sparse file support)"); 2143 /* last zero must be explicitly written, 2144 * so that skipped ones get implicitly translated as zero by FS */ 2145 { const char lastZeroByte[1] = { 0 }; 2146 if (fwrite(lastZeroByte, 1, 1, file) != 1) 2147 EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); 2148 } } 2149} 2150 2151 2152/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode 2153 @return : 0 (no error) */ 2154static int FIO_passThrough(const FIO_prefs_t* const prefs, 2155 FILE* foutput, FILE* finput, 2156 void* buffer, size_t bufferSize, 2157 size_t alreadyLoaded) 2158{ 2159 size_t const blockSize = MIN(64 KB, bufferSize); 2160 size_t readFromInput; 2161 unsigned storedSkips = 0; 2162 2163 /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ 2164 { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); 2165 if (sizeCheck != alreadyLoaded) { 2166 DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); 2167 return 1; 2168 } } 2169 2170 do { 2171 readFromInput = fread(buffer, 1, blockSize, finput); 2172 storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips); 2173 } while (readFromInput == blockSize); 2174 if (ferror(finput)) { 2175 DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno)); 2176 return 1; 2177 } 2178 assert(feof(finput)); 2179 2180 FIO_fwriteSparseEnd(prefs, foutput, storedSkips); 2181 return 0; 2182} 2183 2184/* FIO_zstdErrorHelp() : 2185 * detailed error message when requested window size is too large */ 2186static void 2187FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, 2188 const dRess_t* ress, 2189 size_t err, const char* srcFileName) 2190{ 2191 ZSTD_frameHeader header; 2192 2193 /* Help message only for one specific error */ 2194 if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) 2195 return; 2196 2197 /* Try to decode the frame header */ 2198 err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); 2199 if (err == 0) { 2200 unsigned long long const windowSize = header.windowSize; 2201 unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); 2202 assert(prefs->memLimit > 0); 2203 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n", 2204 srcFileName, windowSize, prefs->memLimit); 2205 if (windowLog <= ZSTD_WINDOWLOG_MAX) { 2206 unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); 2207 assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ 2208 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n", 2209 srcFileName, windowLog, windowMB); 2210 return; 2211 } } 2212 DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n", 2213 srcFileName, ZSTD_WINDOWLOG_MAX); 2214} 2215 2216/** FIO_decompressFrame() : 2217 * @return : size of decoded zstd frame, or an error code 2218 */ 2219#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) 2220static unsigned long long 2221FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, 2222 const FIO_prefs_t* const prefs, 2223 const char* srcFileName, 2224 U64 alreadyDecoded) /* for multi-frames streams */ 2225{ 2226 U64 frameSize = 0; 2227 U32 storedSkips = 0; 2228 2229 /* display last 20 characters only */ 2230 { size_t const srcFileLength = strlen(srcFileName); 2231 if (srcFileLength>20) srcFileName += srcFileLength-20; 2232 } 2233 2234 ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only); 2235 2236 /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ 2237 { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; 2238 if (ress->srcBufferLoaded < toDecode) { 2239 size_t const toRead = toDecode - ress->srcBufferLoaded; 2240 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 2241 ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); 2242 } } 2243 2244 /* Main decompression Loop */ 2245 while (1) { 2246 ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; 2247 ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; 2248 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); 2249 const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; 2250 UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize); 2251 if (ZSTD_isError(readSizeHint)) { 2252 DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", 2253 srcFileName, ZSTD_getErrorName(readSizeHint)); 2254 FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); 2255 return FIO_ERROR_FRAME_DECODING; 2256 } 2257 2258 /* Write block */ 2259 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); 2260 frameSize += outBuff.pos; 2261 if (fCtx->nbFilesTotal > 1) { 2262 size_t srcFileNameSize = strlen(srcFileName); 2263 if (srcFileNameSize > 18) { 2264 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; 2265 DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ", 2266 fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix); 2267 } else { 2268 DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ", 2269 fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix); 2270 } 2271 } else { 2272 DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ", 2273 srcFileName, hrs.precision, hrs.value, hrs.suffix); 2274 } 2275 2276 if (inBuff.pos > 0) { 2277 memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); 2278 ress->srcBufferLoaded -= inBuff.pos; 2279 } 2280 2281 if (readSizeHint == 0) break; /* end of frame */ 2282 2283 /* Fill input buffer */ 2284 { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ 2285 if (ress->srcBufferLoaded < toDecode) { 2286 size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ 2287 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 2288 size_t const readSize = fread(startPosition, 1, toRead, finput); 2289 if (readSize==0) { 2290 DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", 2291 srcFileName); 2292 return FIO_ERROR_FRAME_DECODING; 2293 } 2294 ress->srcBufferLoaded += readSize; 2295 } } } 2296 2297 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2298 2299 return frameSize; 2300} 2301 2302 2303#ifdef ZSTD_GZDECOMPRESS 2304static unsigned long long 2305FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, 2306 const FIO_prefs_t* const prefs, 2307 const char* srcFileName) 2308{ 2309 unsigned long long outFileSize = 0; 2310 z_stream strm; 2311 int flush = Z_NO_FLUSH; 2312 int decodingError = 0; 2313 unsigned storedSkips = 0; 2314 2315 strm.zalloc = Z_NULL; 2316 strm.zfree = Z_NULL; 2317 strm.opaque = Z_NULL; 2318 strm.next_in = 0; 2319 strm.avail_in = 0; 2320 /* see http://www.zlib.net/manual.html */ 2321 if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) 2322 return FIO_ERROR_FRAME_DECODING; 2323 2324 strm.next_out = (Bytef*)ress->dstBuffer; 2325 strm.avail_out = (uInt)ress->dstBufferSize; 2326 strm.avail_in = (uInt)ress->srcBufferLoaded; 2327 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 2328 2329 for ( ; ; ) { 2330 int ret; 2331 if (strm.avail_in == 0) { 2332 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 2333 if (ress->srcBufferLoaded == 0) flush = Z_FINISH; 2334 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 2335 strm.avail_in = (uInt)ress->srcBufferLoaded; 2336 } 2337 ret = inflate(&strm, flush); 2338 if (ret == Z_BUF_ERROR) { 2339 DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); 2340 decodingError = 1; break; 2341 } 2342 if (ret != Z_OK && ret != Z_STREAM_END) { 2343 DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); 2344 decodingError = 1; break; 2345 } 2346 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 2347 if (decompBytes) { 2348 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); 2349 outFileSize += decompBytes; 2350 strm.next_out = (Bytef*)ress->dstBuffer; 2351 strm.avail_out = (uInt)ress->dstBufferSize; 2352 } 2353 } 2354 if (ret == Z_STREAM_END) break; 2355 } 2356 2357 if (strm.avail_in > 0) 2358 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 2359 ress->srcBufferLoaded = strm.avail_in; 2360 if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ 2361 && (decodingError==0) ) { 2362 DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); 2363 decodingError = 1; 2364 } 2365 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2366 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 2367} 2368#endif 2369 2370 2371#ifdef ZSTD_LZMADECOMPRESS 2372static unsigned long long 2373FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, 2374 const FIO_prefs_t* const prefs, 2375 const char* srcFileName, int plain_lzma) 2376{ 2377 unsigned long long outFileSize = 0; 2378 lzma_stream strm = LZMA_STREAM_INIT; 2379 lzma_action action = LZMA_RUN; 2380 lzma_ret initRet; 2381 int decodingError = 0; 2382 unsigned storedSkips = 0; 2383 2384 strm.next_in = 0; 2385 strm.avail_in = 0; 2386 if (plain_lzma) { 2387 initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ 2388 } else { 2389 initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ 2390 } 2391 2392 if (initRet != LZMA_OK) { 2393 DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", 2394 plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", 2395 srcFileName, initRet); 2396 return FIO_ERROR_FRAME_DECODING; 2397 } 2398 2399 strm.next_out = (BYTE*)ress->dstBuffer; 2400 strm.avail_out = ress->dstBufferSize; 2401 strm.next_in = (BYTE const*)ress->srcBuffer; 2402 strm.avail_in = ress->srcBufferLoaded; 2403 2404 for ( ; ; ) { 2405 lzma_ret ret; 2406 if (strm.avail_in == 0) { 2407 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 2408 if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; 2409 strm.next_in = (BYTE const*)ress->srcBuffer; 2410 strm.avail_in = ress->srcBufferLoaded; 2411 } 2412 ret = lzma_code(&strm, action); 2413 2414 if (ret == LZMA_BUF_ERROR) { 2415 DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); 2416 decodingError = 1; break; 2417 } 2418 if (ret != LZMA_OK && ret != LZMA_STREAM_END) { 2419 DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", 2420 srcFileName, ret); 2421 decodingError = 1; break; 2422 } 2423 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 2424 if (decompBytes) { 2425 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); 2426 outFileSize += decompBytes; 2427 strm.next_out = (BYTE*)ress->dstBuffer; 2428 strm.avail_out = ress->dstBufferSize; 2429 } } 2430 if (ret == LZMA_STREAM_END) break; 2431 } 2432 2433 if (strm.avail_in > 0) 2434 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 2435 ress->srcBufferLoaded = strm.avail_in; 2436 lzma_end(&strm); 2437 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2438 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 2439} 2440#endif 2441 2442#ifdef ZSTD_LZ4DECOMPRESS 2443static unsigned long long 2444FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, 2445 const FIO_prefs_t* const prefs, 2446 const char* srcFileName) 2447{ 2448 unsigned long long filesize = 0; 2449 LZ4F_errorCode_t nextToLoad; 2450 LZ4F_decompressionContext_t dCtx; 2451 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 2452 int decodingError = 0; 2453 unsigned storedSkips = 0; 2454 2455 if (LZ4F_isError(errorCode)) { 2456 DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); 2457 return FIO_ERROR_FRAME_DECODING; 2458 } 2459 2460 /* Init feed with magic number (already consumed from FILE* sFile) */ 2461 { size_t inSize = 4; 2462 size_t outSize= 0; 2463 MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); 2464 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); 2465 if (LZ4F_isError(nextToLoad)) { 2466 DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", 2467 srcFileName, LZ4F_getErrorName(nextToLoad)); 2468 LZ4F_freeDecompressionContext(dCtx); 2469 return FIO_ERROR_FRAME_DECODING; 2470 } } 2471 2472 /* Main Loop */ 2473 for (;nextToLoad;) { 2474 size_t readSize; 2475 size_t pos = 0; 2476 size_t decodedBytes = ress->dstBufferSize; 2477 2478 /* Read input */ 2479 if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; 2480 readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); 2481 if (!readSize) break; /* reached end of file or stream */ 2482 2483 while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ 2484 /* Decode Input (at least partially) */ 2485 size_t remaining = readSize - pos; 2486 decodedBytes = ress->dstBufferSize; 2487 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); 2488 if (LZ4F_isError(nextToLoad)) { 2489 DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", 2490 srcFileName, LZ4F_getErrorName(nextToLoad)); 2491 decodingError = 1; nextToLoad = 0; break; 2492 } 2493 pos += remaining; 2494 2495 /* Write Block */ 2496 if (decodedBytes) { 2497 UTIL_HumanReadableSize_t hrs; 2498 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); 2499 filesize += decodedBytes; 2500 hrs = UTIL_makeHumanReadableSize(filesize); 2501 DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); 2502 } 2503 2504 if (!nextToLoad) break; 2505 } 2506 } 2507 /* can be out because readSize == 0, which could be an fread() error */ 2508 if (ferror(srcFile)) { 2509 DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); 2510 decodingError=1; 2511 } 2512 2513 if (nextToLoad!=0) { 2514 DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); 2515 decodingError=1; 2516 } 2517 2518 LZ4F_freeDecompressionContext(dCtx); 2519 ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ 2520 FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); 2521 2522 return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; 2523} 2524#endif 2525 2526 2527 2528/** FIO_decompressFrames() : 2529 * Find and decode frames inside srcFile 2530 * srcFile presumed opened and valid 2531 * @return : 0 : OK 2532 * 1 : error 2533 */ 2534static int FIO_decompressFrames(FIO_ctx_t* const fCtx, 2535 dRess_t ress, FILE* srcFile, 2536 const FIO_prefs_t* const prefs, 2537 const char* dstFileName, const char* srcFileName) 2538{ 2539 unsigned readSomething = 0; 2540 unsigned long long filesize = 0; 2541 assert(srcFile != NULL); 2542 2543 /* for each frame */ 2544 for ( ; ; ) { 2545 /* check magic number -> version */ 2546 size_t const toRead = 4; 2547 const BYTE* const buf = (const BYTE*)ress.srcBuffer; 2548 if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ 2549 ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, 2550 (size_t)1, toRead - ress.srcBufferLoaded, srcFile); 2551 if (ress.srcBufferLoaded==0) { 2552 if (readSomething==0) { /* srcFile is empty (which is invalid) */ 2553 DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); 2554 return 1; 2555 } /* else, just reached frame boundary */ 2556 break; /* no more input */ 2557 } 2558 readSomething = 1; /* there is at least 1 byte in srcFile */ 2559 if (ress.srcBufferLoaded < toRead) { 2560 DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); 2561 return 1; 2562 } 2563 if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { 2564 unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); 2565 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2566 filesize += frameSize; 2567 } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ 2568#ifdef ZSTD_GZDECOMPRESS 2569 unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName); 2570 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2571 filesize += frameSize; 2572#else 2573 DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); 2574 return 1; 2575#endif 2576 } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ 2577 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ 2578#ifdef ZSTD_LZMADECOMPRESS 2579 unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD); 2580 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2581 filesize += frameSize; 2582#else 2583 DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); 2584 return 1; 2585#endif 2586 } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { 2587#ifdef ZSTD_LZ4DECOMPRESS 2588 unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName); 2589 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 2590 filesize += frameSize; 2591#else 2592 DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); 2593 return 1; 2594#endif 2595 } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ 2596 return FIO_passThrough(prefs, 2597 ress.dstFile, srcFile, 2598 ress.srcBuffer, ress.srcBufferSize, 2599 ress.srcBufferLoaded); 2600 } else { 2601 DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); 2602 return 1; 2603 } } /* for each frame */ 2604 2605 /* Final Status */ 2606 fCtx->totalBytesOutput += (size_t)filesize; 2607 DISPLAYLEVEL(2, "\r%79s\r", ""); 2608 /* No status message in pipe mode (stdin - stdout) or multi-files mode */ 2609 if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || 2610 g_display_prefs.displayLevel >= 3 || 2611 g_display_prefs.progressSetting == FIO_ps_always) { 2612 DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize); 2613 } 2614 2615 return 0; 2616} 2617 2618/** FIO_decompressDstFile() : 2619 open `dstFileName`, 2620 or path-through if ress.dstFile is already != 0, 2621 then start decompression process (FIO_decompressFrames()). 2622 @return : 0 : OK 2623 1 : operation aborted 2624*/ 2625static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, 2626 FIO_prefs_t* const prefs, 2627 dRess_t ress, FILE* srcFile, 2628 const char* dstFileName, const char* srcFileName) 2629{ 2630 int result; 2631 stat_t statbuf; 2632 int releaseDstFile = 0; 2633 int transferMTime = 0; 2634 2635 if ((ress.dstFile == NULL) && (prefs->testMode==0)) { 2636 int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; 2637 if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ 2638 && strcmp(dstFileName, stdoutmark) 2639 && UTIL_stat(srcFileName, &statbuf) 2640 && UTIL_isRegularFileStat(&statbuf) ) { 2641 dstFilePermissions = statbuf.st_mode; 2642 transferMTime = 1; 2643 } 2644 2645 releaseDstFile = 1; 2646 2647 ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); 2648 if (ress.dstFile==NULL) return 1; 2649 2650 /* Must only be added after FIO_openDstFile() succeeds. 2651 * Otherwise we may delete the destination file if it already exists, 2652 * and the user presses Ctrl-C when asked if they wish to overwrite. 2653 */ 2654 addHandler(dstFileName); 2655 } 2656 2657 result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); 2658 2659 if (releaseDstFile) { 2660 FILE* const dstFile = ress.dstFile; 2661 clearHandler(); 2662 ress.dstFile = NULL; 2663 if (fclose(dstFile)) { 2664 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 2665 result = 1; 2666 } 2667 2668 if (transferMTime) { 2669 UTIL_utime(dstFileName, &statbuf); 2670 } 2671 2672 if ( (result != 0) /* operation failure */ 2673 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 2674 ) { 2675 FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ 2676 } 2677 } 2678 2679 return result; 2680} 2681 2682 2683/** FIO_decompressSrcFile() : 2684 Open `srcFileName`, transfer control to decompressDstFile() 2685 @return : 0 : OK 2686 1 : error 2687*/ 2688static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) 2689{ 2690 FILE* srcFile; 2691 int result; 2692 2693 if (UTIL_isDirectory(srcFileName)) { 2694 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 2695 return 1; 2696 } 2697 2698 srcFile = FIO_openSrcFile(prefs, srcFileName); 2699 if (srcFile==NULL) return 1; 2700 ress.srcBufferLoaded = 0; 2701 2702 result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); 2703 2704 /* Close file */ 2705 if (fclose(srcFile)) { 2706 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ 2707 return 1; 2708 } 2709 if ( prefs->removeSrcFile /* --rm */ 2710 && (result==0) /* decompression successful */ 2711 && strcmp(srcFileName, stdinmark) ) /* not stdin */ { 2712 /* We must clear the handler, since after this point calling it would 2713 * delete both the source and destination files. 2714 */ 2715 clearHandler(); 2716 if (FIO_removeFile(srcFileName)) { 2717 /* failed to remove src file */ 2718 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 2719 return 1; 2720 } } 2721 return result; 2722} 2723 2724 2725 2726int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, 2727 const char* dstFileName, const char* srcFileName, 2728 const char* dictFileName) 2729{ 2730 dRess_t const ress = FIO_createDResources(prefs, dictFileName); 2731 2732 int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); 2733 2734 FIO_freeDResources(ress); 2735 return decodingError; 2736} 2737 2738static const char *suffixList[] = { 2739 ZSTD_EXTENSION, 2740 TZSTD_EXTENSION, 2741#ifndef ZSTD_NODECOMPRESS 2742 ZSTD_ALT_EXTENSION, 2743#endif 2744#ifdef ZSTD_GZDECOMPRESS 2745 GZ_EXTENSION, 2746 TGZ_EXTENSION, 2747#endif 2748#ifdef ZSTD_LZMADECOMPRESS 2749 LZMA_EXTENSION, 2750 XZ_EXTENSION, 2751 TXZ_EXTENSION, 2752#endif 2753#ifdef ZSTD_LZ4DECOMPRESS 2754 LZ4_EXTENSION, 2755 TLZ4_EXTENSION, 2756#endif 2757 NULL 2758}; 2759 2760static const char *suffixListStr = 2761 ZSTD_EXTENSION "/" TZSTD_EXTENSION 2762#ifdef ZSTD_GZDECOMPRESS 2763 "/" GZ_EXTENSION "/" TGZ_EXTENSION 2764#endif 2765#ifdef ZSTD_LZMADECOMPRESS 2766 "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION 2767#endif 2768#ifdef ZSTD_LZ4DECOMPRESS 2769 "/" LZ4_EXTENSION "/" TLZ4_EXTENSION 2770#endif 2771; 2772 2773/* FIO_determineDstName() : 2774 * create a destination filename from a srcFileName. 2775 * @return a pointer to it. 2776 * @return == NULL if there is an error */ 2777static const char* 2778FIO_determineDstName(const char* srcFileName, const char* outDirName) 2779{ 2780 static size_t dfnbCapacity = 0; 2781 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 2782 size_t dstFileNameEndPos; 2783 char* outDirFilename = NULL; 2784 const char* dstSuffix = ""; 2785 size_t dstSuffixLen = 0; 2786 2787 size_t sfnSize = strlen(srcFileName); 2788 2789 size_t srcSuffixLen; 2790 const char* const srcSuffix = strrchr(srcFileName, '.'); 2791 if (srcSuffix == NULL) { 2792 DISPLAYLEVEL(1, 2793 "zstd: %s: unknown suffix (%s expected). " 2794 "Can't derive the output file name. " 2795 "Specify it with -o dstFileName. Ignoring.\n", 2796 srcFileName, suffixListStr); 2797 return NULL; 2798 } 2799 srcSuffixLen = strlen(srcSuffix); 2800 2801 { 2802 const char** matchedSuffixPtr; 2803 for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) { 2804 if (!strcmp(*matchedSuffixPtr, srcSuffix)) { 2805 break; 2806 } 2807 } 2808 2809 /* check suffix is authorized */ 2810 if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) { 2811 DISPLAYLEVEL(1, 2812 "zstd: %s: unknown suffix (%s expected). " 2813 "Can't derive the output file name. " 2814 "Specify it with -o dstFileName. Ignoring.\n", 2815 srcFileName, suffixListStr); 2816 return NULL; 2817 } 2818 2819 if ((*matchedSuffixPtr)[1] == 't') { 2820 dstSuffix = ".tar"; 2821 dstSuffixLen = strlen(dstSuffix); 2822 } 2823 } 2824 2825 if (outDirName) { 2826 outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); 2827 sfnSize = strlen(outDirFilename); 2828 assert(outDirFilename != NULL); 2829 } 2830 2831 if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) { 2832 /* allocate enough space to write dstFilename into it */ 2833 free(dstFileNameBuffer); 2834 dfnbCapacity = sfnSize + 20; 2835 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 2836 if (dstFileNameBuffer==NULL) 2837 EXM_THROW(74, "%s : not enough memory for dstFileName", 2838 strerror(errno)); 2839 } 2840 2841 /* return dst name == src name truncated from suffix */ 2842 assert(dstFileNameBuffer != NULL); 2843 dstFileNameEndPos = sfnSize - srcSuffixLen; 2844 if (outDirFilename) { 2845 memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos); 2846 free(outDirFilename); 2847 } else { 2848 memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); 2849 } 2850 2851 /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" 2852 * extension on decompression. Also writes terminating null. */ 2853 strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); 2854 return dstFileNameBuffer; 2855 2856 /* note : dstFileNameBuffer memory is not going to be free */ 2857} 2858 2859int 2860FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, 2861 FIO_prefs_t* const prefs, 2862 const char** srcNamesTable, 2863 const char* outMirroredRootDirName, 2864 const char* outDirName, const char* outFileName, 2865 const char* dictFileName) 2866{ 2867 int status; 2868 int error = 0; 2869 dRess_t ress = FIO_createDResources(prefs, dictFileName); 2870 2871 if (outFileName) { 2872 if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { 2873 FIO_freeDResources(ress); 2874 return 1; 2875 } 2876 if (!prefs->testMode) { 2877 ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); 2878 if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); 2879 } 2880 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { 2881 status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); 2882 if (!status) fCtx->nbFilesProcessed++; 2883 error |= status; 2884 } 2885 if ((!prefs->testMode) && (fclose(ress.dstFile))) 2886 EXM_THROW(72, "Write error : %s : cannot properly close output file", 2887 strerror(errno)); 2888 } else { 2889 if (outMirroredRootDirName) 2890 UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); 2891 2892 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ 2893 const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; 2894 const char* dstFileName = NULL; 2895 if (outMirroredRootDirName) { 2896 char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); 2897 if (validMirroredDirName) { 2898 dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName); 2899 free(validMirroredDirName); 2900 } else { 2901 DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName); 2902 } 2903 } else { 2904 dstFileName = FIO_determineDstName(srcFileName, outDirName); 2905 } 2906 if (dstFileName == NULL) { error=1; continue; } 2907 status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); 2908 if (!status) fCtx->nbFilesProcessed++; 2909 error |= status; 2910 } 2911 if (outDirName) 2912 FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal); 2913 } 2914 2915 if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0) 2916 DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); 2917 2918 FIO_freeDResources(ress); 2919 return error; 2920} 2921 2922/* ************************************************************************** 2923 * .zst file info (--list command) 2924 ***************************************************************************/ 2925 2926typedef struct { 2927 U64 decompressedSize; 2928 U64 compressedSize; 2929 U64 windowSize; 2930 int numActualFrames; 2931 int numSkippableFrames; 2932 int decompUnavailable; 2933 int usesCheck; 2934 U32 nbFiles; 2935} fileInfo_t; 2936 2937typedef enum { 2938 info_success=0, 2939 info_frame_error=1, 2940 info_not_zstd=2, 2941 info_file_error=3, 2942 info_truncated_input=4, 2943} InfoError; 2944 2945#define ERROR_IF(c,n,...) { \ 2946 if (c) { \ 2947 DISPLAYLEVEL(1, __VA_ARGS__); \ 2948 DISPLAYLEVEL(1, " \n"); \ 2949 return n; \ 2950 } \ 2951} 2952 2953static InfoError 2954FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) 2955{ 2956 /* begin analyzing frame */ 2957 for ( ; ; ) { 2958 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 2959 size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); 2960 if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) { 2961 if ( feof(srcFile) 2962 && (numBytesRead == 0) 2963 && (info->compressedSize > 0) 2964 && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { 2965 unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); 2966 unsigned long long file_size = (unsigned long long) info->compressedSize; 2967 ERROR_IF(file_position != file_size, info_truncated_input, 2968 "Error: seeked to position %llu, which is beyond file size of %llu\n", 2969 file_position, 2970 file_size); 2971 break; /* correct end of file => success */ 2972 } 2973 ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); 2974 ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); 2975 } 2976 { U32 const magicNumber = MEM_readLE32(headerBuffer); 2977 /* Zstandard frame */ 2978 if (magicNumber == ZSTD_MAGICNUMBER) { 2979 ZSTD_frameHeader header; 2980 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); 2981 if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR 2982 || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { 2983 info->decompUnavailable = 1; 2984 } else { 2985 info->decompressedSize += frameContentSize; 2986 } 2987 ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, 2988 info_frame_error, "Error: could not decode frame header"); 2989 info->windowSize = header.windowSize; 2990 /* move to the end of the frame header */ 2991 { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); 2992 ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); 2993 ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, 2994 info_frame_error, "Error: could not move to end of frame header"); 2995 } 2996 2997 /* skip all blocks in the frame */ 2998 { int lastBlock = 0; 2999 do { 3000 BYTE blockHeaderBuffer[3]; 3001 ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, 3002 info_frame_error, "Error while reading block header"); 3003 { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); 3004 U32 const blockTypeID = (blockHeader >> 1) & 3; 3005 U32 const isRLE = (blockTypeID == 1); 3006 U32 const isWrongBlock = (blockTypeID == 3); 3007 long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); 3008 ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); 3009 lastBlock = blockHeader & 1; 3010 ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, 3011 info_frame_error, "Error: could not skip to end of block"); 3012 } 3013 } while (lastBlock != 1); 3014 } 3015 3016 /* check if checksum is used */ 3017 { BYTE const frameHeaderDescriptor = headerBuffer[4]; 3018 int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; 3019 if (contentChecksumFlag) { 3020 info->usesCheck = 1; 3021 ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, 3022 info_frame_error, "Error: could not skip past checksum"); 3023 } } 3024 info->numActualFrames++; 3025 } 3026 /* Skippable frame */ 3027 else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 3028 U32 const frameSize = MEM_readLE32(headerBuffer + 4); 3029 long const seek = (long)(8 + frameSize - numBytesRead); 3030 ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, 3031 info_frame_error, "Error: could not find end of skippable frame"); 3032 info->numSkippableFrames++; 3033 } 3034 /* unknown content */ 3035 else { 3036 return info_not_zstd; 3037 } 3038 } /* magic number analysis */ 3039 } /* end analyzing frames */ 3040 return info_success; 3041} 3042 3043 3044static InfoError 3045getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) 3046{ 3047 InfoError status; 3048 FILE* const srcFile = FIO_openSrcFile(NULL, inFileName); 3049 ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); 3050 3051 info->compressedSize = UTIL_getFileSize(inFileName); 3052 status = FIO_analyzeFrames(info, srcFile); 3053 3054 fclose(srcFile); 3055 info->nbFiles = 1; 3056 return status; 3057} 3058 3059 3060/** getFileInfo() : 3061 * Reads information from file, stores in *info 3062 * @return : InfoError status 3063 */ 3064static InfoError 3065getFileInfo(fileInfo_t* info, const char* srcFileName) 3066{ 3067 ERROR_IF(!UTIL_isRegularFile(srcFileName), 3068 info_file_error, "Error : %s is not a file", srcFileName); 3069 return getFileInfo_fileConfirmed(info, srcFileName); 3070} 3071 3072 3073static void 3074displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) 3075{ 3076 UTIL_HumanReadableSize_t const window_hrs = UTIL_makeHumanReadableSize(info->windowSize); 3077 UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(info->compressedSize); 3078 UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(info->decompressedSize); 3079 double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize; 3080 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); 3081 if (displayLevel <= 2) { 3082 if (!info->decompUnavailable) { 3083 DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %s\n", 3084 info->numSkippableFrames + info->numActualFrames, 3085 info->numSkippableFrames, 3086 compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, 3087 decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, 3088 ratio, checkString, inFileName); 3089 } else { 3090 DISPLAYOUT("%6d %5d %6.*f%4s %5s %s\n", 3091 info->numSkippableFrames + info->numActualFrames, 3092 info->numSkippableFrames, 3093 compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, 3094 checkString, inFileName); 3095 } 3096 } else { 3097 DISPLAYOUT("%s \n", inFileName); 3098 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); 3099 if (info->numSkippableFrames) 3100 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); 3101 DISPLAYOUT("Window Size: %.*f%s (%llu B)\n", 3102 window_hrs.precision, window_hrs.value, window_hrs.suffix, 3103 (unsigned long long)info->windowSize); 3104 DISPLAYOUT("Compressed Size: %.*f%s (%llu B)\n", 3105 compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, 3106 (unsigned long long)info->compressedSize); 3107 if (!info->decompUnavailable) { 3108 DISPLAYOUT("Decompressed Size: %.*f%s (%llu B)\n", 3109 decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, 3110 (unsigned long long)info->decompressedSize); 3111 DISPLAYOUT("Ratio: %.4f\n", ratio); 3112 } 3113 DISPLAYOUT("Check: %s\n", checkString); 3114 DISPLAYOUT("\n"); 3115 } 3116} 3117 3118static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) 3119{ 3120 fileInfo_t total; 3121 memset(&total, 0, sizeof(total)); 3122 total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; 3123 total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; 3124 total.compressedSize = fi1.compressedSize + fi2.compressedSize; 3125 total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; 3126 total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; 3127 total.usesCheck = fi1.usesCheck & fi2.usesCheck; 3128 total.nbFiles = fi1.nbFiles + fi2.nbFiles; 3129 return total; 3130} 3131 3132static int 3133FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) 3134{ 3135 fileInfo_t info; 3136 memset(&info, 0, sizeof(info)); 3137 { InfoError const error = getFileInfo(&info, inFileName); 3138 switch (error) { 3139 case info_frame_error: 3140 /* display error, but provide output */ 3141 DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); 3142 break; 3143 case info_not_zstd: 3144 DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); 3145 if (displayLevel > 2) DISPLAYOUT("\n"); 3146 return 1; 3147 case info_file_error: 3148 /* error occurred while opening the file */ 3149 if (displayLevel > 2) DISPLAYOUT("\n"); 3150 return 1; 3151 case info_truncated_input: 3152 DISPLAYOUT("File \"%s\" is truncated \n", inFileName); 3153 if (displayLevel > 2) DISPLAYOUT("\n"); 3154 return 1; 3155 case info_success: 3156 default: 3157 break; 3158 } 3159 3160 displayInfo(inFileName, &info, displayLevel); 3161 *total = FIO_addFInfo(*total, info); 3162 assert(error == info_success || error == info_frame_error); 3163 return (int)error; 3164 } 3165} 3166 3167int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel) 3168{ 3169 /* ensure no specified input is stdin (needs fseek() capability) */ 3170 { unsigned u; 3171 for (u=0; u<numFiles;u++) { 3172 ERROR_IF(!strcmp (filenameTable[u], stdinmark), 3173 1, "zstd: --list does not support reading from standard input"); 3174 } } 3175 3176 if (numFiles == 0) { 3177 if (!IS_CONSOLE(stdin)) { 3178 DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); 3179 } 3180 DISPLAYLEVEL(1, "No files given \n"); 3181 return 1; 3182 } 3183 3184 if (displayLevel <= 2) { 3185 DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); 3186 } 3187 { int error = 0; 3188 fileInfo_t total; 3189 memset(&total, 0, sizeof(total)); 3190 total.usesCheck = 1; 3191 /* --list each file, and check for any error */ 3192 { unsigned u; 3193 for (u=0; u<numFiles;u++) { 3194 error |= FIO_listFile(&total, filenameTable[u], displayLevel); 3195 } } 3196 if (numFiles > 1 && displayLevel <= 2) { /* display total */ 3197 UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(total.compressedSize); 3198 UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(total.decompressedSize); 3199 double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize; 3200 const char* const checkString = (total.usesCheck ? "XXH64" : ""); 3201 DISPLAYOUT("----------------------------------------------------------------- \n"); 3202 if (total.decompUnavailable) { 3203 DISPLAYOUT("%6d %5d %6.*f%4s %5s %u files\n", 3204 total.numSkippableFrames + total.numActualFrames, 3205 total.numSkippableFrames, 3206 compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, 3207 checkString, (unsigned)total.nbFiles); 3208 } else { 3209 DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %u files\n", 3210 total.numSkippableFrames + total.numActualFrames, 3211 total.numSkippableFrames, 3212 compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, 3213 decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, 3214 ratio, checkString, (unsigned)total.nbFiles); 3215 } } 3216 return error; 3217 } 3218} 3219 3220 3221#endif /* #ifndef ZSTD_NODECOMPRESS */ 3222