1/*
2 * Copyright (c) Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12/*-************************************
13*  Tuning parameters
14**************************************/
15#ifndef ZSTDCLI_CLEVEL_DEFAULT
16#  define ZSTDCLI_CLEVEL_DEFAULT 3
17#endif
18
19#ifndef ZSTDCLI_CLEVEL_MAX
20#  define ZSTDCLI_CLEVEL_MAX 19   /* without using --ultra */
21#endif
22
23#ifndef ZSTDCLI_NBTHREADS_DEFAULT
24#  define ZSTDCLI_NBTHREADS_DEFAULT 1
25#endif
26
27/*-************************************
28*  Dependencies
29**************************************/
30#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
31#include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
32#include <stdlib.h>   /* getenv */
33#include <string.h>   /* strcmp, strlen */
34#include <stdio.h>    /* fprintf(), stdin, stdout, stderr */
35#include <errno.h>    /* errno */
36#include <assert.h>   /* assert */
37
38#include "fileio.h"   /* stdinmark, stdoutmark, ZSTD_EXTENSION */
39#ifndef ZSTD_NOBENCH
40#  include "benchzstd.h"  /* BMK_benchFiles */
41#endif
42#ifndef ZSTD_NODICT
43#  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
44#endif
45#ifndef ZSTD_NOTRACE
46#  include "zstdcli_trace.h"
47#endif
48#include "../lib/zstd.h"  /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
49
50
51/*-************************************
52*  Constants
53**************************************/
54#define COMPRESSOR_NAME "zstd command line interface"
55#ifndef ZSTD_VERSION
56#  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
57#endif
58#define AUTHOR "Yann Collet"
59#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
60
61#define ZSTD_ZSTDMT "zstdmt"
62#define ZSTD_UNZSTD "unzstd"
63#define ZSTD_CAT "zstdcat"
64#define ZSTD_ZCAT "zcat"
65#define ZSTD_GZ "gzip"
66#define ZSTD_GUNZIP "gunzip"
67#define ZSTD_GZCAT "gzcat"
68#define ZSTD_LZMA "lzma"
69#define ZSTD_UNLZMA "unlzma"
70#define ZSTD_XZ "xz"
71#define ZSTD_UNXZ "unxz"
72#define ZSTD_LZ4 "lz4"
73#define ZSTD_UNLZ4 "unlz4"
74
75#define KB *(1 <<10)
76#define MB *(1 <<20)
77#define GB *(1U<<30)
78
79#define DISPLAY_LEVEL_DEFAULT 2
80
81static const char*    g_defaultDictName = "dictionary";
82static const unsigned g_defaultMaxDictSize = 110 KB;
83static const int      g_defaultDictCLevel = 3;
84static const unsigned g_defaultSelectivityLevel = 9;
85static const unsigned g_defaultMaxWindowLog = 27;
86#define OVERLAP_LOG_DEFAULT 9999
87#define LDM_PARAM_DEFAULT 9999  /* Default for parameters where 0 is valid */
88static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
89static U32 g_ldmHashLog = 0;
90static U32 g_ldmMinMatch = 0;
91static U32 g_ldmHashRateLog = LDM_PARAM_DEFAULT;
92static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
93
94
95#define DEFAULT_ACCEL 1
96
97typedef enum { cover, fastCover, legacy } dictType;
98
99/*-************************************
100*  Display Macros
101**************************************/
102#define DISPLAY_F(f, ...)    fprintf((f), __VA_ARGS__)
103#define DISPLAYOUT(...)      DISPLAY_F(stdout, __VA_ARGS__)
104#define DISPLAY(...)         DISPLAY_F(stderr, __VA_ARGS__)
105#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
106static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
107
108
109/*-************************************
110*  Check Version (when CLI linked to dynamic library)
111**************************************/
112
113/* Due to usage of experimental symbols and capabilities by the CLI,
114 * the CLI must be linked against a dynamic library of same version */
115static void checkLibVersion(void)
116{
117    if (strcmp(ZSTD_VERSION_STRING, ZSTD_versionString())) {
118        DISPLAYLEVEL(1, "Error : incorrect library version (expecting : %s ; actual : %s ) \n",
119                    ZSTD_VERSION_STRING, ZSTD_versionString());
120        DISPLAYLEVEL(1, "Please update library to version %s, or use stand-alone zstd binary \n",
121                    ZSTD_VERSION_STRING);
122        exit(1);
123    }
124}
125
126
127/*-************************************
128*  Command Line
129**************************************/
130/* print help either in `stderr` or `stdout` depending on originating request
131 * error (badusage) => stderr
132 * help (usage_advanced) => stdout
133 */
134static void usage(FILE* f, const char* programName)
135{
136    DISPLAY_F(f, "Usage : \n");
137    DISPLAY_F(f, "      %s [args] [FILE(s)] [-o file] \n", programName);
138    DISPLAY_F(f, "\n");
139    DISPLAY_F(f, "FILE    : a filename \n");
140    DISPLAY_F(f, "          with no FILE, or when FILE is - , read standard input\n");
141    DISPLAY_F(f, "Arguments : \n");
142#ifndef ZSTD_NOCOMPRESS
143    DISPLAY_F(f, " -#     : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
144#endif
145#ifndef ZSTD_NODECOMPRESS
146    DISPLAY_F(f, " -d     : decompression \n");
147#endif
148    DISPLAY_F(f, " -D DICT: use DICT as Dictionary for compression or decompression \n");
149    DISPLAY_F(f, " -o file: result stored into `file` (only 1 output file) \n");
150    DISPLAY_F(f, " -f     : disable input and output checks. Allows overwriting existing files,\n");
151    DISPLAY_F(f, "          input from console, output to stdout, operating on links,\n");
152    DISPLAY_F(f, "          block devices, etc.\n");
153    DISPLAY_F(f, "--rm    : remove source file(s) after successful de/compression \n");
154    DISPLAY_F(f, " -k     : preserve source file(s) (default) \n");
155    DISPLAY_F(f, " -h/-H  : display help/long help and exit \n");
156}
157
158static void usage_advanced(const char* programName)
159{
160    DISPLAYOUT(WELCOME_MESSAGE);
161    usage(stdout, programName);
162    DISPLAYOUT( "\n");
163    DISPLAYOUT( "Advanced arguments : \n");
164    DISPLAYOUT( " -V     : display Version number and exit \n");
165
166    DISPLAYOUT( " -c     : write to standard output (even if it is the console) \n");
167
168    DISPLAYOUT( " -v     : verbose mode; specify multiple times to increase verbosity \n");
169    DISPLAYOUT( " -q     : suppress warnings; specify twice to suppress errors too \n");
170    DISPLAYOUT( "--[no-]progress : forcibly display, or never display the progress counter.\n");
171    DISPLAYOUT( "                  note: any (de)compressed output to terminal will mix with progress counter text. \n");
172
173#ifdef UTIL_HAS_CREATEFILELIST
174    DISPLAYOUT( " -r     : operate recursively on directories \n");
175    DISPLAYOUT( "--filelist FILE : read list of files to operate upon from FILE \n");
176    DISPLAYOUT( "--output-dir-flat DIR : processed files are stored into DIR \n");
177#endif
178
179#ifdef UTIL_HAS_MIRRORFILELIST
180    DISPLAYOUT( "--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure \n");
181#endif
182
183
184#ifndef ZSTD_NOCOMPRESS
185    DISPLAYOUT( "--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled)");
186#ifndef ZSTD_NODECOMPRESS
187    DISPLAYOUT( ". If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate).");
188#endif
189#else
190#ifdef ZSTD_NOCOMPRESS
191    DISPLAYOUT( "--[no-]check : during decompression, ignore/validate checksums in compressed frame (default: validate).");
192#endif
193#endif /* ZSTD_NOCOMPRESS */
194
195#ifndef ZSTD_NOTRACE
196    DISPLAYOUT( "\n");
197    DISPLAYOUT( "--trace FILE : log tracing information to FILE.");
198#endif
199    DISPLAYOUT( "\n");
200
201    DISPLAYOUT( "--      : All arguments after \"--\" are treated as files \n");
202
203#ifndef ZSTD_NOCOMPRESS
204    DISPLAYOUT( "\n");
205    DISPLAYOUT( "Advanced compression arguments : \n");
206    DISPLAYOUT( "--ultra : enable levels beyond %i, up to %i (requires more memory) \n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
207    DISPLAYOUT( "--long[=#]: enable long distance matching with given window log (default: %u) \n", g_defaultMaxWindowLog);
208    DISPLAYOUT( "--fast[=#]: switch to very fast compression levels (default: %u) \n", 1);
209    DISPLAYOUT( "--adapt : dynamically adapt compression level to I/O conditions \n");
210    DISPLAYOUT( "--[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies \n");
211    DISPLAYOUT( "--patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n");
212# ifdef ZSTD_MULTITHREAD
213    DISPLAYOUT( " -T#    : spawns # compression threads (default: 1, 0==# cores) \n");
214    DISPLAYOUT( " -B#    : select size of each job (default: 0==automatic) \n");
215    DISPLAYOUT( "--single-thread : use a single thread for both I/O and compression (result slightly different than -T1) \n");
216    DISPLAYOUT( "--auto-threads={physical,logical} (default: physical} : use either physical cores or logical cores as default when specifying -T0 \n");
217    DISPLAYOUT( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
218# endif
219    DISPLAYOUT( "--exclude-compressed: only compress files that are not already compressed \n");
220    DISPLAYOUT( "--stream-size=# : specify size of streaming input from `stdin` \n");
221    DISPLAYOUT( "--size-hint=# optimize compression parameters for streaming input of approximately this size \n");
222    DISPLAYOUT( "--target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
223    DISPLAYOUT( "--no-dictID : don't write dictID into header (dictionary compression only) \n");
224    DISPLAYOUT( "--[no-]compress-literals : force (un)compressed literals \n");
225
226    DISPLAYOUT( "--format=zstd : compress files to the .zst format (default) \n");
227#ifdef ZSTD_GZCOMPRESS
228    DISPLAYOUT( "--format=gzip : compress files to the .gz format \n");
229#endif
230#ifdef ZSTD_LZMACOMPRESS
231    DISPLAYOUT( "--format=xz : compress files to the .xz format \n");
232    DISPLAYOUT( "--format=lzma : compress files to the .lzma format \n");
233#endif
234#ifdef ZSTD_LZ4COMPRESS
235    DISPLAYOUT( "--format=lz4 : compress files to the .lz4 format \n");
236#endif
237#endif  /* !ZSTD_NOCOMPRESS */
238
239#ifndef ZSTD_NODECOMPRESS
240    DISPLAYOUT( "\n");
241    DISPLAYOUT( "Advanced decompression arguments : \n");
242    DISPLAYOUT( " -l     : print information about zstd compressed files \n");
243    DISPLAYOUT( "--test  : test compressed file integrity \n");
244    DISPLAYOUT( " -M#    : Set a memory usage limit for decompression \n");
245# if ZSTD_SPARSE_DEFAULT
246    DISPLAYOUT( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout) \n");
247# else
248    DISPLAYOUT( "--[no-]sparse : sparse mode (default: disabled) \n");
249# endif
250#endif  /* ZSTD_NODECOMPRESS */
251
252#ifndef ZSTD_NODICT
253    DISPLAYOUT( "\n");
254    DISPLAYOUT( "Dictionary builder : \n");
255    DISPLAYOUT( "--train ## : create a dictionary from a training set of files \n");
256    DISPLAYOUT( "--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args \n");
257    DISPLAYOUT( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args \n");
258    DISPLAYOUT( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u) \n", g_defaultSelectivityLevel);
259    DISPLAYOUT( " -o DICT : DICT is dictionary name (default: %s) \n", g_defaultDictName);
260    DISPLAYOUT( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
261    DISPLAYOUT( "--dictID=# : force dictionary ID to specified value (default: random) \n");
262#endif
263
264#ifndef ZSTD_NOBENCH
265    DISPLAYOUT( "\n");
266    DISPLAYOUT( "Benchmark arguments : \n");
267    DISPLAYOUT( " -b#    : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
268    DISPLAYOUT( " -e#    : test all compression levels successively from -b# to -e# (default: 1) \n");
269    DISPLAYOUT( " -i#    : minimum evaluation time in seconds (default: 3s) \n");
270    DISPLAYOUT( " -B#    : cut file into independent blocks of size # (default: no block) \n");
271    DISPLAYOUT( " -S     : output one benchmark result per input file (default: consolidated result) \n");
272    DISPLAYOUT( "--priority=rt : set process priority to real-time \n");
273#endif
274
275}
276
277static void badusage(const char* programName)
278{
279    DISPLAYLEVEL(1, "Incorrect parameters \n");
280    if (g_displayLevel >= 2) usage(stderr, programName);
281}
282
283static void waitEnter(void)
284{
285    int unused;
286    DISPLAY("Press enter to continue... \n");
287    unused = getchar();
288    (void)unused;
289}
290
291static const char* lastNameFromPath(const char* path)
292{
293    const char* name = path;
294    if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
295    if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
296    return name;
297}
298
299/*! exeNameMatch() :
300    @return : a non-zero value if exeName matches test, excluding the extension
301   */
302static int exeNameMatch(const char* exeName, const char* test)
303{
304    return !strncmp(exeName, test, strlen(test)) &&
305        (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
306}
307
308static void errorOut(const char* msg)
309{
310    DISPLAY("%s \n", msg); exit(1);
311}
312
313/*! readU32FromCharChecked() :
314 * @return 0 if success, and store the result in *value.
315 *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
316 *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
317 * @return 1 if an overflow error occurs */
318static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
319{
320    unsigned result = 0;
321    while ((**stringPtr >='0') && (**stringPtr <='9')) {
322        unsigned const max = ((unsigned)(-1)) / 10;
323        unsigned last = result;
324        if (result > max) return 1; /* overflow error */
325        result *= 10;
326        result += (unsigned)(**stringPtr - '0');
327        if (result < last) return 1; /* overflow error */
328        (*stringPtr)++ ;
329    }
330    if ((**stringPtr=='K') || (**stringPtr=='M')) {
331        unsigned const maxK = ((unsigned)(-1)) >> 10;
332        if (result > maxK) return 1; /* overflow error */
333        result <<= 10;
334        if (**stringPtr=='M') {
335            if (result > maxK) return 1; /* overflow error */
336            result <<= 10;
337        }
338        (*stringPtr)++;  /* skip `K` or `M` */
339        if (**stringPtr=='i') (*stringPtr)++;
340        if (**stringPtr=='B') (*stringPtr)++;
341    }
342    *value = result;
343    return 0;
344}
345
346/*! readU32FromChar() :
347 * @return : unsigned integer value read from input in `char` format.
348 *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
349 *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
350 *  Note : function will exit() program if digit sequence overflows */
351static unsigned readU32FromChar(const char** stringPtr) {
352    static const char errorMsg[] = "error: numeric value overflows 32-bit unsigned int";
353    unsigned result;
354    if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
355    return result;
356}
357
358/*! readIntFromChar() :
359 * @return : signed integer value read from input in `char` format.
360 *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
361 *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
362 *  Note : function will exit() program if digit sequence overflows */
363static int readIntFromChar(const char** stringPtr) {
364    static const char errorMsg[] = "error: numeric value overflows 32-bit int";
365    int sign = 1;
366    unsigned result;
367    if (**stringPtr=='-') {
368        (*stringPtr)++;
369        sign = -1;
370    }
371    if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
372    return (int) result * sign;
373}
374
375/*! readSizeTFromCharChecked() :
376 * @return 0 if success, and store the result in *value.
377 *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
378 *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
379 * @return 1 if an overflow error occurs */
380static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
381{
382    size_t result = 0;
383    while ((**stringPtr >='0') && (**stringPtr <='9')) {
384        size_t const max = ((size_t)(-1)) / 10;
385        size_t last = result;
386        if (result > max) return 1; /* overflow error */
387        result *= 10;
388        result += (size_t)(**stringPtr - '0');
389        if (result < last) return 1; /* overflow error */
390        (*stringPtr)++ ;
391    }
392    if ((**stringPtr=='K') || (**stringPtr=='M')) {
393        size_t const maxK = ((size_t)(-1)) >> 10;
394        if (result > maxK) return 1; /* overflow error */
395        result <<= 10;
396        if (**stringPtr=='M') {
397            if (result > maxK) return 1; /* overflow error */
398            result <<= 10;
399        }
400        (*stringPtr)++;  /* skip `K` or `M` */
401        if (**stringPtr=='i') (*stringPtr)++;
402        if (**stringPtr=='B') (*stringPtr)++;
403    }
404    *value = result;
405    return 0;
406}
407
408/*! readSizeTFromChar() :
409 * @return : size_t value read from input in `char` format.
410 *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
411 *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
412 *  Note : function will exit() program if digit sequence overflows */
413static size_t readSizeTFromChar(const char** stringPtr) {
414    static const char errorMsg[] = "error: numeric value overflows size_t";
415    size_t result;
416    if (readSizeTFromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
417    return result;
418}
419
420/** longCommandWArg() :
421 *  check if *stringPtr is the same as longCommand.
422 *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
423 * @return 0 and doesn't modify *stringPtr otherwise.
424 */
425static int longCommandWArg(const char** stringPtr, const char* longCommand)
426{
427    size_t const comSize = strlen(longCommand);
428    int const result = !strncmp(*stringPtr, longCommand, comSize);
429    if (result) *stringPtr += comSize;
430    return result;
431}
432
433
434#ifndef ZSTD_NODICT
435
436static const unsigned kDefaultRegression = 1;
437/**
438 * parseCoverParameters() :
439 * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params
440 * @return 1 means that cover parameters were correct
441 * @return 0 in case of malformed parameters
442 */
443static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params)
444{
445    memset(params, 0, sizeof(*params));
446    for (; ;) {
447        if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
448        if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
449        if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
450        if (longCommandWArg(&stringPtr, "split=")) {
451          unsigned splitPercentage = readU32FromChar(&stringPtr);
452          params->splitPoint = (double)splitPercentage / 100.0;
453          if (stringPtr[0]==',') { stringPtr++; continue; } else break;
454        }
455        if (longCommandWArg(&stringPtr, "shrink")) {
456          params->shrinkDictMaxRegression = kDefaultRegression;
457          params->shrinkDict = 1;
458          if (stringPtr[0]=='=') {
459            stringPtr++;
460            params->shrinkDictMaxRegression = readU32FromChar(&stringPtr);
461          }
462          if (stringPtr[0]==',') {
463            stringPtr++;
464            continue;
465          }
466          else break;
467        }
468        return 0;
469    }
470    if (stringPtr[0] != 0) return 0;
471    DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\nshrink%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100), params->shrinkDictMaxRegression);
472    return 1;
473}
474
475/**
476 * parseFastCoverParameters() :
477 * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
478 * @return 1 means that fastcover parameters were correct
479 * @return 0 in case of malformed parameters
480 */
481static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
482{
483    memset(params, 0, sizeof(*params));
484    for (; ;) {
485        if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
486        if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
487        if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
488        if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
489        if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
490        if (longCommandWArg(&stringPtr, "split=")) {
491          unsigned splitPercentage = readU32FromChar(&stringPtr);
492          params->splitPoint = (double)splitPercentage / 100.0;
493          if (stringPtr[0]==',') { stringPtr++; continue; } else break;
494        }
495        if (longCommandWArg(&stringPtr, "shrink")) {
496          params->shrinkDictMaxRegression = kDefaultRegression;
497          params->shrinkDict = 1;
498          if (stringPtr[0]=='=') {
499            stringPtr++;
500            params->shrinkDictMaxRegression = readU32FromChar(&stringPtr);
501          }
502          if (stringPtr[0]==',') {
503            stringPtr++;
504            continue;
505          }
506          else break;
507        }
508        return 0;
509    }
510    if (stringPtr[0] != 0) return 0;
511    DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\nshrink=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel, params->shrinkDictMaxRegression);
512    return 1;
513}
514
515/**
516 * parseLegacyParameters() :
517 * reads legacy dictionary builder parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
518 * @return 1 means that legacy dictionary builder parameters were correct
519 * @return 0 in case of malformed parameters
520 */
521static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity)
522{
523    if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; }
524    *selectivity = readU32FromChar(&stringPtr);
525    if (stringPtr[0] != 0) return 0;
526    DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity);
527    return 1;
528}
529
530static ZDICT_cover_params_t defaultCoverParams(void)
531{
532    ZDICT_cover_params_t params;
533    memset(&params, 0, sizeof(params));
534    params.d = 8;
535    params.steps = 4;
536    params.splitPoint = 1.0;
537    params.shrinkDict = 0;
538    params.shrinkDictMaxRegression = kDefaultRegression;
539    return params;
540}
541
542static ZDICT_fastCover_params_t defaultFastCoverParams(void)
543{
544    ZDICT_fastCover_params_t params;
545    memset(&params, 0, sizeof(params));
546    params.d = 8;
547    params.f = 20;
548    params.steps = 4;
549    params.splitPoint = 0.75; /* different from default splitPoint of cover */
550    params.accel = DEFAULT_ACCEL;
551    params.shrinkDict = 0;
552    params.shrinkDictMaxRegression = kDefaultRegression;
553    return params;
554}
555#endif
556
557
558/** parseAdaptParameters() :
559 *  reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr.
560 *  Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized.
561 *  There is no guarantee that any of these values will be updated.
562 *  @return 1 means that parsing was successful,
563 *  @return 0 in case of malformed parameters
564 */
565static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
566{
567    for ( ; ;) {
568        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readIntFromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
569        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readIntFromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
570        DISPLAYLEVEL(4, "invalid compression parameter \n");
571        return 0;
572    }
573    if (stringPtr[0] != 0) return 0; /* check the end of string */
574    if (*adaptMinPtr > *adaptMaxPtr) {
575        DISPLAYLEVEL(4, "incoherent adaptation limits \n");
576        return 0;
577    }
578    return 1;
579}
580
581
582/** parseCompressionParameters() :
583 *  reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6") into *params
584 *  @return 1 means that compression parameters were correct
585 *  @return 0 in case of malformed parameters
586 */
587static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
588{
589    for ( ; ;) {
590        if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
591        if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
592        if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
593        if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
594        if (longCommandWArg(&stringPtr, "minMatch=") || longCommandWArg(&stringPtr, "mml=")) { params->minMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
595        if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
596        if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
597        if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
598        if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "lhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
599        if (longCommandWArg(&stringPtr, "ldmMinMatch=") || longCommandWArg(&stringPtr, "lmml=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
600        if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "lblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
601        if (longCommandWArg(&stringPtr, "ldmHashRateLog=") || longCommandWArg(&stringPtr, "lhrlog=")) { g_ldmHashRateLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
602        DISPLAYLEVEL(4, "invalid compression parameter \n");
603        return 0;
604    }
605
606    DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
607    DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
608    if (stringPtr[0] != 0) return 0; /* check the end of string */
609    return 1;
610}
611
612static void printVersion(void)
613{
614    if (g_displayLevel < DISPLAY_LEVEL_DEFAULT) {
615        DISPLAYOUT("%s\n", ZSTD_VERSION_STRING);
616        return;
617    }
618
619    DISPLAYOUT(WELCOME_MESSAGE);
620    if (g_displayLevel >= 3) {
621    /* format support */
622        DISPLAYOUT("*** supports: zstd");
623    #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
624        DISPLAYOUT(", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
625    #endif
626    #ifdef ZSTD_GZCOMPRESS
627        DISPLAYOUT(", gzip");
628    #endif
629    #ifdef ZSTD_LZ4COMPRESS
630        DISPLAYOUT(", lz4");
631    #endif
632    #ifdef ZSTD_LZMACOMPRESS
633        DISPLAYOUT(", lzma, xz ");
634    #endif
635        DISPLAYOUT("\n");
636        if (g_displayLevel >= 4) {
637            /* posix support */
638        #ifdef _POSIX_C_SOURCE
639            DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
640        #endif
641        #ifdef _POSIX_VERSION
642            DISPLAYOUT("_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
643        #endif
644        #ifdef PLATFORM_POSIX_VERSION
645            DISPLAYOUT("PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
646        #endif
647    }   }
648}
649
650#define ZSTD_NB_STRATEGIES 9
651static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast",
652                "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2",
653                "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"};
654
655#ifndef ZSTD_NOCOMPRESS
656
657static void printDefaultCParams(const char* filename, const char* dictFileName, int cLevel) {
658    unsigned long long fileSize = UTIL_getFileSize(filename);
659    const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
660    const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
661    if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filename, (unsigned)fileSize);
662    else DISPLAY("%s (src size unknown)\n", filename);
663    DISPLAY(" - windowLog     : %u\n", cParams.windowLog);
664    DISPLAY(" - chainLog      : %u\n", cParams.chainLog);
665    DISPLAY(" - hashLog       : %u\n", cParams.hashLog);
666    DISPLAY(" - searchLog     : %u\n", cParams.searchLog);
667    DISPLAY(" - minMatch      : %u\n", cParams.minMatch);
668    DISPLAY(" - targetLength  : %u\n", cParams.targetLength);
669    assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1);
670    DISPLAY(" - strategy      : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy);
671}
672
673static void printActualCParams(const char* filename, const char* dictFileName, int cLevel, const ZSTD_compressionParameters* cParams) {
674    unsigned long long fileSize = UTIL_getFileSize(filename);
675    const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
676    ZSTD_compressionParameters actualCParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
677    assert(g_displayLevel >= 4);
678    actualCParams.windowLog = cParams->windowLog == 0 ? actualCParams.windowLog : cParams->windowLog;
679    actualCParams.chainLog = cParams->chainLog == 0 ? actualCParams.chainLog : cParams->chainLog;
680    actualCParams.hashLog = cParams->hashLog == 0 ? actualCParams.hashLog : cParams->hashLog;
681    actualCParams.searchLog = cParams->searchLog == 0 ? actualCParams.searchLog : cParams->searchLog;
682    actualCParams.minMatch = cParams->minMatch == 0 ? actualCParams.minMatch : cParams->minMatch;
683    actualCParams.targetLength = cParams->targetLength == 0 ? actualCParams.targetLength : cParams->targetLength;
684    actualCParams.strategy = cParams->strategy == 0 ? actualCParams.strategy : cParams->strategy;
685    DISPLAY("--zstd=wlog=%d,clog=%d,hlog=%d,slog=%d,mml=%d,tlen=%d,strat=%d\n",
686            actualCParams.windowLog, actualCParams.chainLog, actualCParams.hashLog, actualCParams.searchLog,
687            actualCParams.minMatch, actualCParams.targetLength, actualCParams.strategy);
688}
689
690#endif
691
692/* Environment variables for parameter setting */
693#define ENV_CLEVEL "ZSTD_CLEVEL"
694#define ENV_NBTHREADS "ZSTD_NBTHREADS"    /* takes lower precedence than directly specifying -T# in the CLI */
695
696/* pick up environment variable */
697static int init_cLevel(void) {
698    const char* const env = getenv(ENV_CLEVEL);
699    if (env != NULL) {
700        const char* ptr = env;
701        int sign = 1;
702        if (*ptr == '-') {
703            sign = -1;
704            ptr++;
705        } else if (*ptr == '+') {
706            ptr++;
707        }
708
709        if ((*ptr>='0') && (*ptr<='9')) {
710            unsigned absLevel;
711            if (readU32FromCharChecked(&ptr, &absLevel)) {
712                DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_CLEVEL, env);
713                return ZSTDCLI_CLEVEL_DEFAULT;
714            } else if (*ptr == 0) {
715                return sign * (int)absLevel;
716        }   }
717
718        DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value \n", ENV_CLEVEL, env);
719    }
720
721    return ZSTDCLI_CLEVEL_DEFAULT;
722}
723
724#ifdef ZSTD_MULTITHREAD
725static unsigned init_nbThreads(void) {
726    const char* const env = getenv(ENV_NBTHREADS);
727    if (env != NULL) {
728        const char* ptr = env;
729        if ((*ptr>='0') && (*ptr<='9')) {
730            unsigned nbThreads;
731            if (readU32FromCharChecked(&ptr, &nbThreads)) {
732                DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
733                return ZSTDCLI_NBTHREADS_DEFAULT;
734            } else if (*ptr == 0) {
735                return nbThreads;
736            }
737        }
738        DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
739    }
740
741    return ZSTDCLI_NBTHREADS_DEFAULT;
742}
743#endif
744
745#define NEXT_FIELD(ptr) {         \
746    if (*argument == '=') {       \
747        ptr = ++argument;         \
748        argument += strlen(ptr);  \
749    } else {                      \
750        argNb++;                  \
751        if (argNb >= argCount) {  \
752            DISPLAY("error: missing command argument \n"); \
753            CLEAN_RETURN(1);      \
754        }                         \
755        ptr = argv[argNb];        \
756        assert(ptr != NULL);      \
757        if (ptr[0]=='-') {        \
758            DISPLAY("error: command cannot be separated from its argument by another command \n"); \
759            CLEAN_RETURN(1);      \
760}   }   }
761
762#define NEXT_UINT32(val32) {      \
763    const char* __nb;             \
764    NEXT_FIELD(__nb);             \
765    val32 = readU32FromChar(&__nb); \
766}
767
768typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
769
770#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
771
772#ifdef ZSTD_NOCOMPRESS
773/* symbols from compression library are not defined and should not be invoked */
774# define MINCLEVEL  -99
775# define MAXCLEVEL   22
776#else
777# define MINCLEVEL  ZSTD_minCLevel()
778# define MAXCLEVEL  ZSTD_maxCLevel()
779#endif
780
781int main(int argCount, const char* argv[])
782{
783    int argNb,
784        followLinks = 0,
785        allowBlockDevices = 0,
786        forceStdin = 0,
787        forceStdout = 0,
788        hasStdout = 0,
789        ldmFlag = 0,
790        main_pause = 0,
791        nbWorkers = 0,
792        adapt = 0,
793        useRowMatchFinder = 0,
794        adaptMin = MINCLEVEL,
795        adaptMax = MAXCLEVEL,
796        rsyncable = 0,
797        nextArgumentsAreFiles = 0,
798        operationResult = 0,
799        separateFiles = 0,
800        setRealTimePrio = 0,
801        singleThread = 0,
802#ifdef ZSTD_MULTITHREAD
803        defaultLogicalCores = 0,
804#endif
805        showDefaultCParams = 0,
806        ultra=0,
807        contentSize=1;
808    double compressibility = 0.5;
809    unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
810    size_t blockSize = 0;
811
812    FIO_prefs_t* const prefs = FIO_createPreferences();
813    FIO_ctx_t* const fCtx = FIO_createContext();
814    zstd_operation_mode operation = zom_compress;
815    ZSTD_compressionParameters compressionParams;
816    int cLevel = init_cLevel();
817    int cLevelLast = MINCLEVEL - 1;  /* lower than minimum */
818    unsigned recursive = 0;
819    unsigned memLimit = 0;
820    FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount);  /* argCount >= 1 */
821    FileNamesTable* file_of_names = UTIL_allocateFileNamesTable((size_t)argCount);  /* argCount >= 1 */
822    const char* programName = argv[0];
823    const char* outFileName = NULL;
824    const char* outDirName = NULL;
825    const char* outMirroredDirName = NULL;
826    const char* dictFileName = NULL;
827    const char* patchFromDictFileName = NULL;
828    const char* suffix = ZSTD_EXTENSION;
829    unsigned maxDictSize = g_defaultMaxDictSize;
830    unsigned dictID = 0;
831    size_t streamSrcSize = 0;
832    size_t targetCBlockSize = 0;
833    size_t srcSizeHint = 0;
834    int dictCLevel = g_defaultDictCLevel;
835    unsigned dictSelect = g_defaultSelectivityLevel;
836#ifndef ZSTD_NODICT
837    ZDICT_cover_params_t coverParams = defaultCoverParams();
838    ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
839    dictType dict = fastCover;
840#endif
841#ifndef ZSTD_NOBENCH
842    BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
843#endif
844    ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
845
846
847    /* init */
848    checkLibVersion();
849    (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
850    (void)memLimit;
851    assert(argCount >= 1);
852    if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
853    programName = lastNameFromPath(programName);
854#ifdef ZSTD_MULTITHREAD
855    nbWorkers = init_nbThreads();
856#endif
857
858    /* preset behaviors */
859    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
860    if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
861    if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
862    if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
863    if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1); }        /* behave like gzip */
864    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); }                                                     /* behave like gunzip, also supports multiple formats */
865    if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
866    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); }    /* behave like lzma */
867    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
868    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }          /* behave like xz */
869    if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }     /* behave like unxz, also supports multiple formats */
870    if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                       /* behave like lz4 */
871    if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); }                                   /* behave like unlz4, also supports multiple formats */
872    memset(&compressionParams, 0, sizeof(compressionParams));
873
874    /* init crash handler */
875    FIO_addAbortHandler();
876
877    /* command switches */
878    for (argNb=1; argNb<argCount; argNb++) {
879        const char* argument = argv[argNb];
880        if (!argument) continue;   /* Protection if argument empty */
881
882        if (nextArgumentsAreFiles) {
883            UTIL_refFilename(filenames, argument);
884            continue;
885        }
886
887        /* "-" means stdin/stdout */
888        if (!strcmp(argument, "-")){
889            UTIL_refFilename(filenames, stdinmark);
890            continue;
891        }
892
893        /* Decode commands (note : aggregated commands are allowed) */
894        if (argument[0]=='-') {
895
896            if (argument[1]=='-') {
897                /* long commands (--long-word) */
898                if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; }   /* only file names allowed from now on */
899                if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
900                if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
901                if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
902                if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
903                if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; continue; }
904                if (!strcmp(argument, "--version")) { printVersion(); CLEAN_RETURN(0); }
905                if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); }
906                if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
907                if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
908                if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
909                if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
910                if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
911                if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
912                if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
913                if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
914                if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
915                if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
916                if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
917                if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
918                if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
919                if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
920                if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
921                if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
922                if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
923                if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
924                if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = 1; continue; }
925                if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = 2; continue; }
926                if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
927                if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
928                if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
929#ifdef ZSTD_GZCOMPRESS
930                if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
931#endif
932#ifdef ZSTD_LZMACOMPRESS
933                if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression);  continue; }
934                if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression);  continue; }
935#endif
936#ifdef ZSTD_LZ4COMPRESS
937                if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression);  continue; }
938#endif
939                if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
940                if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_ps_enable; continue; }
941                if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_ps_disable; continue; }
942                if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
943                if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
944                if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
945
946                /* long commands with arguments */
947#ifndef ZSTD_NODICT
948                if (longCommandWArg(&argument, "--train-cover")) {
949                  operation = zom_train;
950                  if (outFileName == NULL)
951                      outFileName = g_defaultDictName;
952                  dict = cover;
953                  /* Allow optional arguments following an = */
954                  if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
955                  else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
956                  else if (!parseCoverParameters(argument, &coverParams)) { badusage(programName); CLEAN_RETURN(1); }
957                  continue;
958                }
959                if (longCommandWArg(&argument, "--train-fastcover")) {
960                  operation = zom_train;
961                  if (outFileName == NULL)
962                      outFileName = g_defaultDictName;
963                  dict = fastCover;
964                  /* Allow optional arguments following an = */
965                  if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
966                  else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
967                  else if (!parseFastCoverParameters(argument, &fastCoverParams)) { badusage(programName); CLEAN_RETURN(1); }
968                  continue;
969                }
970                if (longCommandWArg(&argument, "--train-legacy")) {
971                  operation = zom_train;
972                  if (outFileName == NULL)
973                      outFileName = g_defaultDictName;
974                  dict = legacy;
975                  /* Allow optional arguments following an = */
976                  if (*argument == 0) { continue; }
977                  else if (*argument++ != '=') { badusage(programName); CLEAN_RETURN(1); }
978                  else if (!parseLegacyParameters(argument, &dictSelect)) { badusage(programName); CLEAN_RETURN(1); }
979                  continue;
980                }
981#endif
982                if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
983                if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
984                if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
985                if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
986                if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; }
987                if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
988                if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
989                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
990                if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; }
991                if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
992                if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
993                if (longCommandWArg(&argument, "--output-dir-flat")) { NEXT_FIELD(outDirName); continue; }
994#ifdef ZSTD_MULTITHREAD
995                if (longCommandWArg(&argument, "--auto-threads")) {
996                    const char* threadDefault = NULL;
997                    NEXT_FIELD(threadDefault);
998                    if (strcmp(threadDefault, "logical") == 0)
999                        defaultLogicalCores = 1;
1000                    continue;
1001                }
1002#endif
1003#ifdef UTIL_HAS_MIRRORFILELIST
1004                if (longCommandWArg(&argument, "--output-dir-mirror")) { NEXT_FIELD(outMirroredDirName); continue; }
1005#endif
1006#ifndef ZSTD_NOTRACE
1007                if (longCommandWArg(&argument, "--trace")) { char const* traceFile; NEXT_FIELD(traceFile); TRACE_enable(traceFile); continue; }
1008#endif
1009                if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; }
1010                if (longCommandWArg(&argument, "--long")) {
1011                    unsigned ldmWindowLog = 0;
1012                    ldmFlag = 1;
1013                    /* Parse optional window log */
1014                    if (*argument == '=') {
1015                        ++argument;
1016                        ldmWindowLog = readU32FromChar(&argument);
1017                    } else if (*argument != 0) {
1018                        /* Invalid character following --long */
1019                        badusage(programName);
1020                        CLEAN_RETURN(1);
1021                    }
1022                    /* Only set windowLog if not already set by --zstd */
1023                    if (compressionParams.windowLog == 0)
1024                        compressionParams.windowLog = ldmWindowLog;
1025                    continue;
1026                }
1027#ifndef ZSTD_NOCOMPRESS   /* linking ZSTD_minCLevel() requires compression support */
1028                if (longCommandWArg(&argument, "--fast")) {
1029                    /* Parse optional acceleration factor */
1030                    if (*argument == '=') {
1031                        U32 const maxFast = (U32)-ZSTD_minCLevel();
1032                        U32 fastLevel;
1033                        ++argument;
1034                        fastLevel = readU32FromChar(&argument);
1035                        if (fastLevel > maxFast) fastLevel = maxFast;
1036                        if (fastLevel) {
1037                            dictCLevel = cLevel = -(int)fastLevel;
1038                        } else {
1039                            badusage(programName);
1040                            CLEAN_RETURN(1);
1041                        }
1042                    } else if (*argument != 0) {
1043                        /* Invalid character following --fast */
1044                        badusage(programName);
1045                        CLEAN_RETURN(1);
1046                    } else {
1047                        cLevel = -1;  /* default for --fast */
1048                    }
1049                    continue;
1050                }
1051#endif
1052
1053                if (longCommandWArg(&argument, "--filelist")) {
1054                    const char* listName;
1055                    NEXT_FIELD(listName);
1056                    UTIL_refFilename(file_of_names, listName);
1057                    continue;
1058                }
1059
1060                /* fall-through, will trigger bad_usage() later on */
1061            }
1062
1063            argument++;
1064            while (argument[0]!=0) {
1065
1066#ifndef ZSTD_NOCOMPRESS
1067                /* compression Level */
1068                if ((*argument>='0') && (*argument<='9')) {
1069                    dictCLevel = cLevel = (int)readU32FromChar(&argument);
1070                    continue;
1071                }
1072#endif
1073
1074                switch(argument[0])
1075                {
1076                    /* Display help */
1077                case 'V': printVersion(); CLEAN_RETURN(0);   /* Version Only */
1078                case 'H':
1079                case 'h': usage_advanced(programName); CLEAN_RETURN(0);
1080
1081                     /* Compress */
1082                case 'z': operation=zom_compress; argument++; break;
1083
1084                     /* Decoding */
1085                case 'd':
1086#ifndef ZSTD_NOBENCH
1087                        benchParams.mode = BMK_decodeOnly;
1088                        if (operation==zom_bench) { argument++; break; }  /* benchmark decode (hidden option) */
1089#endif
1090                        operation=zom_decompress; argument++; break;
1091
1092                    /* Force stdout, even if stdout==console */
1093                case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
1094
1095                    /* Use file content as dictionary */
1096                case 'D': argument++; NEXT_FIELD(dictFileName); break;
1097
1098                    /* Overwrite */
1099                case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; argument++; break;
1100
1101                    /* Verbose mode */
1102                case 'v': g_displayLevel++; argument++; break;
1103
1104                    /* Quiet mode */
1105                case 'q': g_displayLevel--; argument++; break;
1106
1107                    /* keep source file (default) */
1108                case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
1109
1110                    /* Checksum */
1111                case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
1112
1113                    /* test compressed file */
1114                case 't': operation=zom_test; argument++; break;
1115
1116                    /* destination file name */
1117                case 'o': argument++; NEXT_FIELD(outFileName); break;
1118
1119                    /* limit memory */
1120                case 'M':
1121                    argument++;
1122                    memLimit = readU32FromChar(&argument);
1123                    break;
1124                case 'l': operation=zom_list; argument++; break;
1125#ifdef UTIL_HAS_CREATEFILELIST
1126                    /* recursive */
1127                case 'r': recursive=1; argument++; break;
1128#endif
1129
1130#ifndef ZSTD_NOBENCH
1131                    /* Benchmark */
1132                case 'b':
1133                    operation=zom_bench;
1134                    argument++;
1135                    break;
1136
1137                    /* range bench (benchmark only) */
1138                case 'e':
1139                    /* compression Level */
1140                    argument++;
1141                    cLevelLast = (int)readU32FromChar(&argument);
1142                    break;
1143
1144                    /* Modify Nb Iterations (benchmark only) */
1145                case 'i':
1146                    argument++;
1147                    bench_nbSeconds = readU32FromChar(&argument);
1148                    break;
1149
1150                    /* cut input into blocks (benchmark only) */
1151                case 'B':
1152                    argument++;
1153                    blockSize = readU32FromChar(&argument);
1154                    break;
1155
1156                    /* benchmark files separately (hidden option) */
1157                case 'S':
1158                    argument++;
1159                    separateFiles = 1;
1160                    break;
1161
1162#endif   /* ZSTD_NOBENCH */
1163
1164                    /* nb of threads (hidden option) */
1165                case 'T':
1166                    argument++;
1167                    nbWorkers = (int)readU32FromChar(&argument);
1168                    break;
1169
1170                    /* Dictionary Selection level */
1171                case 's':
1172                    argument++;
1173                    dictSelect = readU32FromChar(&argument);
1174                    break;
1175
1176                    /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
1177                case 'p': argument++;
1178#ifndef ZSTD_NOBENCH
1179                    if ((*argument>='0') && (*argument<='9')) {
1180                        benchParams.additionalParam = (int)readU32FromChar(&argument);
1181                    } else
1182#endif
1183                        main_pause=1;
1184                    break;
1185
1186                    /* Select compressibility of synthetic sample */
1187                case 'P':
1188                    argument++;
1189                    compressibility = (double)readU32FromChar(&argument) / 100;
1190                    break;
1191
1192                    /* unknown command */
1193                default : badusage(programName); CLEAN_RETURN(1);
1194                }
1195            }
1196            continue;
1197        }   /* if (argument[0]=='-') */
1198
1199        /* none of the above : add filename to list */
1200        UTIL_refFilename(filenames, argument);
1201    }
1202
1203    /* Welcome message (if verbose) */
1204    DISPLAYLEVEL(3, WELCOME_MESSAGE);
1205
1206#ifdef ZSTD_MULTITHREAD
1207    if ((nbWorkers==0) && (!singleThread)) {
1208        /* automatically set # workers based on # of reported cpus */
1209        if (defaultLogicalCores) {
1210            nbWorkers = UTIL_countLogicalCores();
1211            DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
1212        } else {
1213            nbWorkers = UTIL_countPhysicalCores();
1214            DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
1215        }
1216    }
1217#else
1218    (void)singleThread; (void)nbWorkers;
1219#endif
1220
1221    g_utilDisplayLevel = g_displayLevel;
1222
1223#ifdef UTIL_HAS_CREATEFILELIST
1224    if (!followLinks) {
1225        unsigned u, fileNamesNb;
1226        unsigned const nbFilenames = (unsigned)filenames->tableSize;
1227        for (u=0, fileNamesNb=0; u<nbFilenames; u++) {
1228            if ( UTIL_isLink(filenames->fileNames[u])
1229             && !UTIL_isFIFO(filenames->fileNames[u])
1230            ) {
1231                DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring \n", filenames->fileNames[u]);
1232            } else {
1233                filenames->fileNames[fileNamesNb++] = filenames->fileNames[u];
1234        }   }
1235        if (fileNamesNb == 0 && nbFilenames > 0)  /* all names are eliminated */
1236            CLEAN_RETURN(1);
1237        filenames->tableSize = fileNamesNb;
1238    }   /* if (!followLinks) */
1239
1240    /* read names from a file */
1241    if (file_of_names->tableSize) {
1242        size_t const nbFileLists = file_of_names->tableSize;
1243        size_t flNb;
1244        for (flNb=0; flNb < nbFileLists; flNb++) {
1245            FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]);
1246            if (fnt==NULL) {
1247                DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
1248                CLEAN_RETURN(1);
1249            }
1250            filenames = UTIL_mergeFileNamesTable(filenames, fnt);
1251        }
1252    }
1253
1254    if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
1255        UTIL_expandFNT(&filenames, followLinks);
1256    }
1257#else
1258    (void)followLinks;
1259#endif
1260
1261    if (operation == zom_list) {
1262#ifndef ZSTD_NODECOMPRESS
1263        int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
1264        CLEAN_RETURN(ret);
1265#else
1266        DISPLAY("file information is not supported \n");
1267        CLEAN_RETURN(1);
1268#endif
1269    }
1270
1271    /* Check if benchmark is selected */
1272    if (operation==zom_bench) {
1273#ifndef ZSTD_NOBENCH
1274        benchParams.blockSize = blockSize;
1275        benchParams.nbWorkers = nbWorkers;
1276        benchParams.realTime = (unsigned)setRealTimePrio;
1277        benchParams.nbSeconds = bench_nbSeconds;
1278        benchParams.ldmFlag = ldmFlag;
1279        benchParams.ldmMinMatch = (int)g_ldmMinMatch;
1280        benchParams.ldmHashLog = (int)g_ldmHashLog;
1281        benchParams.useRowMatchFinder = useRowMatchFinder;
1282        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
1283            benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
1284        }
1285        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
1286            benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
1287        }
1288        benchParams.literalCompressionMode = literalCompressionMode;
1289
1290        if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
1291        if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
1292        if (cLevelLast < cLevel) cLevelLast = cLevel;
1293        if (cLevelLast > cLevel)
1294            DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
1295        if (filenames->tableSize > 0) {
1296            if(separateFiles) {
1297                unsigned i;
1298                for(i = 0; i < filenames->tableSize; i++) {
1299                    int c;
1300                    DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
1301                    for(c = cLevel; c <= cLevelLast; c++) {
1302                        BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
1303                }   }
1304            } else {
1305                for(; cLevel <= cLevelLast; cLevel++) {
1306                    BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
1307            }   }
1308        } else {
1309            for(; cLevel <= cLevelLast; cLevel++) {
1310                BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
1311        }   }
1312
1313#else
1314        (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
1315#endif
1316        goto _end;
1317    }
1318
1319    /* Check if dictionary builder is selected */
1320    if (operation==zom_train) {
1321#ifndef ZSTD_NODICT
1322        ZDICT_params_t zParams;
1323        zParams.compressionLevel = dictCLevel;
1324        zParams.notificationLevel = (unsigned)g_displayLevel;
1325        zParams.dictID = dictID;
1326        if (dict == cover) {
1327            int const optimize = !coverParams.k || !coverParams.d;
1328            coverParams.nbThreads = (unsigned)nbWorkers;
1329            coverParams.zParams = zParams;
1330            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit);
1331        } else if (dict == fastCover) {
1332            int const optimize = !fastCoverParams.k || !fastCoverParams.d;
1333            fastCoverParams.nbThreads = (unsigned)nbWorkers;
1334            fastCoverParams.zParams = zParams;
1335            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
1336        } else {
1337            ZDICT_legacy_params_t dictParams;
1338            memset(&dictParams, 0, sizeof(dictParams));
1339            dictParams.selectivityLevel = dictSelect;
1340            dictParams.zParams = zParams;
1341            operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit);
1342        }
1343#else
1344        (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
1345        DISPLAYLEVEL(1, "training mode not available \n");
1346        operationResult = 1;
1347#endif
1348        goto _end;
1349    }
1350
1351#ifndef ZSTD_NODECOMPRESS
1352    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
1353#endif
1354
1355    /* No input filename ==> use stdin and stdout */
1356    if (filenames->tableSize == 0) UTIL_refFilename(filenames, stdinmark);
1357    if (!strcmp(filenames->fileNames[0], stdinmark) && !outFileName)
1358        outFileName = stdoutmark;  /* when input is stdin, default output is stdout */
1359
1360    /* Check if input/output defined as console; trigger an error in this case */
1361    if (!forceStdin
1362     && !strcmp(filenames->fileNames[0], stdinmark)
1363     && IS_CONSOLE(stdin) ) {
1364        DISPLAYLEVEL(1, "stdin is a console, aborting\n");
1365        CLEAN_RETURN(1);
1366    }
1367    if ( outFileName && !strcmp(outFileName, stdoutmark)
1368      && IS_CONSOLE(stdout)
1369      && !strcmp(filenames->fileNames[0], stdinmark)
1370      && !forceStdout
1371      && operation!=zom_decompress ) {
1372        DISPLAYLEVEL(1, "stdout is a console, aborting\n");
1373        CLEAN_RETURN(1);
1374    }
1375
1376#ifndef ZSTD_NOCOMPRESS
1377    /* check compression level limits */
1378    {   int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
1379        if (cLevel > maxCLevel) {
1380            DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
1381            cLevel = maxCLevel;
1382    }   }
1383#endif
1384
1385    if (showDefaultCParams) {
1386        if (operation == zom_decompress) {
1387            DISPLAY("error : can't use --show-default-cparams in decomrpession mode \n");
1388            CLEAN_RETURN(1);
1389        }
1390    }
1391
1392    if (dictFileName != NULL && patchFromDictFileName != NULL) {
1393        DISPLAY("error : can't use -D and --patch-from=# at the same time \n");
1394        CLEAN_RETURN(1);
1395    }
1396
1397    if (patchFromDictFileName != NULL && filenames->tableSize > 1) {
1398        DISPLAY("error : can't use --patch-from=# on multiple files \n");
1399        CLEAN_RETURN(1);
1400    }
1401
1402    /* No status message in pipe mode (stdin - stdout) */
1403    hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
1404
1405    if ((hasStdout || !IS_CONSOLE(stderr)) && (g_displayLevel==2)) g_displayLevel=1;
1406
1407    /* IO Stream/File */
1408    FIO_setHasStdoutOutput(fCtx, hasStdout);
1409    FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
1410    FIO_determineHasStdinInput(fCtx, filenames);
1411    FIO_setNotificationLevel(g_displayLevel);
1412    FIO_setAllowBlockDevices(prefs, allowBlockDevices);
1413    FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
1414    if (memLimit == 0) {
1415        if (compressionParams.windowLog == 0) {
1416            memLimit = (U32)1 << g_defaultMaxWindowLog;
1417        } else {
1418            memLimit = (U32)1 << (compressionParams.windowLog & 31);
1419    }   }
1420    if (patchFromDictFileName != NULL)
1421        dictFileName = patchFromDictFileName;
1422    FIO_setMemLimit(prefs, memLimit);
1423    if (operation==zom_compress) {
1424#ifndef ZSTD_NOCOMPRESS
1425        FIO_setContentSize(prefs, contentSize);
1426        FIO_setNbWorkers(prefs, nbWorkers);
1427        FIO_setBlockSize(prefs, (int)blockSize);
1428        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
1429        FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
1430        FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
1431        FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
1432        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
1433        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
1434        FIO_setAdaptiveMode(prefs, (unsigned)adapt);
1435        FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
1436        FIO_setAdaptMin(prefs, adaptMin);
1437        FIO_setAdaptMax(prefs, adaptMax);
1438        FIO_setRsyncable(prefs, rsyncable);
1439        FIO_setStreamSrcSize(prefs, streamSrcSize);
1440        FIO_setTargetCBlockSize(prefs, targetCBlockSize);
1441        FIO_setSrcSizeHint(prefs, srcSizeHint);
1442        FIO_setLiteralCompressionMode(prefs, literalCompressionMode);
1443        if (adaptMin > cLevel) cLevel = adaptMin;
1444        if (adaptMax < cLevel) cLevel = adaptMax;
1445
1446        /* Compare strategies constant with the ground truth */
1447        { ZSTD_bounds strategyBounds = ZSTD_cParam_getBounds(ZSTD_c_strategy);
1448          assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound);
1449          (void)strategyBounds; }
1450
1451        if (showDefaultCParams || g_displayLevel >= 4) {
1452            size_t fileNb;
1453            for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) {
1454                if (showDefaultCParams)
1455                    printDefaultCParams(filenames->fileNames[fileNb], dictFileName, cLevel);
1456                if (g_displayLevel >= 4)
1457                    printActualCParams(filenames->fileNames[fileNb], dictFileName, cLevel, &compressionParams);
1458            }
1459        }
1460
1461        if (g_displayLevel >= 4)
1462            FIO_displayCompressionParameters(prefs);
1463        if ((filenames->tableSize==1) && outFileName)
1464            operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
1465        else
1466            operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
1467#else
1468        (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
1469        DISPLAY("Compression not supported \n");
1470#endif
1471    } else {  /* decompression or test */
1472#ifndef ZSTD_NODECOMPRESS
1473        if (filenames->tableSize == 1 && outFileName) {
1474            operationResult = FIO_decompressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName);
1475        } else {
1476            operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
1477        }
1478#else
1479        DISPLAY("Decompression not supported \n");
1480#endif
1481    }
1482
1483_end:
1484    FIO_freePreferences(prefs);
1485    FIO_freeContext(fCtx);
1486    if (main_pause) waitEnter();
1487    UTIL_freeFileNamesTable(filenames);
1488    UTIL_freeFileNamesTable(file_of_names);
1489#ifndef ZSTD_NOTRACE
1490    TRACE_finish();
1491#endif
1492
1493    return operationResult;
1494}
1495