2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
11 #include "utils/ScopeGuard.h"
25 unsigned defaultNumThreads() {
26 #ifdef PZSTD_NUM_THREADS
27 return PZSTD_NUM_THREADS;
29 return std::thread::hardware_concurrency();
33 unsigned parseUnsigned(const char **arg) {
35 while (**arg >= '0' && **arg <= '9') {
37 result += **arg - '0';
43 const char *getArgument(const char *options, const char **argv, int &i,
45 if (options[1] != 0) {
50 std::fprintf(stderr, "Option -%c requires an argument, but none provided\n",
57 const std::string kZstdExtension = ".zst";
58 constexpr char kStdIn[] = "-";
59 constexpr char kStdOut[] = "-";
60 constexpr unsigned kDefaultCompressionLevel = 3;
61 constexpr unsigned kMaxNonUltraCompressionLevel = 19;
64 const char nullOutput[] = "nul";
66 const char nullOutput[] = "/dev/null";
69 void notSupported(const char *option) {
70 std::fprintf(stderr, "Operation not supported: %s\n", option);
74 std::fprintf(stderr, "Usage:\n");
75 std::fprintf(stderr, " pzstd [args] [FILE(s)]\n");
76 std::fprintf(stderr, "Parallel ZSTD options:\n");
77 std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:<numcpus>)\n");
79 std::fprintf(stderr, "ZSTD options:\n");
80 std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
81 std::fprintf(stderr, " -d, --decompress : decompression\n");
82 std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n");
83 std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n");
84 std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n");
85 std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n");
86 std::fprintf(stderr, " -h, --help : display help and exit\n");
87 std::fprintf(stderr, " -V, --version : display version number and exit\n");
88 std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n");
89 std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n");
90 std::fprintf(stderr, " -c, --stdout : write to standard output (even if it is the console)\n");
91 #ifdef UTIL_HAS_CREATEFILELIST
92 std::fprintf(stderr, " -r : operate recursively on directories\n");
94 std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel());
95 std::fprintf(stderr, " -C, --check : integrity check (default)\n");
96 std::fprintf(stderr, " --no-check : no integrity check\n");
97 std::fprintf(stderr, " -t, --test : test compressed file integrity\n");
98 std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n");
100 } // anonymous namespace
103 : numThreads(defaultNumThreads()), maxWindowLog(23),
104 compressionLevel(kDefaultCompressionLevel), decompress(false),
105 overwrite(false), keepSource(true), writeMode(WriteMode::Auto),
106 checksum(true), verbosity(2) {}
108 Options::Status Options::parse(int argc, const char **argv) {
110 bool recursive = false;
112 bool forceStdout = false;
113 bool followLinks = false;
114 // Local copy of input files, which are pointers into argv.
115 std::vector<const char *> localInputFiles;
116 for (int i = 1; i < argc; ++i) {
117 const char *arg = argv[i];
118 // Protect against empty arguments
122 // Everything after "--" is an input file
123 if (!std::strcmp(arg, "--")) {
125 std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles));
128 // Long arguments that don't have a short option
130 bool isLongOption = true;
131 if (!std::strcmp(arg, "--rm")) {
133 } else if (!std::strcmp(arg, "--ultra")) {
136 } else if (!std::strcmp(arg, "--no-check")) {
138 } else if (!std::strcmp(arg, "--sparse")) {
139 writeMode = WriteMode::Sparse;
140 notSupported("Sparse mode");
141 return Status::Failure;
142 } else if (!std::strcmp(arg, "--no-sparse")) {
143 writeMode = WriteMode::Regular;
144 notSupported("Sparse mode");
145 return Status::Failure;
146 } else if (!std::strcmp(arg, "--dictID")) {
148 return Status::Failure;
149 } else if (!std::strcmp(arg, "--no-dictID")) {
151 return Status::Failure;
153 isLongOption = false;
159 // Arguments with a short option simply set their short option.
160 const char *options = nullptr;
161 if (!std::strcmp(arg, "--processes")) {
163 } else if (!std::strcmp(arg, "--version")) {
165 } else if (!std::strcmp(arg, "--help")) {
167 } else if (!std::strcmp(arg, "--decompress")) {
169 } else if (!std::strcmp(arg, "--force")) {
171 } else if (!std::strcmp(arg, "--stdout")) {
173 } else if (!std::strcmp(arg, "--keep")) {
175 } else if (!std::strcmp(arg, "--verbose")) {
177 } else if (!std::strcmp(arg, "--quiet")) {
179 } else if (!std::strcmp(arg, "--check")) {
181 } else if (!std::strcmp(arg, "--test")) {
183 } else if (arg[0] == '-' && arg[1] != 0) {
186 localInputFiles.emplace_back(arg);
189 assert(options != nullptr);
191 bool finished = false;
192 while (!finished && *options != 0) {
193 // Parse the compression level
194 if (*options >= '0' && *options <= '9') {
195 compressionLevel = parseUnsigned(&options);
203 return Status::Message;
205 std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING);
206 return Status::Message;
209 const char *optionArgument = getArgument(options, argv, i, argc);
210 if (optionArgument == nullptr) {
211 return Status::Failure;
213 if (*optionArgument < '0' || *optionArgument > '9') {
214 std::fprintf(stderr, "Option -p expects a number, but %s provided\n",
216 return Status::Failure;
218 numThreads = parseUnsigned(&optionArgument);
219 if (*optionArgument != 0) {
221 "Option -p expects a number, but %u%s provided\n",
222 numThreads, optionArgument);
223 return Status::Failure;
229 const char *optionArgument = getArgument(options, argv, i, argc);
230 if (optionArgument == nullptr) {
231 return Status::Failure;
233 outputFile = optionArgument;
254 #ifdef UTIL_HAS_CREATEFILELIST
260 outputFile = kStdOut;
268 // Ignore them for now
270 // Unsupported options from Zstd
273 notSupported("Zstd dictionaries.");
274 return Status::Failure;
279 notSupported("Zstd benchmarking options.");
280 return Status::Failure;
282 std::fprintf(stderr, "Invalid argument: %s\n", arg);
283 return Status::Failure;
288 } // while (*options != 0);
289 } // for (int i = 1; i < argc; ++i);
291 // Set options for test mode
293 outputFile = nullOutput;
297 // Input file defaults to standard input if not provided.
298 if (localInputFiles.empty()) {
299 localInputFiles.emplace_back(kStdIn);
302 // Check validity of input files
303 if (localInputFiles.size() > 1) {
304 const auto it = std::find(localInputFiles.begin(), localInputFiles.end(),
305 std::string{kStdIn});
306 if (it != localInputFiles.end()) {
309 "Cannot specify standard input when handling multiple files\n");
310 return Status::Failure;
313 if (localInputFiles.size() > 1 || recursive) {
314 if (!outputFile.empty() && outputFile != nullOutput) {
317 "Cannot specify an output file when handling multiple inputs\n");
318 return Status::Failure;
322 g_utilDisplayLevel = verbosity;
323 // Remove local input files that are symbolic links
325 std::remove_if(localInputFiles.begin(), localInputFiles.end(),
326 [&](const char *path) {
327 bool isLink = UTIL_isLink(path);
328 if (isLink && verbosity >= 2) {
331 "Warning : %s is symbolic link, ignoring\n",
338 // Translate input files/directories into files to (de)compress
340 FileNamesTable* const files = UTIL_createExpandedFNT(localInputFiles.data(), localInputFiles.size(), followLinks);
341 if (files == nullptr) {
342 std::fprintf(stderr, "Error traversing directories\n");
343 return Status::Failure;
346 makeScopeGuard([&] { UTIL_freeFileNamesTable(files); });
347 if (files->tableSize == 0) {
348 std::fprintf(stderr, "No files found\n");
349 return Status::Failure;
351 inputFiles.resize(files->tableSize);
352 std::copy(files->fileNames, files->fileNames + files->tableSize, inputFiles.begin());
354 inputFiles.resize(localInputFiles.size());
355 std::copy(localInputFiles.begin(), localInputFiles.end(),
358 localInputFiles.clear();
359 assert(!inputFiles.empty());
361 // If reading from standard input, default to standard output
362 if (inputFiles[0] == kStdIn && outputFile.empty()) {
363 assert(inputFiles.size() == 1);
367 if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) {
368 assert(inputFiles.size() == 1);
369 std::fprintf(stderr, "Cannot read input from interactive console\n");
370 return Status::Failure;
372 if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) {
373 std::fprintf(stderr, "Will not write to console stdout unless -c or -f is "
374 "specified and decompressing\n");
375 return Status::Failure;
378 // Check compression level
381 ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel;
382 if (compressionLevel > maxCLevel || compressionLevel == 0) {
383 std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel);
384 return Status::Failure;
388 // Check that numThreads is set
389 if (numThreads == 0) {
390 std::fprintf(stderr, "Invalid arguments: # of threads not specified "
391 "and unable to determine hardware concurrency.\n");
392 return Status::Failure;
396 // If we are piping input and output, turn off interaction
397 if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) {
400 // If we are in multi-file mode, turn off interaction
401 if (inputFiles.size() > 1 && verbosity == 2) {
405 return Status::Success;
408 std::string Options::getOutputFile(const std::string &inputFile) const {
409 if (!outputFile.empty()) {
412 // Attempt to add/remove zstd extension from the input file
414 int stemSize = inputFile.size() - kZstdExtension.size();
415 if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) {
416 return inputFile.substr(0, stemSize);
421 return inputFile + kZstdExtension;