2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
11 #if defined (__cplusplus)
16 /*-****************************************
18 ******************************************/
19 #include "util.h" /* note : ensure that platform.h is included first ! */
20 #include <stdlib.h> /* malloc, realloc, free */
21 #include <stdio.h> /* fprintf */
22 #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
27 # include <sys/utime.h> /* utime */
28 # include <io.h> /* _chmod */
30 # include <unistd.h> /* chown, stat */
31 # if PLATFORM_POSIX_VERSION < 200809L || !defined(st_mtime)
32 # include <utime.h> /* utime */
34 # include <fcntl.h> /* AT_FDCWD */
35 # include <sys/stat.h> /* utimensat */
39 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
40 #include <direct.h> /* needed for _mkdir in windows */
43 #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
44 # include <dirent.h> /* opendir, readdir */
45 # include <string.h> /* strerror, memcpy */
46 #endif /* #ifdef _WIN32 */
48 /*-****************************************
50 ******************************************/
52 /* CONTROL is almost like an assert(), but is never disabled.
53 * It's designed for failures that may happen rarely,
54 * but we don't want to maintain a specific error code path for them,
55 * such as a malloc() returning NULL for example.
56 * Since it's always active, this macro can trigger side effects.
58 #define CONTROL(c) { \
60 UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \
61 __FILE__, __LINE__, #c); \
66 #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
67 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
69 static int g_traceDepth = 0;
70 int g_traceFileStat = 0;
72 #define UTIL_TRACE_CALL(...) \
74 if (g_traceFileStat) { \
75 UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \
76 UTIL_DISPLAY(__VA_ARGS__); \
82 #define UTIL_TRACE_RET(ret) \
84 if (g_traceFileStat) { \
86 UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \
90 /* A modified version of realloc().
91 * If UTIL_realloc() fails the original block is freed.
93 UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
95 void *newptr = realloc(ptr, size);
96 if (newptr) return newptr;
101 #if defined(_MSC_VER)
105 #ifndef ZSTD_HAVE_FCHMOD
106 #if PLATFORM_POSIX_VERSION >= 199309L
107 #define ZSTD_HAVE_FCHMOD
111 #ifndef ZSTD_HAVE_FCHOWN
112 #if PLATFORM_POSIX_VERSION >= 200809L
113 #define ZSTD_HAVE_FCHOWN
117 /*-****************************************
119 ******************************************/
120 int g_utilDisplayLevel;
122 int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
123 const char* acceptableLetters, int hasStdinInput) {
127 UTIL_DISPLAY("stdin is an input - not proceeding.\n");
131 UTIL_DISPLAY("%s", prompt);
134 if (strchr(acceptableLetters, ch) == NULL) {
135 UTIL_DISPLAY("%s \n", abortMsg);
139 while ((ch!=EOF) && (ch!='\n'))
145 /*-*************************************
147 ***************************************/
148 #define LIST_SIZE_INCREASE (8*1024)
149 #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
152 /*-*************************************
154 ***************************************/
156 void UTIL_traceFileStat(void)
161 int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf)
164 UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename);
165 #if defined(_MSC_VER)
167 ret = !_fstat64(fd, statbuf);
169 ret = !_stat64(filename, statbuf);
171 #elif defined(__MINGW32__) && defined (__MSVCRT__)
173 ret = !_fstati64(fd, statbuf);
175 ret = !_stati64(filename, statbuf);
179 ret = !fstat(fd, statbuf);
181 ret = !stat(filename, statbuf);
188 int UTIL_stat(const char* filename, stat_t* statbuf)
190 return UTIL_fstat(-1, filename, statbuf);
193 int UTIL_isRegularFile(const char* infilename)
197 UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename);
198 ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf);
203 int UTIL_isRegularFileStat(const stat_t* statbuf)
205 #if defined(_MSC_VER)
206 return (statbuf->st_mode & S_IFREG) != 0;
208 return S_ISREG(statbuf->st_mode) != 0;
212 /* like chmod, but avoid changing permission of /dev/null */
213 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
215 return UTIL_fchmod(-1, filename, statbuf, permissions);
218 int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions)
221 UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
222 if (statbuf == NULL) {
223 if (!UTIL_fstat(fd, filename, &localStatBuf)) {
227 statbuf = &localStatBuf;
229 if (!UTIL_isRegularFileStat(statbuf)) {
231 return 0; /* pretend success, but don't change anything */
233 #ifdef ZSTD_HAVE_FCHMOD
236 UTIL_TRACE_CALL("fchmod");
237 ret = fchmod(fd, permissions);
245 UTIL_TRACE_CALL("chmod");
246 ret = chmod(filename, permissions);
253 /* set access and modification times */
254 int UTIL_utime(const char* filename, const stat_t *statbuf)
257 UTIL_TRACE_CALL("UTIL_utime(%s)", filename);
258 /* We check that st_mtime is a macro here in order to give us confidence
259 * that struct stat has a struct timespec st_mtim member. We need this
260 * check because there are some platforms that claim to be POSIX 2008
261 * compliant but which do not have st_mtim... */
262 #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime)
265 struct timespec timebuf[2] = { {0, UTIME_NOW} };
266 timebuf[1] = statbuf->st_mtim;
267 ret = utimensat(AT_FDCWD, filename, timebuf, 0);
271 struct utimbuf timebuf;
272 timebuf.actime = time(NULL);
273 timebuf.modtime = statbuf->st_mtime;
274 ret = utime(filename, &timebuf);
282 int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
284 return UTIL_setFDStat(-1, filename, statbuf);
287 int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf)
291 UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename);
293 if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
298 /* Mimic gzip's behavior:
300 * "Change the group first, then the permissions, then the owner.
301 * That way, the permissions will be correct on systems that allow
302 * users to give away files, without introducing a security hole.
303 * Security depends on permissions not containing the setuid or
307 #ifdef ZSTD_HAVE_FCHOWN
309 res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */
313 res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
317 res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
320 #ifdef ZSTD_HAVE_FCHOWN
322 res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */
326 res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
331 UTIL_TRACE_RET(-res);
332 return -res; /* number of errors is returned */
335 int UTIL_isDirectory(const char* infilename)
339 UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename);
340 ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf);
345 int UTIL_isDirectoryStat(const stat_t* statbuf)
348 UTIL_TRACE_CALL("UTIL_isDirectoryStat()");
349 #if defined(_MSC_VER)
350 ret = (statbuf->st_mode & _S_IFDIR) != 0;
352 ret = S_ISDIR(statbuf->st_mode) != 0;
358 int UTIL_compareStr(const void *p1, const void *p2) {
359 return strcmp(* (char * const *) p1, * (char * const *) p2);
362 int UTIL_isSameFile(const char* fName1, const char* fName2)
365 assert(fName1 != NULL); assert(fName2 != NULL);
366 UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2);
367 #if defined(_MSC_VER) || defined(_WIN32)
368 /* note : Visual does not support file identification by inode.
369 * inode does not work on Windows, even with a posix layer, like msys2.
370 * The following work-around is limited to detecting exact name repetition only,
371 * aka `filename` is considered different from `subdir/../filename` */
372 ret = !strcmp(fName1, fName2);
376 ret = UTIL_stat(fName1, &file1Stat)
377 && UTIL_stat(fName2, &file2Stat)
378 && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat);
385 int UTIL_isSameFileStat(
386 const char* fName1, const char* fName2,
387 const stat_t* file1Stat, const stat_t* file2Stat)
390 assert(fName1 != NULL); assert(fName2 != NULL);
391 UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2);
392 #if defined(_MSC_VER) || defined(_WIN32)
393 /* note : Visual does not support file identification by inode.
394 * inode does not work on Windows, even with a posix layer, like msys2.
395 * The following work-around is limited to detecting exact name repetition only,
396 * aka `filename` is considered different from `subdir/../filename` */
399 ret = !strcmp(fName1, fName2);
402 ret = (file1Stat->st_dev == file2Stat->st_dev)
403 && (file1Stat->st_ino == file2Stat->st_ino);
410 /* UTIL_isFIFO : distinguish named pipes */
411 int UTIL_isFIFO(const char* infilename)
413 UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename);
414 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
415 #if PLATFORM_POSIX_VERSION >= 200112L
418 if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) {
429 /* UTIL_isFIFO : distinguish named pipes */
430 int UTIL_isFIFOStat(const stat_t* statbuf)
432 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
433 #if PLATFORM_POSIX_VERSION >= 200112L
434 if (S_ISFIFO(statbuf->st_mode)) return 1;
440 /* UTIL_isBlockDevStat : distinguish named pipes */
441 int UTIL_isBlockDevStat(const stat_t* statbuf)
443 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
444 #if PLATFORM_POSIX_VERSION >= 200112L
445 if (S_ISBLK(statbuf->st_mode)) return 1;
451 int UTIL_isLink(const char* infilename)
453 UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename);
454 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
455 #if PLATFORM_POSIX_VERSION >= 200112L
458 int const r = lstat(infilename, &statbuf);
459 if (!r && S_ISLNK(statbuf.st_mode)) {
470 static int g_fakeStdinIsConsole = 0;
471 static int g_fakeStderrIsConsole = 0;
472 static int g_fakeStdoutIsConsole = 0;
474 int UTIL_isConsole(FILE* file)
477 UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file));
478 if (file == stdin && g_fakeStdinIsConsole)
480 else if (file == stderr && g_fakeStderrIsConsole)
482 else if (file == stdout && g_fakeStdoutIsConsole)
485 ret = IS_CONSOLE(file);
490 void UTIL_fakeStdinIsConsole(void)
492 g_fakeStdinIsConsole = 1;
494 void UTIL_fakeStdoutIsConsole(void)
496 g_fakeStdoutIsConsole = 1;
498 void UTIL_fakeStderrIsConsole(void)
500 g_fakeStderrIsConsole = 1;
503 U64 UTIL_getFileSize(const char* infilename)
506 UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename);
507 if (!UTIL_stat(infilename, &statbuf)) {
509 return UTIL_FILESIZE_UNKNOWN;
512 U64 const size = UTIL_getFileSizeStat(&statbuf);
513 UTIL_TRACE_RET((int)size);
518 U64 UTIL_getFileSizeStat(const stat_t* statbuf)
520 if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN;
521 #if defined(_MSC_VER)
522 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
523 #elif defined(__MINGW32__) && defined (__MSVCRT__)
524 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
526 if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN;
528 return (U64)statbuf->st_size;
531 UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size)
533 UTIL_HumanReadableSize_t hrs;
535 if (g_utilDisplayLevel > 3) {
536 /* In verbose mode, do not scale sizes down, except in the case of
537 * values that exceed the integral precision of a double. */
538 if (size >= (1ull << 53)) {
539 hrs.value = (double)size / (1ull << 20);
541 /* At worst, a double representation of a maximal size will be
542 * accurate to better than tens of kilobytes. */
545 hrs.value = (double)size;
550 /* In regular mode, scale sizes down and use suffixes. */
551 if (size >= (1ull << 60)) {
552 hrs.value = (double)size / (1ull << 60);
554 } else if (size >= (1ull << 50)) {
555 hrs.value = (double)size / (1ull << 50);
557 } else if (size >= (1ull << 40)) {
558 hrs.value = (double)size / (1ull << 40);
560 } else if (size >= (1ull << 30)) {
561 hrs.value = (double)size / (1ull << 30);
563 } else if (size >= (1ull << 20)) {
564 hrs.value = (double)size / (1ull << 20);
566 } else if (size >= (1ull << 10)) {
567 hrs.value = (double)size / (1ull << 10);
570 hrs.value = (double)size;
574 if (hrs.value >= 100 || (U64)hrs.value == size) {
576 } else if (hrs.value >= 10) {
578 } else if (hrs.value > 1) {
588 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
592 UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles);
593 for (n=0; n<nbFiles; n++) {
594 U64 const size = UTIL_getFileSize(fileNamesTable[n]);
595 if (size == UTIL_FILESIZE_UNKNOWN) {
597 return UTIL_FILESIZE_UNKNOWN;
601 UTIL_TRACE_RET((int)total);
606 /* condition : @file must be valid, and not have reached its end.
607 * @return : length of line written into @buf, ended with `\0` instead of '\n',
608 * or 0, if there is no new line */
609 static size_t readLineFromFile(char* buf, size_t len, FILE* file)
612 if ( fgets(buf, (int) len, file) == NULL ) return 0;
613 { size_t linelen = strlen(buf);
614 if (strlen(buf)==0) return 0;
615 if (buf[linelen-1] == '\n') linelen--;
622 * size of @inputFileName file must be < @dstCapacity
623 * @dst must be initialized
624 * @return : nb of lines
625 * or -1 if there's an error
628 readLinesFromFile(void* dst, size_t dstCapacity,
629 const char* inputFileName)
633 char* const buf = (char*)dst;
634 FILE* const inputFile = fopen(inputFileName, "r");
639 if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
643 while ( !feof(inputFile) ) {
644 size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
645 if (lineLength == 0) break;
646 assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */
651 CONTROL( fclose(inputFile) == 0 );
656 /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
658 UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
666 if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf))
669 { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf);
670 if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
672 bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */
675 buf = (char*) malloc(bufSize);
676 CONTROL( buf != NULL );
678 { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);
680 if (ret_nbFiles <= 0) {
684 nbFiles = (size_t)ret_nbFiles;
687 { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
688 CONTROL(filenamesTable != NULL);
691 for (fnb = 0, pos = 0; fnb < nbFiles; fnb++) {
692 filenamesTable[fnb] = buf+pos;
693 pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */
695 assert(pos <= bufSize);
697 return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf);
701 static FileNamesTable*
702 UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf)
704 FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table));
705 CONTROL(table != NULL);
706 table->fileNames = filenames;
708 table->tableSize = tableSize;
709 table->tableCapacity = tableCapacity;
714 UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf)
716 return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf);
719 void UTIL_freeFileNamesTable(FileNamesTable* table)
721 if (table==NULL) return;
722 free((void*)table->fileNames);
727 FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize)
729 const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable));
731 if (fnTable==NULL) return NULL;
732 fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL);
733 fnt->tableSize = 0; /* the table is empty */
737 int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) {
739 for(i=0 ;i < table->tableSize; i++) {
740 if(!strcmp(table->fileNames[i], name)) {
747 void UTIL_refFilename(FileNamesTable* fnt, const char* filename)
749 assert(fnt->tableSize < fnt->tableCapacity);
750 fnt->fileNames[fnt->tableSize] = filename;
754 static size_t getTotalTableSize(FileNamesTable* table)
756 size_t fnb = 0, totalSize = 0;
757 for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) {
758 totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */
764 UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
766 unsigned newTableIdx = 0;
768 size_t newTotalTableSize;
771 FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL);
772 CONTROL( newTable != NULL );
774 newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2);
776 buf = (char*) calloc(newTotalTableSize, sizeof(*buf));
777 CONTROL ( buf != NULL );
780 newTable->tableSize = table1->tableSize + table2->tableSize;
781 newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames)));
782 CONTROL ( newTable->fileNames != NULL );
785 for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) {
786 size_t const curLen = strlen(table1->fileNames[idx1]);
787 memcpy(buf+pos, table1->fileNames[idx1], curLen);
788 assert(newTableIdx <= newTable->tableSize);
789 newTable->fileNames[newTableIdx] = buf+pos;
794 for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
795 size_t const curLen = strlen(table2->fileNames[idx2]);
796 memcpy(buf+pos, table2->fileNames[idx2], curLen);
797 assert(newTableIdx < newTable->tableSize);
798 newTable->fileNames[newTableIdx] = buf+pos;
801 assert(pos <= newTotalTableSize);
802 newTable->tableSize = newTableIdx;
804 UTIL_freeFileNamesTable(table1);
805 UTIL_freeFileNamesTable(table2);
811 static int UTIL_prepareFileList(const char* dirName,
812 char** bufStart, size_t* pos,
813 char** bufEnd, int followLinks)
816 size_t dirLength, pathLength;
818 WIN32_FIND_DATAA cFile;
821 dirLength = strlen(dirName);
822 path = (char*) malloc(dirLength + 3);
825 memcpy(path, dirName, dirLength);
826 path[dirLength] = '\\';
827 path[dirLength+1] = '*';
828 path[dirLength+2] = 0;
830 hFile=FindFirstFileA(path, &cFile);
831 if (hFile == INVALID_HANDLE_VALUE) {
832 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
838 size_t const fnameLength = strlen(cFile.cFileName);
839 path = (char*) malloc(dirLength + fnameLength + 2);
840 if (!path) { FindClose(hFile); return 0; }
841 memcpy(path, dirName, dirLength);
842 path[dirLength] = '\\';
843 memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
844 pathLength = dirLength+1+fnameLength;
845 path[pathLength] = 0;
846 if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
847 if ( strcmp (cFile.cFileName, "..") == 0
848 || strcmp (cFile.cFileName, ".") == 0 )
850 /* Recursively call "UTIL_prepareFileList" with the new path. */
851 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks);
852 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
853 } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL)
854 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE)
855 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) {
856 if (*bufStart + *pos + pathLength >= *bufEnd) {
857 ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
858 *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
859 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
860 *bufEnd = *bufStart + newListSize;
862 if (*bufStart + *pos + pathLength < *bufEnd) {
863 memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
864 *pos += pathLength + 1;
868 } while (FindNextFileA(hFile, &cFile));
874 #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
876 static int UTIL_prepareFileList(const char *dirName,
877 char** bufStart, size_t* pos,
878 char** bufEnd, int followLinks)
881 struct dirent * entry;
885 if (!(dir = opendir(dirName))) {
886 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
890 dirLength = strlen(dirName);
892 while ((entry = readdir(dir)) != NULL) {
894 size_t fnameLength, pathLength;
895 if (strcmp (entry->d_name, "..") == 0 ||
896 strcmp (entry->d_name, ".") == 0) continue;
897 fnameLength = strlen(entry->d_name);
898 path = (char*) malloc(dirLength + fnameLength + 2);
899 if (!path) { closedir(dir); return 0; }
900 memcpy(path, dirName, dirLength);
902 path[dirLength] = '/';
903 memcpy(path+dirLength+1, entry->d_name, fnameLength);
904 pathLength = dirLength+1+fnameLength;
905 path[pathLength] = 0;
907 if (!followLinks && UTIL_isLink(path)) {
908 UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
913 if (UTIL_isDirectory(path)) {
914 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
915 if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
917 if (*bufStart + *pos + pathLength >= *bufEnd) {
918 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
919 assert(newListSize >= 0);
920 *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
921 if (*bufStart != NULL) {
922 *bufEnd = *bufStart + newListSize;
924 free(path); closedir(dir); return 0;
927 if (*bufStart + *pos + pathLength < *bufEnd) {
928 memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
929 *pos += pathLength + 1;
933 errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
937 UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno));
947 static int UTIL_prepareFileList(const char *dirName,
948 char** bufStart, size_t* pos,
949 char** bufEnd, int followLinks)
951 (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
952 UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName);
956 #endif /* #ifdef _WIN32 */
958 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
960 const char* ext = UTIL_getFileExtension(inputName);
961 while(*extensionList!=NULL)
963 const int isCompressedExtension = strcmp(ext,*extensionList);
964 if(isCompressedExtension==0)
971 /*Utility function to get file extension from file */
972 const char* UTIL_getFileExtension(const char* infilename)
974 const char* extension = strrchr(infilename, '.');
975 if(!extension || extension==infilename) return "";
979 static int pathnameHas2Dots(const char *pathname)
981 /* We need to figure out whether any ".." present in the path is a whole
982 * path token, which is the case if it is bordered on both sides by either
983 * the beginning/end of the path or by a directory separator.
985 const char *needle = pathname;
987 needle = strstr(needle, "..");
989 if (needle == NULL) {
993 if ((needle == pathname || needle[-1] == PATH_SEP)
994 && (needle[2] == '\0' || needle[2] == PATH_SEP)) {
998 /* increment so we search for the next match */
1004 static int isFileNameValidForMirroredOutput(const char *filename)
1006 return !pathnameHas2Dots(filename);
1010 #define DIR_DEFAULT_MODE 0755
1011 static mode_t getDirMode(const char *dirName)
1014 if (!UTIL_stat(dirName, &st)) {
1015 UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno));
1016 return DIR_DEFAULT_MODE;
1018 if (!UTIL_isDirectoryStat(&st)) {
1019 UTIL_DISPLAY("zstd: expected directory: %s\n", dirName);
1020 return DIR_DEFAULT_MODE;
1025 static int makeDir(const char *dir, mode_t mode)
1027 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
1028 int ret = _mkdir(dir);
1031 int ret = mkdir(dir, mode);
1034 if (errno == EEXIST)
1036 UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno));
1041 /* this function requires a mutable input string */
1042 static void convertPathnameToDirName(char *pathname)
1046 /* get dir name from pathname similar to 'dirname()' */
1047 assert(pathname != NULL);
1049 /* remove trailing '/' chars */
1050 len = strlen(pathname);
1052 while (pathname[len] == PATH_SEP) {
1053 pathname[len] = '\0';
1056 if (len == 0) return;
1058 /* if input is a single file, return '.' instead. i.e.
1059 * "xyz/abc/file.txt" => "xyz/abc"
1063 pos = strrchr(pathname, PATH_SEP);
1072 /* pathname must be valid */
1073 static const char* trimLeadingRootChar(const char *pathname)
1075 assert(pathname != NULL);
1076 if (pathname[0] == PATH_SEP)
1077 return pathname + 1;
1081 /* pathname must be valid */
1082 static const char* trimLeadingCurrentDirConst(const char *pathname)
1084 assert(pathname != NULL);
1085 if ((pathname[0] == '.') && (pathname[1] == PATH_SEP))
1086 return pathname + 2;
1091 trimLeadingCurrentDir(char *pathname)
1093 /* 'union charunion' can do const-cast without compiler warning */
1098 ptr.cchr = trimLeadingCurrentDirConst(pathname);
1102 /* remove leading './' or '/' chars here */
1103 static const char * trimPath(const char *pathname)
1105 return trimLeadingRootChar(
1106 trimLeadingCurrentDirConst(pathname));
1109 static char* mallocAndJoin2Dir(const char *dir1, const char *dir2)
1111 assert(dir1 != NULL && dir2 != NULL);
1112 { const size_t dir1Size = strlen(dir1);
1113 const size_t dir2Size = strlen(dir2);
1114 char *outDirBuffer, *buffer;
1116 outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2);
1117 CONTROL(outDirBuffer != NULL);
1119 memcpy(outDirBuffer, dir1, dir1Size);
1120 outDirBuffer[dir1Size] = '\0';
1123 return outDirBuffer;
1125 buffer = outDirBuffer + dir1Size;
1126 if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) {
1130 memcpy(buffer, dir2, dir2Size);
1131 buffer[dir2Size] = '\0';
1133 return outDirBuffer;
1137 /* this function will return NULL if input srcFileName is not valid name for mirrored output path */
1138 char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName)
1140 char* pathname = NULL;
1141 if (!isFileNameValidForMirroredOutput(srcFileName))
1144 pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName));
1146 convertPathnameToDirName(pathname);
1151 mirrorSrcDir(char* srcDirName, const char* outDirName)
1155 char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName));
1159 srcMode = getDirMode(srcDirName);
1160 status = makeDir(newDir, srcMode);
1166 mirrorSrcDirRecursive(char* srcDirName, const char* outDirName)
1169 char* pp = trimLeadingCurrentDir(srcDirName);
1172 while ((sp = strchr(pp, PATH_SEP)) != NULL) {
1175 status = mirrorSrcDir(srcDirName, outDirName);
1182 status = mirrorSrcDir(srcDirName, outDirName);
1187 makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName)
1190 for (i = 0; i < nbFile; i++)
1191 mirrorSrcDirRecursive(srcDirNames[i], outDirName);
1195 firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir)
1197 size_t firstDirLen = strlen(firstDir),
1198 secondDirLen = strlen(secondDir);
1199 return firstDirLen <= secondDirLen &&
1200 (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') &&
1201 0 == strncmp(firstDir, secondDir, firstDirLen);
1204 static int compareDir(const void* pathname1, const void* pathname2) {
1205 /* sort it after remove the leading '/' or './'*/
1206 const char* s1 = trimPath(*(char * const *) pathname1);
1207 const char* s2 = trimPath(*(char * const *) pathname2);
1208 return strcmp(s1, s2);
1212 makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName)
1214 unsigned int i = 0, uniqueDirNr = 0;
1215 char** uniqueDirNames = NULL;
1220 uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *));
1221 CONTROL(uniqueDirNames != NULL);
1223 /* if dirs is "a/b/c" and "a/b/c/d", we only need call:
1224 * we just need "a/b/c/d" */
1225 qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir);
1228 uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0];
1229 for (i = 1; i < nbFile; i++) {
1230 char* prevDirName = srcDirNames[i - 1];
1231 char* currDirName = srcDirNames[i];
1233 /* note: we always compare trimmed path, i.e.:
1234 * src dir of "./foo" and "/foo" will be both saved into:
1235 * "outDirName/foo/" */
1236 if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName),
1237 trimPath(currDirName)))
1240 /* we need to maintain original src dir name instead of trimmed
1241 * dir, so we can retrieve the original src dir's mode_t */
1242 uniqueDirNames[uniqueDirNr - 1] = currDirName;
1245 makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName);
1247 free(uniqueDirNames);
1251 makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName)
1254 for (i = 0; i < nbFile; ++i)
1255 convertPathnameToDirName(srcFileNames[i]);
1256 makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName);
1259 void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName)
1261 unsigned int i = 0, validFilenamesNr = 0;
1262 char** srcFileNames = (char **) malloc(nbFile * sizeof (char *));
1263 CONTROL(srcFileNames != NULL);
1265 /* check input filenames is valid */
1266 for (i = 0; i < nbFile; ++i) {
1267 if (isFileNameValidForMirroredOutput(inFileNames[i])) {
1268 char* fname = STRDUP(inFileNames[i]);
1269 CONTROL(fname != NULL);
1270 srcFileNames[validFilenamesNr++] = fname;
1274 if (validFilenamesNr > 0) {
1275 makeDir(outDirName, DIR_DEFAULT_MODE);
1276 makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName);
1279 for (i = 0; i < validFilenamesNr; i++)
1280 free(srcFileNames[i]);
1285 UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks)
1288 char* buf = (char*)malloc(LIST_SIZE_INCREASE);
1289 char* bufend = buf + LIST_SIZE_INCREASE;
1291 if (!buf) return NULL;
1293 { size_t ifnNb, pos;
1294 for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) {
1295 if (!UTIL_isDirectory(inputNames[ifnNb])) {
1296 size_t const len = strlen(inputNames[ifnNb]);
1297 if (buf + pos + len >= bufend) {
1298 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
1299 assert(newListSize >= 0);
1300 buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
1301 if (!buf) return NULL;
1302 bufend = buf + newListSize;
1304 if (buf + pos + len < bufend) {
1305 memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */
1310 nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks);
1311 if (buf == NULL) return NULL;
1314 /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */
1316 { size_t ifnNb, pos;
1317 size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */
1318 const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable));
1319 if (!fileNamesTable) { free(buf); return NULL; }
1321 for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) {
1322 fileNamesTable[ifnNb] = buf + pos;
1323 if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; }
1324 pos += strlen(fileNamesTable[ifnNb]) + 1;
1326 return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf);
1331 void UTIL_expandFNT(FileNamesTable** fnt, int followLinks)
1333 FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks);
1334 CONTROL(newFNT != NULL);
1335 UTIL_freeFileNamesTable(*fnt);
1339 FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames)
1341 size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames);
1342 const char** const newFNTable = (const char**)malloc(sizeof_FNTable);
1343 if (newFNTable==NULL) return NULL;
1344 memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */
1345 return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL);
1349 /*-****************************************
1350 * count the number of cores
1351 ******************************************/
1353 #if defined(_WIN32) || defined(WIN32)
1355 #include <windows.h>
1357 typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
1359 DWORD CountSetBits(ULONG_PTR bitMask)
1361 DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1;
1362 DWORD bitSetCount = 0;
1363 ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT;
1366 for (i = 0; i <= LSHIFT; ++i)
1368 bitSetCount += ((bitMask & bitTest)?1:0);
1375 int UTIL_countCores(int logical)
1377 static int numCores = 0;
1378 if (numCores != 0) return numCores;
1382 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
1383 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
1384 DWORD returnLength = 0;
1385 size_t byteOffset = 0;
1387 #if defined(_MSC_VER)
1388 /* Visual Studio does not like the following cast */
1389 # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
1390 # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
1392 glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
1393 "GetLogicalProcessorInformation");
1400 DWORD rc = glpi(buffer, &returnLength);
1402 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
1405 buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
1407 if (buffer == NULL) {
1412 /* some other error */
1421 while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
1423 if (ptr->Relationship == RelationProcessorCore) {
1425 numCores += CountSetBits(ptr->ProcessorMask);
1431 byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
1440 /* try to fall back on GetSystemInfo */
1441 { SYSTEM_INFO sysinfo;
1442 GetSystemInfo(&sysinfo);
1443 numCores = sysinfo.dwNumberOfProcessors;
1444 if (numCores == 0) numCores = 1; /* just in case */
1449 #elif defined(__APPLE__)
1451 #include <sys/sysctl.h>
1453 /* Use apple-provided syscall
1454 * see: man 3 sysctl */
1455 int UTIL_countCores(int logical)
1457 static S32 numCores = 0; /* apple specifies int32_t */
1458 if (numCores != 0) return numCores;
1460 { size_t size = sizeof(S32);
1461 int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0);
1463 if (errno == ENOENT) {
1464 /* entry not present, fall back on 1 */
1467 perror("zstd: can't get number of cpus");
1476 #elif defined(__linux__)
1478 /* parse /proc/cpuinfo
1479 * siblings / cpu cores should give hyperthreading ratio
1480 * otherwise fall back on sysconf */
1481 int UTIL_countCores(int logical)
1483 static int numCores = 0;
1485 if (numCores != 0) return numCores;
1487 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
1488 if (numCores == -1) {
1489 /* value not queryable, fall back on 1 */
1490 return numCores = 1;
1493 /* try to determine if there's hyperthreading */
1494 { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
1496 char buff[BUF_SIZE];
1502 if (cpuinfo == NULL) {
1503 /* fall back on the sysconf value */
1507 /* assume the cpu cores/siblings values will be constant across all
1508 * present processors */
1509 while (!feof(cpuinfo)) {
1510 if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
1511 if (strncmp(buff, "siblings", 8) == 0) {
1512 const char* const sep = strchr(buff, ':');
1513 if (sep == NULL || *sep == '\0') {
1514 /* formatting was broken? */
1518 siblings = atoi(sep + 1);
1520 if (strncmp(buff, "cpu cores", 9) == 0) {
1521 const char* const sep = strchr(buff, ':');
1522 if (sep == NULL || *sep == '\0') {
1523 /* formatting was broken? */
1527 cpu_cores = atoi(sep + 1);
1529 } else if (ferror(cpuinfo)) {
1530 /* fall back on the sysconf value */
1533 if (siblings && cpu_cores && siblings > cpu_cores) {
1534 ratio = siblings / cpu_cores;
1537 if (ratio && numCores > ratio && !logical) {
1538 numCores = numCores / ratio;
1547 #elif defined(__FreeBSD__)
1549 #include <sys/param.h>
1550 #include <sys/sysctl.h>
1552 /* Use physical core sysctl when available
1553 * see: man 4 smp, man 3 sysctl */
1554 int UTIL_countCores(int logical)
1556 static int numCores = 0; /* freebsd sysctl is native int sized */
1557 #if __FreeBSD_version >= 1300008
1558 static int perCore = 1;
1560 if (numCores != 0) return numCores;
1562 #if __FreeBSD_version >= 1300008
1563 { size_t size = sizeof(numCores);
1564 int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0);
1567 ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0);
1568 /* default to physical cores if logical cannot be read */
1570 numCores *= perCore;
1575 if (errno != ENOENT) {
1576 perror("zstd: can't get number of cpus");
1579 /* sysctl not present, fall through to older sysconf method */
1582 /* suppress unused parameter warning */
1586 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
1587 if (numCores == -1) {
1588 /* value not queryable, fall back on 1 */
1594 #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__)
1596 /* Use POSIX sysconf
1597 * see: man 3 sysconf */
1598 int UTIL_countCores(int logical)
1600 static int numCores = 0;
1602 /* suppress unused parameter warning */
1605 if (numCores != 0) return numCores;
1607 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
1608 if (numCores == -1) {
1609 /* value not queryable, fall back on 1 */
1610 return numCores = 1;
1617 int UTIL_countCores(int logical)
1619 /* suppress unused parameter warning */
1628 int UTIL_countPhysicalCores(void)
1630 return UTIL_countCores(0);
1633 int UTIL_countLogicalCores(void)
1635 return UTIL_countCores(1);
1638 #if defined (__cplusplus)