648db22b |
1 | /* |
2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | * You may select, at your option, one of the above-listed licenses. |
9 | */ |
10 | |
11 | #if defined (__cplusplus) |
12 | extern "C" { |
13 | #endif |
14 | |
15 | |
16 | /*-**************************************** |
17 | * Dependencies |
18 | ******************************************/ |
19 | #include "util.h" /* note : ensure that platform.h is included first ! */ |
20 | #include <stdlib.h> /* malloc, realloc, free */ |
21 | #include <stdio.h> /* fprintf */ |
22 | #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ |
23 | #include <errno.h> |
24 | #include <assert.h> |
25 | |
f535537f |
26 | #if defined(__FreeBSD__) |
27 | #include <sys/param.h> /* __FreeBSD_version */ |
28 | #endif /* #ifdef __FreeBSD__ */ |
29 | |
648db22b |
30 | #if defined(_WIN32) |
31 | # include <sys/utime.h> /* utime */ |
32 | # include <io.h> /* _chmod */ |
f535537f |
33 | # define ZSTD_USE_UTIMENSAT 0 |
648db22b |
34 | #else |
35 | # include <unistd.h> /* chown, stat */ |
f535537f |
36 | # include <sys/stat.h> /* utimensat, st_mtime */ |
37 | # if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \ |
38 | || (defined(__FreeBSD__) && __FreeBSD_version >= 1100056) |
39 | # define ZSTD_USE_UTIMENSAT 1 |
648db22b |
40 | # else |
f535537f |
41 | # define ZSTD_USE_UTIMENSAT 0 |
42 | # endif |
43 | # if ZSTD_USE_UTIMENSAT |
648db22b |
44 | # include <fcntl.h> /* AT_FDCWD */ |
f535537f |
45 | # else |
46 | # include <utime.h> /* utime */ |
648db22b |
47 | # endif |
48 | #endif |
49 | |
50 | #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) |
51 | #include <direct.h> /* needed for _mkdir in windows */ |
52 | #endif |
53 | |
54 | #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ |
55 | # include <dirent.h> /* opendir, readdir */ |
56 | # include <string.h> /* strerror, memcpy */ |
57 | #endif /* #ifdef _WIN32 */ |
58 | |
59 | /*-**************************************** |
60 | * Internal Macros |
61 | ******************************************/ |
62 | |
63 | /* CONTROL is almost like an assert(), but is never disabled. |
64 | * It's designed for failures that may happen rarely, |
65 | * but we don't want to maintain a specific error code path for them, |
66 | * such as a malloc() returning NULL for example. |
67 | * Since it's always active, this macro can trigger side effects. |
68 | */ |
69 | #define CONTROL(c) { \ |
70 | if (!(c)) { \ |
71 | UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ |
72 | __FILE__, __LINE__, #c); \ |
73 | exit(1); \ |
74 | } } |
75 | |
76 | /* console log */ |
77 | #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
78 | #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } |
79 | |
80 | static int g_traceDepth = 0; |
81 | int g_traceFileStat = 0; |
82 | |
83 | #define UTIL_TRACE_CALL(...) \ |
84 | { \ |
85 | if (g_traceFileStat) { \ |
86 | UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ |
87 | UTIL_DISPLAY(__VA_ARGS__); \ |
88 | UTIL_DISPLAY("\n"); \ |
89 | ++g_traceDepth; \ |
90 | } \ |
91 | } |
92 | |
93 | #define UTIL_TRACE_RET(ret) \ |
94 | { \ |
95 | if (g_traceFileStat) { \ |
96 | --g_traceDepth; \ |
97 | UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ |
98 | } \ |
99 | } |
100 | |
101 | /* A modified version of realloc(). |
102 | * If UTIL_realloc() fails the original block is freed. |
103 | */ |
104 | UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) |
105 | { |
106 | void *newptr = realloc(ptr, size); |
107 | if (newptr) return newptr; |
108 | free(ptr); |
109 | return NULL; |
110 | } |
111 | |
112 | #if defined(_MSC_VER) |
113 | #define chmod _chmod |
114 | #endif |
115 | |
116 | #ifndef ZSTD_HAVE_FCHMOD |
117 | #if PLATFORM_POSIX_VERSION >= 199309L |
118 | #define ZSTD_HAVE_FCHMOD |
119 | #endif |
120 | #endif |
121 | |
122 | #ifndef ZSTD_HAVE_FCHOWN |
123 | #if PLATFORM_POSIX_VERSION >= 200809L |
124 | #define ZSTD_HAVE_FCHOWN |
125 | #endif |
126 | #endif |
127 | |
128 | /*-**************************************** |
129 | * Console log |
130 | ******************************************/ |
131 | int g_utilDisplayLevel; |
132 | |
133 | int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, |
134 | const char* acceptableLetters, int hasStdinInput) { |
135 | int ch, result; |
136 | |
137 | if (hasStdinInput) { |
138 | UTIL_DISPLAY("stdin is an input - not proceeding.\n"); |
139 | return 1; |
140 | } |
141 | |
142 | UTIL_DISPLAY("%s", prompt); |
143 | ch = getchar(); |
144 | result = 0; |
145 | if (strchr(acceptableLetters, ch) == NULL) { |
146 | UTIL_DISPLAY("%s \n", abortMsg); |
147 | result = 1; |
148 | } |
149 | /* flush the rest */ |
150 | while ((ch!=EOF) && (ch!='\n')) |
151 | ch = getchar(); |
152 | return result; |
153 | } |
154 | |
155 | |
156 | /*-************************************* |
157 | * Constants |
158 | ***************************************/ |
159 | #define LIST_SIZE_INCREASE (8*1024) |
160 | #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 |
161 | |
162 | |
163 | /*-************************************* |
164 | * Functions |
165 | ***************************************/ |
166 | |
167 | void UTIL_traceFileStat(void) |
168 | { |
169 | g_traceFileStat = 1; |
170 | } |
171 | |
172 | int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) |
173 | { |
174 | int ret; |
175 | UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); |
176 | #if defined(_MSC_VER) |
177 | if (fd >= 0) { |
178 | ret = !_fstat64(fd, statbuf); |
179 | } else { |
180 | ret = !_stat64(filename, statbuf); |
181 | } |
182 | #elif defined(__MINGW32__) && defined (__MSVCRT__) |
183 | if (fd >= 0) { |
184 | ret = !_fstati64(fd, statbuf); |
185 | } else { |
186 | ret = !_stati64(filename, statbuf); |
187 | } |
188 | #else |
189 | if (fd >= 0) { |
190 | ret = !fstat(fd, statbuf); |
191 | } else { |
192 | ret = !stat(filename, statbuf); |
193 | } |
194 | #endif |
195 | UTIL_TRACE_RET(ret); |
196 | return ret; |
197 | } |
198 | |
199 | int UTIL_stat(const char* filename, stat_t* statbuf) |
200 | { |
201 | return UTIL_fstat(-1, filename, statbuf); |
202 | } |
203 | |
204 | int UTIL_isRegularFile(const char* infilename) |
205 | { |
206 | stat_t statbuf; |
207 | int ret; |
208 | UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); |
209 | ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); |
210 | UTIL_TRACE_RET(ret); |
211 | return ret; |
212 | } |
213 | |
214 | int UTIL_isRegularFileStat(const stat_t* statbuf) |
215 | { |
216 | #if defined(_MSC_VER) |
217 | return (statbuf->st_mode & S_IFREG) != 0; |
218 | #else |
219 | return S_ISREG(statbuf->st_mode) != 0; |
220 | #endif |
221 | } |
222 | |
223 | /* like chmod, but avoid changing permission of /dev/null */ |
224 | int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) |
225 | { |
226 | return UTIL_fchmod(-1, filename, statbuf, permissions); |
227 | } |
228 | |
229 | int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) |
230 | { |
231 | stat_t localStatBuf; |
232 | UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); |
233 | if (statbuf == NULL) { |
234 | if (!UTIL_fstat(fd, filename, &localStatBuf)) { |
235 | UTIL_TRACE_RET(0); |
236 | return 0; |
237 | } |
238 | statbuf = &localStatBuf; |
239 | } |
240 | if (!UTIL_isRegularFileStat(statbuf)) { |
241 | UTIL_TRACE_RET(0); |
242 | return 0; /* pretend success, but don't change anything */ |
243 | } |
244 | #ifdef ZSTD_HAVE_FCHMOD |
245 | if (fd >= 0) { |
246 | int ret; |
247 | UTIL_TRACE_CALL("fchmod"); |
248 | ret = fchmod(fd, permissions); |
249 | UTIL_TRACE_RET(ret); |
250 | UTIL_TRACE_RET(ret); |
251 | return ret; |
252 | } else |
253 | #endif |
254 | { |
255 | int ret; |
256 | UTIL_TRACE_CALL("chmod"); |
257 | ret = chmod(filename, permissions); |
258 | UTIL_TRACE_RET(ret); |
259 | UTIL_TRACE_RET(ret); |
260 | return ret; |
261 | } |
262 | } |
263 | |
264 | /* set access and modification times */ |
265 | int UTIL_utime(const char* filename, const stat_t *statbuf) |
266 | { |
267 | int ret; |
268 | UTIL_TRACE_CALL("UTIL_utime(%s)", filename); |
269 | /* We check that st_mtime is a macro here in order to give us confidence |
270 | * that struct stat has a struct timespec st_mtim member. We need this |
271 | * check because there are some platforms that claim to be POSIX 2008 |
272 | * compliant but which do not have st_mtim... */ |
f535537f |
273 | /* FreeBSD has implemented POSIX 2008 for a long time but still only |
274 | * advertises support for POSIX 2001. They have a version macro that |
275 | * lets us safely gate them in. |
276 | * See https://docs.freebsd.org/en/books/porters-handbook/versions/. |
277 | */ |
278 | #if ZSTD_USE_UTIMENSAT |
648db22b |
279 | { |
280 | /* (atime, mtime) */ |
281 | struct timespec timebuf[2] = { {0, UTIME_NOW} }; |
282 | timebuf[1] = statbuf->st_mtim; |
283 | ret = utimensat(AT_FDCWD, filename, timebuf, 0); |
284 | } |
285 | #else |
286 | { |
287 | struct utimbuf timebuf; |
288 | timebuf.actime = time(NULL); |
289 | timebuf.modtime = statbuf->st_mtime; |
290 | ret = utime(filename, &timebuf); |
291 | } |
292 | #endif |
293 | errno = 0; |
294 | UTIL_TRACE_RET(ret); |
295 | return ret; |
296 | } |
297 | |
298 | int UTIL_setFileStat(const char *filename, const stat_t *statbuf) |
299 | { |
300 | return UTIL_setFDStat(-1, filename, statbuf); |
301 | } |
302 | |
303 | int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) |
304 | { |
305 | int res = 0; |
306 | stat_t curStatBuf; |
307 | UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); |
308 | |
309 | if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { |
310 | UTIL_TRACE_RET(-1); |
311 | return -1; |
312 | } |
313 | |
314 | /* Mimic gzip's behavior: |
315 | * |
316 | * "Change the group first, then the permissions, then the owner. |
317 | * That way, the permissions will be correct on systems that allow |
318 | * users to give away files, without introducing a security hole. |
319 | * Security depends on permissions not containing the setuid or |
320 | * setgid bits." */ |
321 | |
322 | #if !defined(_WIN32) |
323 | #ifdef ZSTD_HAVE_FCHOWN |
324 | if (fd >= 0) { |
325 | res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ |
326 | } else |
327 | #endif |
328 | { |
329 | res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ |
330 | } |
331 | #endif |
332 | |
333 | res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ |
334 | |
335 | #if !defined(_WIN32) |
336 | #ifdef ZSTD_HAVE_FCHOWN |
337 | if (fd >= 0) { |
338 | res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ |
339 | } else |
340 | #endif |
341 | { |
342 | res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ |
343 | } |
344 | #endif |
345 | |
346 | errno = 0; |
347 | UTIL_TRACE_RET(-res); |
348 | return -res; /* number of errors is returned */ |
349 | } |
350 | |
351 | int UTIL_isDirectory(const char* infilename) |
352 | { |
353 | stat_t statbuf; |
354 | int ret; |
355 | UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); |
356 | ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); |
357 | UTIL_TRACE_RET(ret); |
358 | return ret; |
359 | } |
360 | |
361 | int UTIL_isDirectoryStat(const stat_t* statbuf) |
362 | { |
363 | int ret; |
364 | UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); |
365 | #if defined(_MSC_VER) |
366 | ret = (statbuf->st_mode & _S_IFDIR) != 0; |
367 | #else |
368 | ret = S_ISDIR(statbuf->st_mode) != 0; |
369 | #endif |
370 | UTIL_TRACE_RET(ret); |
371 | return ret; |
372 | } |
373 | |
374 | int UTIL_compareStr(const void *p1, const void *p2) { |
375 | return strcmp(* (char * const *) p1, * (char * const *) p2); |
376 | } |
377 | |
378 | int UTIL_isSameFile(const char* fName1, const char* fName2) |
379 | { |
380 | int ret; |
381 | assert(fName1 != NULL); assert(fName2 != NULL); |
382 | UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); |
383 | #if defined(_MSC_VER) || defined(_WIN32) |
384 | /* note : Visual does not support file identification by inode. |
385 | * inode does not work on Windows, even with a posix layer, like msys2. |
386 | * The following work-around is limited to detecting exact name repetition only, |
387 | * aka `filename` is considered different from `subdir/../filename` */ |
388 | ret = !strcmp(fName1, fName2); |
389 | #else |
390 | { stat_t file1Stat; |
391 | stat_t file2Stat; |
392 | ret = UTIL_stat(fName1, &file1Stat) |
393 | && UTIL_stat(fName2, &file2Stat) |
394 | && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); |
395 | } |
396 | #endif |
397 | UTIL_TRACE_RET(ret); |
398 | return ret; |
399 | } |
400 | |
401 | int UTIL_isSameFileStat( |
402 | const char* fName1, const char* fName2, |
403 | const stat_t* file1Stat, const stat_t* file2Stat) |
404 | { |
405 | int ret; |
406 | assert(fName1 != NULL); assert(fName2 != NULL); |
407 | UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); |
408 | #if defined(_MSC_VER) || defined(_WIN32) |
409 | /* note : Visual does not support file identification by inode. |
410 | * inode does not work on Windows, even with a posix layer, like msys2. |
411 | * The following work-around is limited to detecting exact name repetition only, |
412 | * aka `filename` is considered different from `subdir/../filename` */ |
413 | (void)file1Stat; |
414 | (void)file2Stat; |
415 | ret = !strcmp(fName1, fName2); |
416 | #else |
417 | { |
418 | ret = (file1Stat->st_dev == file2Stat->st_dev) |
419 | && (file1Stat->st_ino == file2Stat->st_ino); |
420 | } |
421 | #endif |
422 | UTIL_TRACE_RET(ret); |
423 | return ret; |
424 | } |
425 | |
426 | /* UTIL_isFIFO : distinguish named pipes */ |
427 | int UTIL_isFIFO(const char* infilename) |
428 | { |
429 | UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); |
430 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
431 | #if PLATFORM_POSIX_VERSION >= 200112L |
432 | { |
433 | stat_t statbuf; |
434 | if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { |
435 | UTIL_TRACE_RET(1); |
436 | return 1; |
437 | } |
438 | } |
439 | #endif |
440 | (void)infilename; |
441 | UTIL_TRACE_RET(0); |
442 | return 0; |
443 | } |
444 | |
445 | /* UTIL_isFIFO : distinguish named pipes */ |
446 | int UTIL_isFIFOStat(const stat_t* statbuf) |
447 | { |
448 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
449 | #if PLATFORM_POSIX_VERSION >= 200112L |
450 | if (S_ISFIFO(statbuf->st_mode)) return 1; |
451 | #endif |
452 | (void)statbuf; |
453 | return 0; |
454 | } |
455 | |
456 | /* UTIL_isBlockDevStat : distinguish named pipes */ |
457 | int UTIL_isBlockDevStat(const stat_t* statbuf) |
458 | { |
459 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
460 | #if PLATFORM_POSIX_VERSION >= 200112L |
461 | if (S_ISBLK(statbuf->st_mode)) return 1; |
462 | #endif |
463 | (void)statbuf; |
464 | return 0; |
465 | } |
466 | |
467 | int UTIL_isLink(const char* infilename) |
468 | { |
469 | UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); |
470 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
471 | #if PLATFORM_POSIX_VERSION >= 200112L |
472 | { |
473 | stat_t statbuf; |
474 | int const r = lstat(infilename, &statbuf); |
475 | if (!r && S_ISLNK(statbuf.st_mode)) { |
476 | UTIL_TRACE_RET(1); |
477 | return 1; |
478 | } |
479 | } |
480 | #endif |
481 | (void)infilename; |
482 | UTIL_TRACE_RET(0); |
483 | return 0; |
484 | } |
485 | |
486 | static int g_fakeStdinIsConsole = 0; |
487 | static int g_fakeStderrIsConsole = 0; |
488 | static int g_fakeStdoutIsConsole = 0; |
489 | |
490 | int UTIL_isConsole(FILE* file) |
491 | { |
492 | int ret; |
493 | UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); |
494 | if (file == stdin && g_fakeStdinIsConsole) |
495 | ret = 1; |
496 | else if (file == stderr && g_fakeStderrIsConsole) |
497 | ret = 1; |
498 | else if (file == stdout && g_fakeStdoutIsConsole) |
499 | ret = 1; |
500 | else |
501 | ret = IS_CONSOLE(file); |
502 | UTIL_TRACE_RET(ret); |
503 | return ret; |
504 | } |
505 | |
506 | void UTIL_fakeStdinIsConsole(void) |
507 | { |
508 | g_fakeStdinIsConsole = 1; |
509 | } |
510 | void UTIL_fakeStdoutIsConsole(void) |
511 | { |
512 | g_fakeStdoutIsConsole = 1; |
513 | } |
514 | void UTIL_fakeStderrIsConsole(void) |
515 | { |
516 | g_fakeStderrIsConsole = 1; |
517 | } |
518 | |
519 | U64 UTIL_getFileSize(const char* infilename) |
520 | { |
521 | stat_t statbuf; |
522 | UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); |
523 | if (!UTIL_stat(infilename, &statbuf)) { |
524 | UTIL_TRACE_RET(-1); |
525 | return UTIL_FILESIZE_UNKNOWN; |
526 | } |
527 | { |
528 | U64 const size = UTIL_getFileSizeStat(&statbuf); |
529 | UTIL_TRACE_RET((int)size); |
530 | return size; |
531 | } |
532 | } |
533 | |
534 | U64 UTIL_getFileSizeStat(const stat_t* statbuf) |
535 | { |
536 | if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; |
537 | #if defined(_MSC_VER) |
538 | if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; |
539 | #elif defined(__MINGW32__) && defined (__MSVCRT__) |
540 | if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; |
541 | #else |
542 | if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; |
543 | #endif |
544 | return (U64)statbuf->st_size; |
545 | } |
546 | |
547 | UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) |
548 | { |
549 | UTIL_HumanReadableSize_t hrs; |
550 | |
551 | if (g_utilDisplayLevel > 3) { |
552 | /* In verbose mode, do not scale sizes down, except in the case of |
553 | * values that exceed the integral precision of a double. */ |
554 | if (size >= (1ull << 53)) { |
555 | hrs.value = (double)size / (1ull << 20); |
556 | hrs.suffix = " MiB"; |
557 | /* At worst, a double representation of a maximal size will be |
558 | * accurate to better than tens of kilobytes. */ |
559 | hrs.precision = 2; |
560 | } else { |
561 | hrs.value = (double)size; |
562 | hrs.suffix = " B"; |
563 | hrs.precision = 0; |
564 | } |
565 | } else { |
566 | /* In regular mode, scale sizes down and use suffixes. */ |
567 | if (size >= (1ull << 60)) { |
568 | hrs.value = (double)size / (1ull << 60); |
569 | hrs.suffix = " EiB"; |
570 | } else if (size >= (1ull << 50)) { |
571 | hrs.value = (double)size / (1ull << 50); |
572 | hrs.suffix = " PiB"; |
573 | } else if (size >= (1ull << 40)) { |
574 | hrs.value = (double)size / (1ull << 40); |
575 | hrs.suffix = " TiB"; |
576 | } else if (size >= (1ull << 30)) { |
577 | hrs.value = (double)size / (1ull << 30); |
578 | hrs.suffix = " GiB"; |
579 | } else if (size >= (1ull << 20)) { |
580 | hrs.value = (double)size / (1ull << 20); |
581 | hrs.suffix = " MiB"; |
582 | } else if (size >= (1ull << 10)) { |
583 | hrs.value = (double)size / (1ull << 10); |
584 | hrs.suffix = " KiB"; |
585 | } else { |
586 | hrs.value = (double)size; |
587 | hrs.suffix = " B"; |
588 | } |
589 | |
590 | if (hrs.value >= 100 || (U64)hrs.value == size) { |
591 | hrs.precision = 0; |
592 | } else if (hrs.value >= 10) { |
593 | hrs.precision = 1; |
594 | } else if (hrs.value > 1) { |
595 | hrs.precision = 2; |
596 | } else { |
597 | hrs.precision = 3; |
598 | } |
599 | } |
600 | |
601 | return hrs; |
602 | } |
603 | |
604 | U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) |
605 | { |
606 | U64 total = 0; |
607 | unsigned n; |
608 | UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); |
609 | for (n=0; n<nbFiles; n++) { |
610 | U64 const size = UTIL_getFileSize(fileNamesTable[n]); |
611 | if (size == UTIL_FILESIZE_UNKNOWN) { |
612 | UTIL_TRACE_RET(-1); |
613 | return UTIL_FILESIZE_UNKNOWN; |
614 | } |
615 | total += size; |
616 | } |
617 | UTIL_TRACE_RET((int)total); |
618 | return total; |
619 | } |
620 | |
621 | |
622 | /* condition : @file must be valid, and not have reached its end. |
623 | * @return : length of line written into @buf, ended with `\0` instead of '\n', |
624 | * or 0, if there is no new line */ |
625 | static size_t readLineFromFile(char* buf, size_t len, FILE* file) |
626 | { |
627 | assert(!feof(file)); |
628 | if ( fgets(buf, (int) len, file) == NULL ) return 0; |
629 | { size_t linelen = strlen(buf); |
630 | if (strlen(buf)==0) return 0; |
631 | if (buf[linelen-1] == '\n') linelen--; |
632 | buf[linelen] = '\0'; |
633 | return linelen+1; |
634 | } |
635 | } |
636 | |
637 | /* Conditions : |
638 | * size of @inputFileName file must be < @dstCapacity |
639 | * @dst must be initialized |
640 | * @return : nb of lines |
641 | * or -1 if there's an error |
642 | */ |
643 | static int |
644 | readLinesFromFile(void* dst, size_t dstCapacity, |
645 | const char* inputFileName) |
646 | { |
647 | int nbFiles = 0; |
648 | size_t pos = 0; |
649 | char* const buf = (char*)dst; |
650 | FILE* const inputFile = fopen(inputFileName, "r"); |
651 | |
652 | assert(dst != NULL); |
653 | |
654 | if(!inputFile) { |
655 | if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile"); |
656 | return -1; |
657 | } |
658 | |
659 | while ( !feof(inputFile) ) { |
660 | size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); |
661 | if (lineLength == 0) break; |
662 | assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ |
663 | pos += lineLength; |
664 | ++nbFiles; |
665 | } |
666 | |
667 | CONTROL( fclose(inputFile) == 0 ); |
668 | |
669 | return nbFiles; |
670 | } |
671 | |
672 | /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ |
673 | FileNamesTable* |
674 | UTIL_createFileNamesTable_fromFileName(const char* inputFileName) |
675 | { |
676 | size_t nbFiles = 0; |
677 | char* buf; |
678 | size_t bufSize; |
648db22b |
679 | stat_t statbuf; |
680 | |
681 | if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) |
682 | return NULL; |
683 | |
684 | { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); |
685 | if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) |
686 | return NULL; |
687 | bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ |
688 | } |
689 | |
690 | buf = (char*) malloc(bufSize); |
691 | CONTROL( buf != NULL ); |
692 | |
693 | { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); |
694 | |
695 | if (ret_nbFiles <= 0) { |
696 | free(buf); |
697 | return NULL; |
698 | } |
699 | nbFiles = (size_t)ret_nbFiles; |
700 | } |
701 | |
702 | { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); |
703 | CONTROL(filenamesTable != NULL); |
704 | |
f535537f |
705 | { size_t fnb, pos = 0; |
706 | for (fnb = 0; fnb < nbFiles; fnb++) { |
648db22b |
707 | filenamesTable[fnb] = buf+pos; |
708 | pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ |
f535537f |
709 | } |
648db22b |
710 | assert(pos <= bufSize); |
f535537f |
711 | } |
648db22b |
712 | |
713 | return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); |
714 | } |
715 | } |
716 | |
717 | static FileNamesTable* |
718 | UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) |
719 | { |
720 | FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); |
721 | CONTROL(table != NULL); |
722 | table->fileNames = filenames; |
723 | table->buf = buf; |
724 | table->tableSize = tableSize; |
725 | table->tableCapacity = tableCapacity; |
726 | return table; |
727 | } |
728 | |
729 | FileNamesTable* |
730 | UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) |
731 | { |
732 | return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); |
733 | } |
734 | |
735 | void UTIL_freeFileNamesTable(FileNamesTable* table) |
736 | { |
737 | if (table==NULL) return; |
738 | free((void*)table->fileNames); |
739 | free(table->buf); |
740 | free(table); |
741 | } |
742 | |
743 | FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) |
744 | { |
745 | const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); |
746 | FileNamesTable* fnt; |
747 | if (fnTable==NULL) return NULL; |
748 | fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); |
749 | fnt->tableSize = 0; /* the table is empty */ |
750 | return fnt; |
751 | } |
752 | |
753 | int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { |
754 | size_t i; |
755 | for(i=0 ;i < table->tableSize; i++) { |
756 | if(!strcmp(table->fileNames[i], name)) { |
757 | return (int)i; |
758 | } |
759 | } |
760 | return -1; |
761 | } |
762 | |
763 | void UTIL_refFilename(FileNamesTable* fnt, const char* filename) |
764 | { |
765 | assert(fnt->tableSize < fnt->tableCapacity); |
766 | fnt->fileNames[fnt->tableSize] = filename; |
767 | fnt->tableSize++; |
768 | } |
769 | |
770 | static size_t getTotalTableSize(FileNamesTable* table) |
771 | { |
f535537f |
772 | size_t fnb, totalSize = 0; |
648db22b |
773 | for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { |
774 | totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ |
775 | } |
776 | return totalSize; |
777 | } |
778 | |
779 | FileNamesTable* |
780 | UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) |
781 | { |
782 | unsigned newTableIdx = 0; |
783 | size_t pos = 0; |
784 | size_t newTotalTableSize; |
785 | char* buf; |
786 | |
787 | FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); |
788 | CONTROL( newTable != NULL ); |
789 | |
790 | newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); |
791 | |
792 | buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); |
793 | CONTROL ( buf != NULL ); |
794 | |
795 | newTable->buf = buf; |
796 | newTable->tableSize = table1->tableSize + table2->tableSize; |
797 | newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); |
798 | CONTROL ( newTable->fileNames != NULL ); |
799 | |
800 | { unsigned idx1; |
801 | for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { |
802 | size_t const curLen = strlen(table1->fileNames[idx1]); |
803 | memcpy(buf+pos, table1->fileNames[idx1], curLen); |
804 | assert(newTableIdx <= newTable->tableSize); |
805 | newTable->fileNames[newTableIdx] = buf+pos; |
806 | pos += curLen+1; |
807 | } } |
808 | |
809 | { unsigned idx2; |
810 | for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { |
811 | size_t const curLen = strlen(table2->fileNames[idx2]); |
812 | memcpy(buf+pos, table2->fileNames[idx2], curLen); |
813 | assert(newTableIdx < newTable->tableSize); |
814 | newTable->fileNames[newTableIdx] = buf+pos; |
815 | pos += curLen+1; |
816 | } } |
817 | assert(pos <= newTotalTableSize); |
818 | newTable->tableSize = newTableIdx; |
819 | |
820 | UTIL_freeFileNamesTable(table1); |
821 | UTIL_freeFileNamesTable(table2); |
822 | |
823 | return newTable; |
824 | } |
825 | |
826 | #ifdef _WIN32 |
827 | static int UTIL_prepareFileList(const char* dirName, |
828 | char** bufStart, size_t* pos, |
829 | char** bufEnd, int followLinks) |
830 | { |
831 | char* path; |
832 | size_t dirLength, pathLength; |
833 | int nbFiles = 0; |
834 | WIN32_FIND_DATAA cFile; |
835 | HANDLE hFile; |
836 | |
837 | dirLength = strlen(dirName); |
838 | path = (char*) malloc(dirLength + 3); |
839 | if (!path) return 0; |
840 | |
841 | memcpy(path, dirName, dirLength); |
842 | path[dirLength] = '\\'; |
843 | path[dirLength+1] = '*'; |
844 | path[dirLength+2] = 0; |
845 | |
846 | hFile=FindFirstFileA(path, &cFile); |
847 | if (hFile == INVALID_HANDLE_VALUE) { |
848 | UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); |
849 | return 0; |
850 | } |
851 | free(path); |
852 | |
853 | do { |
854 | size_t const fnameLength = strlen(cFile.cFileName); |
855 | path = (char*) malloc(dirLength + fnameLength + 2); |
856 | if (!path) { FindClose(hFile); return 0; } |
857 | memcpy(path, dirName, dirLength); |
858 | path[dirLength] = '\\'; |
859 | memcpy(path+dirLength+1, cFile.cFileName, fnameLength); |
860 | pathLength = dirLength+1+fnameLength; |
861 | path[pathLength] = 0; |
862 | if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { |
863 | if ( strcmp (cFile.cFileName, "..") == 0 |
864 | || strcmp (cFile.cFileName, ".") == 0 ) |
865 | continue; |
866 | /* Recursively call "UTIL_prepareFileList" with the new path. */ |
867 | nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); |
868 | if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } |
869 | } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) |
870 | || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) |
871 | || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { |
872 | if (*bufStart + *pos + pathLength >= *bufEnd) { |
873 | ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; |
874 | *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); |
875 | if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } |
876 | *bufEnd = *bufStart + newListSize; |
877 | } |
878 | if (*bufStart + *pos + pathLength < *bufEnd) { |
879 | memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); |
880 | *pos += pathLength + 1; |
881 | nbFiles++; |
882 | } } |
883 | free(path); |
884 | } while (FindNextFileA(hFile, &cFile)); |
885 | |
886 | FindClose(hFile); |
887 | return nbFiles; |
888 | } |
889 | |
890 | #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ |
891 | |
892 | static int UTIL_prepareFileList(const char *dirName, |
893 | char** bufStart, size_t* pos, |
894 | char** bufEnd, int followLinks) |
895 | { |
896 | DIR* dir; |
897 | struct dirent * entry; |
898 | size_t dirLength; |
899 | int nbFiles = 0; |
900 | |
901 | if (!(dir = opendir(dirName))) { |
902 | UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); |
903 | return 0; |
904 | } |
905 | |
906 | dirLength = strlen(dirName); |
907 | errno = 0; |
908 | while ((entry = readdir(dir)) != NULL) { |
909 | char* path; |
910 | size_t fnameLength, pathLength; |
911 | if (strcmp (entry->d_name, "..") == 0 || |
912 | strcmp (entry->d_name, ".") == 0) continue; |
913 | fnameLength = strlen(entry->d_name); |
914 | path = (char*) malloc(dirLength + fnameLength + 2); |
915 | if (!path) { closedir(dir); return 0; } |
916 | memcpy(path, dirName, dirLength); |
917 | |
918 | path[dirLength] = '/'; |
919 | memcpy(path+dirLength+1, entry->d_name, fnameLength); |
920 | pathLength = dirLength+1+fnameLength; |
921 | path[pathLength] = 0; |
922 | |
923 | if (!followLinks && UTIL_isLink(path)) { |
924 | UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); |
925 | free(path); |
926 | continue; |
927 | } |
928 | |
929 | if (UTIL_isDirectory(path)) { |
930 | nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ |
931 | if (*bufStart == NULL) { free(path); closedir(dir); return 0; } |
932 | } else { |
933 | if (*bufStart + *pos + pathLength >= *bufEnd) { |
934 | ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; |
935 | assert(newListSize >= 0); |
936 | *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); |
937 | if (*bufStart != NULL) { |
938 | *bufEnd = *bufStart + newListSize; |
939 | } else { |
940 | free(path); closedir(dir); return 0; |
941 | } |
942 | } |
943 | if (*bufStart + *pos + pathLength < *bufEnd) { |
944 | memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ |
945 | *pos += pathLength + 1; |
946 | nbFiles++; |
947 | } } |
948 | free(path); |
949 | errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ |
950 | } |
951 | |
952 | if (errno != 0) { |
953 | UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); |
954 | free(*bufStart); |
955 | *bufStart = NULL; |
956 | } |
957 | closedir(dir); |
958 | return nbFiles; |
959 | } |
960 | |
961 | #else |
962 | |
963 | static int UTIL_prepareFileList(const char *dirName, |
964 | char** bufStart, size_t* pos, |
965 | char** bufEnd, int followLinks) |
966 | { |
967 | (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; |
968 | UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); |
969 | return 0; |
970 | } |
971 | |
972 | #endif /* #ifdef _WIN32 */ |
973 | |
974 | int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) |
975 | { |
976 | const char* ext = UTIL_getFileExtension(inputName); |
977 | while(*extensionList!=NULL) |
978 | { |
979 | const int isCompressedExtension = strcmp(ext,*extensionList); |
980 | if(isCompressedExtension==0) |
981 | return 1; |
982 | ++extensionList; |
983 | } |
984 | return 0; |
985 | } |
986 | |
987 | /*Utility function to get file extension from file */ |
988 | const char* UTIL_getFileExtension(const char* infilename) |
989 | { |
990 | const char* extension = strrchr(infilename, '.'); |
991 | if(!extension || extension==infilename) return ""; |
992 | return extension; |
993 | } |
994 | |
995 | static int pathnameHas2Dots(const char *pathname) |
996 | { |
997 | /* We need to figure out whether any ".." present in the path is a whole |
998 | * path token, which is the case if it is bordered on both sides by either |
999 | * the beginning/end of the path or by a directory separator. |
1000 | */ |
1001 | const char *needle = pathname; |
1002 | while (1) { |
1003 | needle = strstr(needle, ".."); |
1004 | |
1005 | if (needle == NULL) { |
1006 | return 0; |
1007 | } |
1008 | |
1009 | if ((needle == pathname || needle[-1] == PATH_SEP) |
1010 | && (needle[2] == '\0' || needle[2] == PATH_SEP)) { |
1011 | return 1; |
1012 | } |
1013 | |
1014 | /* increment so we search for the next match */ |
1015 | needle++; |
1016 | }; |
1017 | return 0; |
1018 | } |
1019 | |
1020 | static int isFileNameValidForMirroredOutput(const char *filename) |
1021 | { |
1022 | return !pathnameHas2Dots(filename); |
1023 | } |
1024 | |
1025 | |
1026 | #define DIR_DEFAULT_MODE 0755 |
1027 | static mode_t getDirMode(const char *dirName) |
1028 | { |
1029 | stat_t st; |
1030 | if (!UTIL_stat(dirName, &st)) { |
1031 | UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); |
1032 | return DIR_DEFAULT_MODE; |
1033 | } |
1034 | if (!UTIL_isDirectoryStat(&st)) { |
1035 | UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); |
1036 | return DIR_DEFAULT_MODE; |
1037 | } |
1038 | return st.st_mode; |
1039 | } |
1040 | |
1041 | static int makeDir(const char *dir, mode_t mode) |
1042 | { |
1043 | #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) |
1044 | int ret = _mkdir(dir); |
1045 | (void) mode; |
1046 | #else |
1047 | int ret = mkdir(dir, mode); |
1048 | #endif |
1049 | if (ret != 0) { |
1050 | if (errno == EEXIST) |
1051 | return 0; |
1052 | UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); |
1053 | } |
1054 | return ret; |
1055 | } |
1056 | |
1057 | /* this function requires a mutable input string */ |
1058 | static void convertPathnameToDirName(char *pathname) |
1059 | { |
1060 | size_t len = 0; |
1061 | char* pos = NULL; |
1062 | /* get dir name from pathname similar to 'dirname()' */ |
1063 | assert(pathname != NULL); |
1064 | |
1065 | /* remove trailing '/' chars */ |
1066 | len = strlen(pathname); |
1067 | assert(len > 0); |
1068 | while (pathname[len] == PATH_SEP) { |
1069 | pathname[len] = '\0'; |
1070 | len--; |
1071 | } |
1072 | if (len == 0) return; |
1073 | |
1074 | /* if input is a single file, return '.' instead. i.e. |
1075 | * "xyz/abc/file.txt" => "xyz/abc" |
1076 | "./file.txt" => "." |
1077 | "file.txt" => "." |
1078 | */ |
1079 | pos = strrchr(pathname, PATH_SEP); |
1080 | if (pos == NULL) { |
1081 | pathname[0] = '.'; |
1082 | pathname[1] = '\0'; |
1083 | } else { |
1084 | *pos = '\0'; |
1085 | } |
1086 | } |
1087 | |
1088 | /* pathname must be valid */ |
1089 | static const char* trimLeadingRootChar(const char *pathname) |
1090 | { |
1091 | assert(pathname != NULL); |
1092 | if (pathname[0] == PATH_SEP) |
1093 | return pathname + 1; |
1094 | return pathname; |
1095 | } |
1096 | |
1097 | /* pathname must be valid */ |
1098 | static const char* trimLeadingCurrentDirConst(const char *pathname) |
1099 | { |
1100 | assert(pathname != NULL); |
1101 | if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) |
1102 | return pathname + 2; |
1103 | return pathname; |
1104 | } |
1105 | |
1106 | static char* |
1107 | trimLeadingCurrentDir(char *pathname) |
1108 | { |
1109 | /* 'union charunion' can do const-cast without compiler warning */ |
1110 | union charunion { |
1111 | char *chr; |
1112 | const char* cchr; |
1113 | } ptr; |
1114 | ptr.cchr = trimLeadingCurrentDirConst(pathname); |
1115 | return ptr.chr; |
1116 | } |
1117 | |
1118 | /* remove leading './' or '/' chars here */ |
1119 | static const char * trimPath(const char *pathname) |
1120 | { |
1121 | return trimLeadingRootChar( |
1122 | trimLeadingCurrentDirConst(pathname)); |
1123 | } |
1124 | |
1125 | static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) |
1126 | { |
1127 | assert(dir1 != NULL && dir2 != NULL); |
1128 | { const size_t dir1Size = strlen(dir1); |
1129 | const size_t dir2Size = strlen(dir2); |
1130 | char *outDirBuffer, *buffer; |
1131 | |
1132 | outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); |
1133 | CONTROL(outDirBuffer != NULL); |
1134 | |
1135 | memcpy(outDirBuffer, dir1, dir1Size); |
1136 | outDirBuffer[dir1Size] = '\0'; |
1137 | |
648db22b |
1138 | buffer = outDirBuffer + dir1Size; |
1139 | if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { |
1140 | *buffer = PATH_SEP; |
1141 | buffer++; |
1142 | } |
1143 | memcpy(buffer, dir2, dir2Size); |
1144 | buffer[dir2Size] = '\0'; |
1145 | |
1146 | return outDirBuffer; |
1147 | } |
1148 | } |
1149 | |
1150 | /* this function will return NULL if input srcFileName is not valid name for mirrored output path */ |
1151 | char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) |
1152 | { |
1153 | char* pathname = NULL; |
1154 | if (!isFileNameValidForMirroredOutput(srcFileName)) |
1155 | return NULL; |
1156 | |
1157 | pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); |
1158 | |
1159 | convertPathnameToDirName(pathname); |
1160 | return pathname; |
1161 | } |
1162 | |
1163 | static int |
1164 | mirrorSrcDir(char* srcDirName, const char* outDirName) |
1165 | { |
1166 | mode_t srcMode; |
1167 | int status = 0; |
1168 | char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); |
1169 | if (!newDir) |
1170 | return -ENOMEM; |
1171 | |
1172 | srcMode = getDirMode(srcDirName); |
1173 | status = makeDir(newDir, srcMode); |
1174 | free(newDir); |
1175 | return status; |
1176 | } |
1177 | |
1178 | static int |
1179 | mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) |
1180 | { |
1181 | int status = 0; |
1182 | char* pp = trimLeadingCurrentDir(srcDirName); |
1183 | char* sp = NULL; |
1184 | |
1185 | while ((sp = strchr(pp, PATH_SEP)) != NULL) { |
1186 | if (sp != pp) { |
1187 | *sp = '\0'; |
1188 | status = mirrorSrcDir(srcDirName, outDirName); |
1189 | if (status != 0) |
1190 | return status; |
1191 | *sp = PATH_SEP; |
1192 | } |
1193 | pp = sp + 1; |
1194 | } |
1195 | status = mirrorSrcDir(srcDirName, outDirName); |
1196 | return status; |
1197 | } |
1198 | |
1199 | static void |
1200 | makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) |
1201 | { |
1202 | unsigned int i = 0; |
1203 | for (i = 0; i < nbFile; i++) |
1204 | mirrorSrcDirRecursive(srcDirNames[i], outDirName); |
1205 | } |
1206 | |
1207 | static int |
1208 | firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) |
1209 | { |
1210 | size_t firstDirLen = strlen(firstDir), |
1211 | secondDirLen = strlen(secondDir); |
1212 | return firstDirLen <= secondDirLen && |
1213 | (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && |
1214 | 0 == strncmp(firstDir, secondDir, firstDirLen); |
1215 | } |
1216 | |
1217 | static int compareDir(const void* pathname1, const void* pathname2) { |
1218 | /* sort it after remove the leading '/' or './'*/ |
1219 | const char* s1 = trimPath(*(char * const *) pathname1); |
1220 | const char* s2 = trimPath(*(char * const *) pathname2); |
1221 | return strcmp(s1, s2); |
1222 | } |
1223 | |
1224 | static void |
1225 | makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) |
1226 | { |
1227 | unsigned int i = 0, uniqueDirNr = 0; |
1228 | char** uniqueDirNames = NULL; |
1229 | |
1230 | if (nbFile == 0) |
1231 | return; |
1232 | |
1233 | uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); |
1234 | CONTROL(uniqueDirNames != NULL); |
1235 | |
1236 | /* if dirs is "a/b/c" and "a/b/c/d", we only need call: |
1237 | * we just need "a/b/c/d" */ |
1238 | qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); |
1239 | |
1240 | uniqueDirNr = 1; |
1241 | uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; |
1242 | for (i = 1; i < nbFile; i++) { |
1243 | char* prevDirName = srcDirNames[i - 1]; |
1244 | char* currDirName = srcDirNames[i]; |
1245 | |
1246 | /* note: we always compare trimmed path, i.e.: |
1247 | * src dir of "./foo" and "/foo" will be both saved into: |
1248 | * "outDirName/foo/" */ |
1249 | if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), |
1250 | trimPath(currDirName))) |
1251 | uniqueDirNr++; |
1252 | |
1253 | /* we need to maintain original src dir name instead of trimmed |
1254 | * dir, so we can retrieve the original src dir's mode_t */ |
1255 | uniqueDirNames[uniqueDirNr - 1] = currDirName; |
1256 | } |
1257 | |
1258 | makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); |
1259 | |
1260 | free(uniqueDirNames); |
1261 | } |
1262 | |
1263 | static void |
1264 | makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) |
1265 | { |
1266 | unsigned int i = 0; |
1267 | for (i = 0; i < nbFile; ++i) |
1268 | convertPathnameToDirName(srcFileNames[i]); |
1269 | makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); |
1270 | } |
1271 | |
1272 | void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) |
1273 | { |
1274 | unsigned int i = 0, validFilenamesNr = 0; |
1275 | char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); |
1276 | CONTROL(srcFileNames != NULL); |
1277 | |
1278 | /* check input filenames is valid */ |
1279 | for (i = 0; i < nbFile; ++i) { |
1280 | if (isFileNameValidForMirroredOutput(inFileNames[i])) { |
1281 | char* fname = STRDUP(inFileNames[i]); |
1282 | CONTROL(fname != NULL); |
1283 | srcFileNames[validFilenamesNr++] = fname; |
1284 | } |
1285 | } |
1286 | |
1287 | if (validFilenamesNr > 0) { |
1288 | makeDir(outDirName, DIR_DEFAULT_MODE); |
1289 | makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); |
1290 | } |
1291 | |
1292 | for (i = 0; i < validFilenamesNr; i++) |
1293 | free(srcFileNames[i]); |
1294 | free(srcFileNames); |
1295 | } |
1296 | |
1297 | FileNamesTable* |
1298 | UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) |
1299 | { |
1300 | unsigned nbFiles; |
1301 | char* buf = (char*)malloc(LIST_SIZE_INCREASE); |
1302 | char* bufend = buf + LIST_SIZE_INCREASE; |
1303 | |
1304 | if (!buf) return NULL; |
1305 | |
1306 | { size_t ifnNb, pos; |
1307 | for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) { |
1308 | if (!UTIL_isDirectory(inputNames[ifnNb])) { |
1309 | size_t const len = strlen(inputNames[ifnNb]); |
1310 | if (buf + pos + len >= bufend) { |
1311 | ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; |
1312 | assert(newListSize >= 0); |
1313 | buf = (char*)UTIL_realloc(buf, (size_t)newListSize); |
1314 | if (!buf) return NULL; |
1315 | bufend = buf + newListSize; |
1316 | } |
1317 | if (buf + pos + len < bufend) { |
1318 | memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ |
1319 | pos += len + 1; |
1320 | nbFiles++; |
1321 | } |
1322 | } else { |
1323 | nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); |
1324 | if (buf == NULL) return NULL; |
1325 | } } } |
1326 | |
1327 | /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ |
1328 | |
1329 | { size_t ifnNb, pos; |
1330 | size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ |
1331 | const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); |
1332 | if (!fileNamesTable) { free(buf); return NULL; } |
1333 | |
1334 | for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { |
1335 | fileNamesTable[ifnNb] = buf + pos; |
1336 | if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } |
1337 | pos += strlen(fileNamesTable[ifnNb]) + 1; |
1338 | } |
1339 | return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); |
1340 | } |
1341 | } |
1342 | |
1343 | |
1344 | void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) |
1345 | { |
1346 | FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); |
1347 | CONTROL(newFNT != NULL); |
1348 | UTIL_freeFileNamesTable(*fnt); |
1349 | *fnt = newFNT; |
1350 | } |
1351 | |
1352 | FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) |
1353 | { |
1354 | size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); |
1355 | const char** const newFNTable = (const char**)malloc(sizeof_FNTable); |
1356 | if (newFNTable==NULL) return NULL; |
1357 | memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ |
1358 | return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); |
1359 | } |
1360 | |
1361 | |
1362 | /*-**************************************** |
1363 | * count the number of cores |
1364 | ******************************************/ |
1365 | |
1366 | #if defined(_WIN32) || defined(WIN32) |
1367 | |
1368 | #include <windows.h> |
1369 | |
1370 | typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); |
1371 | |
1372 | DWORD CountSetBits(ULONG_PTR bitMask) |
1373 | { |
1374 | DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; |
1375 | DWORD bitSetCount = 0; |
1376 | ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; |
1377 | DWORD i; |
1378 | |
1379 | for (i = 0; i <= LSHIFT; ++i) |
1380 | { |
1381 | bitSetCount += ((bitMask & bitTest)?1:0); |
1382 | bitTest/=2; |
1383 | } |
1384 | |
1385 | return bitSetCount; |
1386 | } |
1387 | |
1388 | int UTIL_countCores(int logical) |
1389 | { |
1390 | static int numCores = 0; |
1391 | if (numCores != 0) return numCores; |
1392 | |
1393 | { LPFN_GLPI glpi; |
1394 | BOOL done = FALSE; |
1395 | PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; |
1396 | PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; |
1397 | DWORD returnLength = 0; |
1398 | size_t byteOffset = 0; |
1399 | |
1400 | #if defined(_MSC_VER) |
1401 | /* Visual Studio does not like the following cast */ |
1402 | # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ |
1403 | # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ |
1404 | #endif |
1405 | glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), |
1406 | "GetLogicalProcessorInformation"); |
1407 | |
1408 | if (glpi == NULL) { |
1409 | goto failed; |
1410 | } |
1411 | |
1412 | while(!done) { |
1413 | DWORD rc = glpi(buffer, &returnLength); |
1414 | if (FALSE == rc) { |
1415 | if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { |
1416 | if (buffer) |
1417 | free(buffer); |
1418 | buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); |
1419 | |
1420 | if (buffer == NULL) { |
1421 | perror("zstd"); |
1422 | exit(1); |
1423 | } |
1424 | } else { |
1425 | /* some other error */ |
1426 | goto failed; |
1427 | } |
1428 | } else { |
1429 | done = TRUE; |
1430 | } } |
1431 | |
1432 | ptr = buffer; |
1433 | |
1434 | while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { |
1435 | |
1436 | if (ptr->Relationship == RelationProcessorCore) { |
1437 | if (logical) |
1438 | numCores += CountSetBits(ptr->ProcessorMask); |
1439 | else |
1440 | numCores++; |
1441 | } |
1442 | |
1443 | ptr++; |
1444 | byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); |
1445 | } |
1446 | |
1447 | free(buffer); |
1448 | |
1449 | return numCores; |
1450 | } |
1451 | |
1452 | failed: |
1453 | /* try to fall back on GetSystemInfo */ |
1454 | { SYSTEM_INFO sysinfo; |
1455 | GetSystemInfo(&sysinfo); |
1456 | numCores = sysinfo.dwNumberOfProcessors; |
1457 | if (numCores == 0) numCores = 1; /* just in case */ |
1458 | } |
1459 | return numCores; |
1460 | } |
1461 | |
1462 | #elif defined(__APPLE__) |
1463 | |
1464 | #include <sys/sysctl.h> |
1465 | |
1466 | /* Use apple-provided syscall |
1467 | * see: man 3 sysctl */ |
1468 | int UTIL_countCores(int logical) |
1469 | { |
1470 | static S32 numCores = 0; /* apple specifies int32_t */ |
1471 | if (numCores != 0) return numCores; |
1472 | |
1473 | { size_t size = sizeof(S32); |
1474 | int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); |
1475 | if (ret != 0) { |
1476 | if (errno == ENOENT) { |
1477 | /* entry not present, fall back on 1 */ |
1478 | numCores = 1; |
1479 | } else { |
1480 | perror("zstd: can't get number of cpus"); |
1481 | exit(1); |
1482 | } |
1483 | } |
1484 | |
1485 | return numCores; |
1486 | } |
1487 | } |
1488 | |
1489 | #elif defined(__linux__) |
1490 | |
1491 | /* parse /proc/cpuinfo |
1492 | * siblings / cpu cores should give hyperthreading ratio |
1493 | * otherwise fall back on sysconf */ |
1494 | int UTIL_countCores(int logical) |
1495 | { |
1496 | static int numCores = 0; |
1497 | |
1498 | if (numCores != 0) return numCores; |
1499 | |
1500 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1501 | if (numCores == -1) { |
1502 | /* value not queryable, fall back on 1 */ |
1503 | return numCores = 1; |
1504 | } |
1505 | |
1506 | /* try to determine if there's hyperthreading */ |
1507 | { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); |
1508 | #define BUF_SIZE 80 |
1509 | char buff[BUF_SIZE]; |
1510 | |
1511 | int siblings = 0; |
1512 | int cpu_cores = 0; |
1513 | int ratio = 1; |
1514 | |
1515 | if (cpuinfo == NULL) { |
1516 | /* fall back on the sysconf value */ |
1517 | return numCores; |
1518 | } |
1519 | |
1520 | /* assume the cpu cores/siblings values will be constant across all |
1521 | * present processors */ |
1522 | while (!feof(cpuinfo)) { |
1523 | if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { |
1524 | if (strncmp(buff, "siblings", 8) == 0) { |
1525 | const char* const sep = strchr(buff, ':'); |
1526 | if (sep == NULL || *sep == '\0') { |
1527 | /* formatting was broken? */ |
1528 | goto failed; |
1529 | } |
1530 | |
1531 | siblings = atoi(sep + 1); |
1532 | } |
1533 | if (strncmp(buff, "cpu cores", 9) == 0) { |
1534 | const char* const sep = strchr(buff, ':'); |
1535 | if (sep == NULL || *sep == '\0') { |
1536 | /* formatting was broken? */ |
1537 | goto failed; |
1538 | } |
1539 | |
1540 | cpu_cores = atoi(sep + 1); |
1541 | } |
1542 | } else if (ferror(cpuinfo)) { |
1543 | /* fall back on the sysconf value */ |
1544 | goto failed; |
1545 | } } |
1546 | if (siblings && cpu_cores && siblings > cpu_cores) { |
1547 | ratio = siblings / cpu_cores; |
1548 | } |
1549 | |
1550 | if (ratio && numCores > ratio && !logical) { |
1551 | numCores = numCores / ratio; |
1552 | } |
1553 | |
1554 | failed: |
1555 | fclose(cpuinfo); |
1556 | return numCores; |
1557 | } |
1558 | } |
1559 | |
1560 | #elif defined(__FreeBSD__) |
1561 | |
648db22b |
1562 | #include <sys/sysctl.h> |
1563 | |
1564 | /* Use physical core sysctl when available |
1565 | * see: man 4 smp, man 3 sysctl */ |
1566 | int UTIL_countCores(int logical) |
1567 | { |
1568 | static int numCores = 0; /* freebsd sysctl is native int sized */ |
1569 | #if __FreeBSD_version >= 1300008 |
1570 | static int perCore = 1; |
1571 | #endif |
1572 | if (numCores != 0) return numCores; |
1573 | |
1574 | #if __FreeBSD_version >= 1300008 |
1575 | { size_t size = sizeof(numCores); |
1576 | int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); |
1577 | if (ret == 0) { |
1578 | if (logical) { |
1579 | ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); |
1580 | /* default to physical cores if logical cannot be read */ |
1581 | if (ret == 0) |
1582 | numCores *= perCore; |
1583 | } |
1584 | |
1585 | return numCores; |
1586 | } |
1587 | if (errno != ENOENT) { |
1588 | perror("zstd: can't get number of cpus"); |
1589 | exit(1); |
1590 | } |
1591 | /* sysctl not present, fall through to older sysconf method */ |
1592 | } |
1593 | #else |
1594 | /* suppress unused parameter warning */ |
1595 | (void) logical; |
1596 | #endif |
1597 | |
1598 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1599 | if (numCores == -1) { |
1600 | /* value not queryable, fall back on 1 */ |
1601 | numCores = 1; |
1602 | } |
1603 | return numCores; |
1604 | } |
1605 | |
1606 | #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) |
1607 | |
1608 | /* Use POSIX sysconf |
1609 | * see: man 3 sysconf */ |
1610 | int UTIL_countCores(int logical) |
1611 | { |
1612 | static int numCores = 0; |
1613 | |
1614 | /* suppress unused parameter warning */ |
1615 | (void)logical; |
1616 | |
1617 | if (numCores != 0) return numCores; |
1618 | |
1619 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1620 | if (numCores == -1) { |
1621 | /* value not queryable, fall back on 1 */ |
1622 | return numCores = 1; |
1623 | } |
1624 | return numCores; |
1625 | } |
1626 | |
1627 | #else |
1628 | |
1629 | int UTIL_countCores(int logical) |
1630 | { |
1631 | /* suppress unused parameter warning */ |
1632 | (void)logical; |
1633 | |
1634 | /* assume 1 */ |
1635 | return 1; |
1636 | } |
1637 | |
1638 | #endif |
1639 | |
1640 | int UTIL_countPhysicalCores(void) |
1641 | { |
1642 | return UTIL_countCores(0); |
1643 | } |
1644 | |
1645 | int UTIL_countLogicalCores(void) |
1646 | { |
1647 | return UTIL_countCores(1); |
1648 | } |
1649 | |
1650 | #if defined (__cplusplus) |
1651 | } |
1652 | #endif |