648db22b |
1 | /* |
2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | * You may select, at your option, one of the above-listed licenses. |
9 | */ |
10 | |
11 | #if defined (__cplusplus) |
12 | extern "C" { |
13 | #endif |
14 | |
15 | |
16 | /*-**************************************** |
17 | * Dependencies |
18 | ******************************************/ |
19 | #include "util.h" /* note : ensure that platform.h is included first ! */ |
20 | #include <stdlib.h> /* malloc, realloc, free */ |
21 | #include <stdio.h> /* fprintf */ |
22 | #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ |
23 | #include <errno.h> |
24 | #include <assert.h> |
25 | |
26 | #if defined(_WIN32) |
27 | # include <sys/utime.h> /* utime */ |
28 | # include <io.h> /* _chmod */ |
29 | #else |
30 | # include <unistd.h> /* chown, stat */ |
31 | # if PLATFORM_POSIX_VERSION < 200809L || !defined(st_mtime) |
32 | # include <utime.h> /* utime */ |
33 | # else |
34 | # include <fcntl.h> /* AT_FDCWD */ |
35 | # include <sys/stat.h> /* utimensat */ |
36 | # endif |
37 | #endif |
38 | |
39 | #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) |
40 | #include <direct.h> /* needed for _mkdir in windows */ |
41 | #endif |
42 | |
43 | #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ |
44 | # include <dirent.h> /* opendir, readdir */ |
45 | # include <string.h> /* strerror, memcpy */ |
46 | #endif /* #ifdef _WIN32 */ |
47 | |
48 | /*-**************************************** |
49 | * Internal Macros |
50 | ******************************************/ |
51 | |
52 | /* CONTROL is almost like an assert(), but is never disabled. |
53 | * It's designed for failures that may happen rarely, |
54 | * but we don't want to maintain a specific error code path for them, |
55 | * such as a malloc() returning NULL for example. |
56 | * Since it's always active, this macro can trigger side effects. |
57 | */ |
58 | #define CONTROL(c) { \ |
59 | if (!(c)) { \ |
60 | UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ |
61 | __FILE__, __LINE__, #c); \ |
62 | exit(1); \ |
63 | } } |
64 | |
65 | /* console log */ |
66 | #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
67 | #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } |
68 | |
69 | static int g_traceDepth = 0; |
70 | int g_traceFileStat = 0; |
71 | |
72 | #define UTIL_TRACE_CALL(...) \ |
73 | { \ |
74 | if (g_traceFileStat) { \ |
75 | UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ |
76 | UTIL_DISPLAY(__VA_ARGS__); \ |
77 | UTIL_DISPLAY("\n"); \ |
78 | ++g_traceDepth; \ |
79 | } \ |
80 | } |
81 | |
82 | #define UTIL_TRACE_RET(ret) \ |
83 | { \ |
84 | if (g_traceFileStat) { \ |
85 | --g_traceDepth; \ |
86 | UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ |
87 | } \ |
88 | } |
89 | |
90 | /* A modified version of realloc(). |
91 | * If UTIL_realloc() fails the original block is freed. |
92 | */ |
93 | UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) |
94 | { |
95 | void *newptr = realloc(ptr, size); |
96 | if (newptr) return newptr; |
97 | free(ptr); |
98 | return NULL; |
99 | } |
100 | |
101 | #if defined(_MSC_VER) |
102 | #define chmod _chmod |
103 | #endif |
104 | |
105 | #ifndef ZSTD_HAVE_FCHMOD |
106 | #if PLATFORM_POSIX_VERSION >= 199309L |
107 | #define ZSTD_HAVE_FCHMOD |
108 | #endif |
109 | #endif |
110 | |
111 | #ifndef ZSTD_HAVE_FCHOWN |
112 | #if PLATFORM_POSIX_VERSION >= 200809L |
113 | #define ZSTD_HAVE_FCHOWN |
114 | #endif |
115 | #endif |
116 | |
117 | /*-**************************************** |
118 | * Console log |
119 | ******************************************/ |
120 | int g_utilDisplayLevel; |
121 | |
122 | int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, |
123 | const char* acceptableLetters, int hasStdinInput) { |
124 | int ch, result; |
125 | |
126 | if (hasStdinInput) { |
127 | UTIL_DISPLAY("stdin is an input - not proceeding.\n"); |
128 | return 1; |
129 | } |
130 | |
131 | UTIL_DISPLAY("%s", prompt); |
132 | ch = getchar(); |
133 | result = 0; |
134 | if (strchr(acceptableLetters, ch) == NULL) { |
135 | UTIL_DISPLAY("%s \n", abortMsg); |
136 | result = 1; |
137 | } |
138 | /* flush the rest */ |
139 | while ((ch!=EOF) && (ch!='\n')) |
140 | ch = getchar(); |
141 | return result; |
142 | } |
143 | |
144 | |
145 | /*-************************************* |
146 | * Constants |
147 | ***************************************/ |
148 | #define LIST_SIZE_INCREASE (8*1024) |
149 | #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 |
150 | |
151 | |
152 | /*-************************************* |
153 | * Functions |
154 | ***************************************/ |
155 | |
156 | void UTIL_traceFileStat(void) |
157 | { |
158 | g_traceFileStat = 1; |
159 | } |
160 | |
161 | int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) |
162 | { |
163 | int ret; |
164 | UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); |
165 | #if defined(_MSC_VER) |
166 | if (fd >= 0) { |
167 | ret = !_fstat64(fd, statbuf); |
168 | } else { |
169 | ret = !_stat64(filename, statbuf); |
170 | } |
171 | #elif defined(__MINGW32__) && defined (__MSVCRT__) |
172 | if (fd >= 0) { |
173 | ret = !_fstati64(fd, statbuf); |
174 | } else { |
175 | ret = !_stati64(filename, statbuf); |
176 | } |
177 | #else |
178 | if (fd >= 0) { |
179 | ret = !fstat(fd, statbuf); |
180 | } else { |
181 | ret = !stat(filename, statbuf); |
182 | } |
183 | #endif |
184 | UTIL_TRACE_RET(ret); |
185 | return ret; |
186 | } |
187 | |
188 | int UTIL_stat(const char* filename, stat_t* statbuf) |
189 | { |
190 | return UTIL_fstat(-1, filename, statbuf); |
191 | } |
192 | |
193 | int UTIL_isRegularFile(const char* infilename) |
194 | { |
195 | stat_t statbuf; |
196 | int ret; |
197 | UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); |
198 | ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); |
199 | UTIL_TRACE_RET(ret); |
200 | return ret; |
201 | } |
202 | |
203 | int UTIL_isRegularFileStat(const stat_t* statbuf) |
204 | { |
205 | #if defined(_MSC_VER) |
206 | return (statbuf->st_mode & S_IFREG) != 0; |
207 | #else |
208 | return S_ISREG(statbuf->st_mode) != 0; |
209 | #endif |
210 | } |
211 | |
212 | /* like chmod, but avoid changing permission of /dev/null */ |
213 | int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) |
214 | { |
215 | return UTIL_fchmod(-1, filename, statbuf, permissions); |
216 | } |
217 | |
218 | int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) |
219 | { |
220 | stat_t localStatBuf; |
221 | UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); |
222 | if (statbuf == NULL) { |
223 | if (!UTIL_fstat(fd, filename, &localStatBuf)) { |
224 | UTIL_TRACE_RET(0); |
225 | return 0; |
226 | } |
227 | statbuf = &localStatBuf; |
228 | } |
229 | if (!UTIL_isRegularFileStat(statbuf)) { |
230 | UTIL_TRACE_RET(0); |
231 | return 0; /* pretend success, but don't change anything */ |
232 | } |
233 | #ifdef ZSTD_HAVE_FCHMOD |
234 | if (fd >= 0) { |
235 | int ret; |
236 | UTIL_TRACE_CALL("fchmod"); |
237 | ret = fchmod(fd, permissions); |
238 | UTIL_TRACE_RET(ret); |
239 | UTIL_TRACE_RET(ret); |
240 | return ret; |
241 | } else |
242 | #endif |
243 | { |
244 | int ret; |
245 | UTIL_TRACE_CALL("chmod"); |
246 | ret = chmod(filename, permissions); |
247 | UTIL_TRACE_RET(ret); |
248 | UTIL_TRACE_RET(ret); |
249 | return ret; |
250 | } |
251 | } |
252 | |
253 | /* set access and modification times */ |
254 | int UTIL_utime(const char* filename, const stat_t *statbuf) |
255 | { |
256 | int ret; |
257 | UTIL_TRACE_CALL("UTIL_utime(%s)", filename); |
258 | /* We check that st_mtime is a macro here in order to give us confidence |
259 | * that struct stat has a struct timespec st_mtim member. We need this |
260 | * check because there are some platforms that claim to be POSIX 2008 |
261 | * compliant but which do not have st_mtim... */ |
262 | #if (PLATFORM_POSIX_VERSION >= 200809L) && defined(st_mtime) |
263 | { |
264 | /* (atime, mtime) */ |
265 | struct timespec timebuf[2] = { {0, UTIME_NOW} }; |
266 | timebuf[1] = statbuf->st_mtim; |
267 | ret = utimensat(AT_FDCWD, filename, timebuf, 0); |
268 | } |
269 | #else |
270 | { |
271 | struct utimbuf timebuf; |
272 | timebuf.actime = time(NULL); |
273 | timebuf.modtime = statbuf->st_mtime; |
274 | ret = utime(filename, &timebuf); |
275 | } |
276 | #endif |
277 | errno = 0; |
278 | UTIL_TRACE_RET(ret); |
279 | return ret; |
280 | } |
281 | |
282 | int UTIL_setFileStat(const char *filename, const stat_t *statbuf) |
283 | { |
284 | return UTIL_setFDStat(-1, filename, statbuf); |
285 | } |
286 | |
287 | int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) |
288 | { |
289 | int res = 0; |
290 | stat_t curStatBuf; |
291 | UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); |
292 | |
293 | if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { |
294 | UTIL_TRACE_RET(-1); |
295 | return -1; |
296 | } |
297 | |
298 | /* Mimic gzip's behavior: |
299 | * |
300 | * "Change the group first, then the permissions, then the owner. |
301 | * That way, the permissions will be correct on systems that allow |
302 | * users to give away files, without introducing a security hole. |
303 | * Security depends on permissions not containing the setuid or |
304 | * setgid bits." */ |
305 | |
306 | #if !defined(_WIN32) |
307 | #ifdef ZSTD_HAVE_FCHOWN |
308 | if (fd >= 0) { |
309 | res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ |
310 | } else |
311 | #endif |
312 | { |
313 | res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ |
314 | } |
315 | #endif |
316 | |
317 | res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ |
318 | |
319 | #if !defined(_WIN32) |
320 | #ifdef ZSTD_HAVE_FCHOWN |
321 | if (fd >= 0) { |
322 | res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ |
323 | } else |
324 | #endif |
325 | { |
326 | res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ |
327 | } |
328 | #endif |
329 | |
330 | errno = 0; |
331 | UTIL_TRACE_RET(-res); |
332 | return -res; /* number of errors is returned */ |
333 | } |
334 | |
335 | int UTIL_isDirectory(const char* infilename) |
336 | { |
337 | stat_t statbuf; |
338 | int ret; |
339 | UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); |
340 | ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); |
341 | UTIL_TRACE_RET(ret); |
342 | return ret; |
343 | } |
344 | |
345 | int UTIL_isDirectoryStat(const stat_t* statbuf) |
346 | { |
347 | int ret; |
348 | UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); |
349 | #if defined(_MSC_VER) |
350 | ret = (statbuf->st_mode & _S_IFDIR) != 0; |
351 | #else |
352 | ret = S_ISDIR(statbuf->st_mode) != 0; |
353 | #endif |
354 | UTIL_TRACE_RET(ret); |
355 | return ret; |
356 | } |
357 | |
358 | int UTIL_compareStr(const void *p1, const void *p2) { |
359 | return strcmp(* (char * const *) p1, * (char * const *) p2); |
360 | } |
361 | |
362 | int UTIL_isSameFile(const char* fName1, const char* fName2) |
363 | { |
364 | int ret; |
365 | assert(fName1 != NULL); assert(fName2 != NULL); |
366 | UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); |
367 | #if defined(_MSC_VER) || defined(_WIN32) |
368 | /* note : Visual does not support file identification by inode. |
369 | * inode does not work on Windows, even with a posix layer, like msys2. |
370 | * The following work-around is limited to detecting exact name repetition only, |
371 | * aka `filename` is considered different from `subdir/../filename` */ |
372 | ret = !strcmp(fName1, fName2); |
373 | #else |
374 | { stat_t file1Stat; |
375 | stat_t file2Stat; |
376 | ret = UTIL_stat(fName1, &file1Stat) |
377 | && UTIL_stat(fName2, &file2Stat) |
378 | && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); |
379 | } |
380 | #endif |
381 | UTIL_TRACE_RET(ret); |
382 | return ret; |
383 | } |
384 | |
385 | int UTIL_isSameFileStat( |
386 | const char* fName1, const char* fName2, |
387 | const stat_t* file1Stat, const stat_t* file2Stat) |
388 | { |
389 | int ret; |
390 | assert(fName1 != NULL); assert(fName2 != NULL); |
391 | UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); |
392 | #if defined(_MSC_VER) || defined(_WIN32) |
393 | /* note : Visual does not support file identification by inode. |
394 | * inode does not work on Windows, even with a posix layer, like msys2. |
395 | * The following work-around is limited to detecting exact name repetition only, |
396 | * aka `filename` is considered different from `subdir/../filename` */ |
397 | (void)file1Stat; |
398 | (void)file2Stat; |
399 | ret = !strcmp(fName1, fName2); |
400 | #else |
401 | { |
402 | ret = (file1Stat->st_dev == file2Stat->st_dev) |
403 | && (file1Stat->st_ino == file2Stat->st_ino); |
404 | } |
405 | #endif |
406 | UTIL_TRACE_RET(ret); |
407 | return ret; |
408 | } |
409 | |
410 | /* UTIL_isFIFO : distinguish named pipes */ |
411 | int UTIL_isFIFO(const char* infilename) |
412 | { |
413 | UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); |
414 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
415 | #if PLATFORM_POSIX_VERSION >= 200112L |
416 | { |
417 | stat_t statbuf; |
418 | if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { |
419 | UTIL_TRACE_RET(1); |
420 | return 1; |
421 | } |
422 | } |
423 | #endif |
424 | (void)infilename; |
425 | UTIL_TRACE_RET(0); |
426 | return 0; |
427 | } |
428 | |
429 | /* UTIL_isFIFO : distinguish named pipes */ |
430 | int UTIL_isFIFOStat(const stat_t* statbuf) |
431 | { |
432 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
433 | #if PLATFORM_POSIX_VERSION >= 200112L |
434 | if (S_ISFIFO(statbuf->st_mode)) return 1; |
435 | #endif |
436 | (void)statbuf; |
437 | return 0; |
438 | } |
439 | |
440 | /* UTIL_isBlockDevStat : distinguish named pipes */ |
441 | int UTIL_isBlockDevStat(const stat_t* statbuf) |
442 | { |
443 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
444 | #if PLATFORM_POSIX_VERSION >= 200112L |
445 | if (S_ISBLK(statbuf->st_mode)) return 1; |
446 | #endif |
447 | (void)statbuf; |
448 | return 0; |
449 | } |
450 | |
451 | int UTIL_isLink(const char* infilename) |
452 | { |
453 | UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); |
454 | /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ |
455 | #if PLATFORM_POSIX_VERSION >= 200112L |
456 | { |
457 | stat_t statbuf; |
458 | int const r = lstat(infilename, &statbuf); |
459 | if (!r && S_ISLNK(statbuf.st_mode)) { |
460 | UTIL_TRACE_RET(1); |
461 | return 1; |
462 | } |
463 | } |
464 | #endif |
465 | (void)infilename; |
466 | UTIL_TRACE_RET(0); |
467 | return 0; |
468 | } |
469 | |
470 | static int g_fakeStdinIsConsole = 0; |
471 | static int g_fakeStderrIsConsole = 0; |
472 | static int g_fakeStdoutIsConsole = 0; |
473 | |
474 | int UTIL_isConsole(FILE* file) |
475 | { |
476 | int ret; |
477 | UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); |
478 | if (file == stdin && g_fakeStdinIsConsole) |
479 | ret = 1; |
480 | else if (file == stderr && g_fakeStderrIsConsole) |
481 | ret = 1; |
482 | else if (file == stdout && g_fakeStdoutIsConsole) |
483 | ret = 1; |
484 | else |
485 | ret = IS_CONSOLE(file); |
486 | UTIL_TRACE_RET(ret); |
487 | return ret; |
488 | } |
489 | |
490 | void UTIL_fakeStdinIsConsole(void) |
491 | { |
492 | g_fakeStdinIsConsole = 1; |
493 | } |
494 | void UTIL_fakeStdoutIsConsole(void) |
495 | { |
496 | g_fakeStdoutIsConsole = 1; |
497 | } |
498 | void UTIL_fakeStderrIsConsole(void) |
499 | { |
500 | g_fakeStderrIsConsole = 1; |
501 | } |
502 | |
503 | U64 UTIL_getFileSize(const char* infilename) |
504 | { |
505 | stat_t statbuf; |
506 | UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); |
507 | if (!UTIL_stat(infilename, &statbuf)) { |
508 | UTIL_TRACE_RET(-1); |
509 | return UTIL_FILESIZE_UNKNOWN; |
510 | } |
511 | { |
512 | U64 const size = UTIL_getFileSizeStat(&statbuf); |
513 | UTIL_TRACE_RET((int)size); |
514 | return size; |
515 | } |
516 | } |
517 | |
518 | U64 UTIL_getFileSizeStat(const stat_t* statbuf) |
519 | { |
520 | if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; |
521 | #if defined(_MSC_VER) |
522 | if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; |
523 | #elif defined(__MINGW32__) && defined (__MSVCRT__) |
524 | if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; |
525 | #else |
526 | if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; |
527 | #endif |
528 | return (U64)statbuf->st_size; |
529 | } |
530 | |
531 | UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) |
532 | { |
533 | UTIL_HumanReadableSize_t hrs; |
534 | |
535 | if (g_utilDisplayLevel > 3) { |
536 | /* In verbose mode, do not scale sizes down, except in the case of |
537 | * values that exceed the integral precision of a double. */ |
538 | if (size >= (1ull << 53)) { |
539 | hrs.value = (double)size / (1ull << 20); |
540 | hrs.suffix = " MiB"; |
541 | /* At worst, a double representation of a maximal size will be |
542 | * accurate to better than tens of kilobytes. */ |
543 | hrs.precision = 2; |
544 | } else { |
545 | hrs.value = (double)size; |
546 | hrs.suffix = " B"; |
547 | hrs.precision = 0; |
548 | } |
549 | } else { |
550 | /* In regular mode, scale sizes down and use suffixes. */ |
551 | if (size >= (1ull << 60)) { |
552 | hrs.value = (double)size / (1ull << 60); |
553 | hrs.suffix = " EiB"; |
554 | } else if (size >= (1ull << 50)) { |
555 | hrs.value = (double)size / (1ull << 50); |
556 | hrs.suffix = " PiB"; |
557 | } else if (size >= (1ull << 40)) { |
558 | hrs.value = (double)size / (1ull << 40); |
559 | hrs.suffix = " TiB"; |
560 | } else if (size >= (1ull << 30)) { |
561 | hrs.value = (double)size / (1ull << 30); |
562 | hrs.suffix = " GiB"; |
563 | } else if (size >= (1ull << 20)) { |
564 | hrs.value = (double)size / (1ull << 20); |
565 | hrs.suffix = " MiB"; |
566 | } else if (size >= (1ull << 10)) { |
567 | hrs.value = (double)size / (1ull << 10); |
568 | hrs.suffix = " KiB"; |
569 | } else { |
570 | hrs.value = (double)size; |
571 | hrs.suffix = " B"; |
572 | } |
573 | |
574 | if (hrs.value >= 100 || (U64)hrs.value == size) { |
575 | hrs.precision = 0; |
576 | } else if (hrs.value >= 10) { |
577 | hrs.precision = 1; |
578 | } else if (hrs.value > 1) { |
579 | hrs.precision = 2; |
580 | } else { |
581 | hrs.precision = 3; |
582 | } |
583 | } |
584 | |
585 | return hrs; |
586 | } |
587 | |
588 | U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) |
589 | { |
590 | U64 total = 0; |
591 | unsigned n; |
592 | UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); |
593 | for (n=0; n<nbFiles; n++) { |
594 | U64 const size = UTIL_getFileSize(fileNamesTable[n]); |
595 | if (size == UTIL_FILESIZE_UNKNOWN) { |
596 | UTIL_TRACE_RET(-1); |
597 | return UTIL_FILESIZE_UNKNOWN; |
598 | } |
599 | total += size; |
600 | } |
601 | UTIL_TRACE_RET((int)total); |
602 | return total; |
603 | } |
604 | |
605 | |
606 | /* condition : @file must be valid, and not have reached its end. |
607 | * @return : length of line written into @buf, ended with `\0` instead of '\n', |
608 | * or 0, if there is no new line */ |
609 | static size_t readLineFromFile(char* buf, size_t len, FILE* file) |
610 | { |
611 | assert(!feof(file)); |
612 | if ( fgets(buf, (int) len, file) == NULL ) return 0; |
613 | { size_t linelen = strlen(buf); |
614 | if (strlen(buf)==0) return 0; |
615 | if (buf[linelen-1] == '\n') linelen--; |
616 | buf[linelen] = '\0'; |
617 | return linelen+1; |
618 | } |
619 | } |
620 | |
621 | /* Conditions : |
622 | * size of @inputFileName file must be < @dstCapacity |
623 | * @dst must be initialized |
624 | * @return : nb of lines |
625 | * or -1 if there's an error |
626 | */ |
627 | static int |
628 | readLinesFromFile(void* dst, size_t dstCapacity, |
629 | const char* inputFileName) |
630 | { |
631 | int nbFiles = 0; |
632 | size_t pos = 0; |
633 | char* const buf = (char*)dst; |
634 | FILE* const inputFile = fopen(inputFileName, "r"); |
635 | |
636 | assert(dst != NULL); |
637 | |
638 | if(!inputFile) { |
639 | if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile"); |
640 | return -1; |
641 | } |
642 | |
643 | while ( !feof(inputFile) ) { |
644 | size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); |
645 | if (lineLength == 0) break; |
646 | assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ |
647 | pos += lineLength; |
648 | ++nbFiles; |
649 | } |
650 | |
651 | CONTROL( fclose(inputFile) == 0 ); |
652 | |
653 | return nbFiles; |
654 | } |
655 | |
656 | /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ |
657 | FileNamesTable* |
658 | UTIL_createFileNamesTable_fromFileName(const char* inputFileName) |
659 | { |
660 | size_t nbFiles = 0; |
661 | char* buf; |
662 | size_t bufSize; |
663 | size_t pos = 0; |
664 | stat_t statbuf; |
665 | |
666 | if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) |
667 | return NULL; |
668 | |
669 | { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); |
670 | if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) |
671 | return NULL; |
672 | bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ |
673 | } |
674 | |
675 | buf = (char*) malloc(bufSize); |
676 | CONTROL( buf != NULL ); |
677 | |
678 | { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); |
679 | |
680 | if (ret_nbFiles <= 0) { |
681 | free(buf); |
682 | return NULL; |
683 | } |
684 | nbFiles = (size_t)ret_nbFiles; |
685 | } |
686 | |
687 | { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); |
688 | CONTROL(filenamesTable != NULL); |
689 | |
690 | { size_t fnb; |
691 | for (fnb = 0, pos = 0; fnb < nbFiles; fnb++) { |
692 | filenamesTable[fnb] = buf+pos; |
693 | pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ |
694 | } } |
695 | assert(pos <= bufSize); |
696 | |
697 | return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); |
698 | } |
699 | } |
700 | |
701 | static FileNamesTable* |
702 | UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) |
703 | { |
704 | FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); |
705 | CONTROL(table != NULL); |
706 | table->fileNames = filenames; |
707 | table->buf = buf; |
708 | table->tableSize = tableSize; |
709 | table->tableCapacity = tableCapacity; |
710 | return table; |
711 | } |
712 | |
713 | FileNamesTable* |
714 | UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) |
715 | { |
716 | return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); |
717 | } |
718 | |
719 | void UTIL_freeFileNamesTable(FileNamesTable* table) |
720 | { |
721 | if (table==NULL) return; |
722 | free((void*)table->fileNames); |
723 | free(table->buf); |
724 | free(table); |
725 | } |
726 | |
727 | FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) |
728 | { |
729 | const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); |
730 | FileNamesTable* fnt; |
731 | if (fnTable==NULL) return NULL; |
732 | fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); |
733 | fnt->tableSize = 0; /* the table is empty */ |
734 | return fnt; |
735 | } |
736 | |
737 | int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { |
738 | size_t i; |
739 | for(i=0 ;i < table->tableSize; i++) { |
740 | if(!strcmp(table->fileNames[i], name)) { |
741 | return (int)i; |
742 | } |
743 | } |
744 | return -1; |
745 | } |
746 | |
747 | void UTIL_refFilename(FileNamesTable* fnt, const char* filename) |
748 | { |
749 | assert(fnt->tableSize < fnt->tableCapacity); |
750 | fnt->fileNames[fnt->tableSize] = filename; |
751 | fnt->tableSize++; |
752 | } |
753 | |
754 | static size_t getTotalTableSize(FileNamesTable* table) |
755 | { |
756 | size_t fnb = 0, totalSize = 0; |
757 | for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { |
758 | totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ |
759 | } |
760 | return totalSize; |
761 | } |
762 | |
763 | FileNamesTable* |
764 | UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) |
765 | { |
766 | unsigned newTableIdx = 0; |
767 | size_t pos = 0; |
768 | size_t newTotalTableSize; |
769 | char* buf; |
770 | |
771 | FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); |
772 | CONTROL( newTable != NULL ); |
773 | |
774 | newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); |
775 | |
776 | buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); |
777 | CONTROL ( buf != NULL ); |
778 | |
779 | newTable->buf = buf; |
780 | newTable->tableSize = table1->tableSize + table2->tableSize; |
781 | newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); |
782 | CONTROL ( newTable->fileNames != NULL ); |
783 | |
784 | { unsigned idx1; |
785 | for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { |
786 | size_t const curLen = strlen(table1->fileNames[idx1]); |
787 | memcpy(buf+pos, table1->fileNames[idx1], curLen); |
788 | assert(newTableIdx <= newTable->tableSize); |
789 | newTable->fileNames[newTableIdx] = buf+pos; |
790 | pos += curLen+1; |
791 | } } |
792 | |
793 | { unsigned idx2; |
794 | for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { |
795 | size_t const curLen = strlen(table2->fileNames[idx2]); |
796 | memcpy(buf+pos, table2->fileNames[idx2], curLen); |
797 | assert(newTableIdx < newTable->tableSize); |
798 | newTable->fileNames[newTableIdx] = buf+pos; |
799 | pos += curLen+1; |
800 | } } |
801 | assert(pos <= newTotalTableSize); |
802 | newTable->tableSize = newTableIdx; |
803 | |
804 | UTIL_freeFileNamesTable(table1); |
805 | UTIL_freeFileNamesTable(table2); |
806 | |
807 | return newTable; |
808 | } |
809 | |
810 | #ifdef _WIN32 |
811 | static int UTIL_prepareFileList(const char* dirName, |
812 | char** bufStart, size_t* pos, |
813 | char** bufEnd, int followLinks) |
814 | { |
815 | char* path; |
816 | size_t dirLength, pathLength; |
817 | int nbFiles = 0; |
818 | WIN32_FIND_DATAA cFile; |
819 | HANDLE hFile; |
820 | |
821 | dirLength = strlen(dirName); |
822 | path = (char*) malloc(dirLength + 3); |
823 | if (!path) return 0; |
824 | |
825 | memcpy(path, dirName, dirLength); |
826 | path[dirLength] = '\\'; |
827 | path[dirLength+1] = '*'; |
828 | path[dirLength+2] = 0; |
829 | |
830 | hFile=FindFirstFileA(path, &cFile); |
831 | if (hFile == INVALID_HANDLE_VALUE) { |
832 | UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); |
833 | return 0; |
834 | } |
835 | free(path); |
836 | |
837 | do { |
838 | size_t const fnameLength = strlen(cFile.cFileName); |
839 | path = (char*) malloc(dirLength + fnameLength + 2); |
840 | if (!path) { FindClose(hFile); return 0; } |
841 | memcpy(path, dirName, dirLength); |
842 | path[dirLength] = '\\'; |
843 | memcpy(path+dirLength+1, cFile.cFileName, fnameLength); |
844 | pathLength = dirLength+1+fnameLength; |
845 | path[pathLength] = 0; |
846 | if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { |
847 | if ( strcmp (cFile.cFileName, "..") == 0 |
848 | || strcmp (cFile.cFileName, ".") == 0 ) |
849 | continue; |
850 | /* Recursively call "UTIL_prepareFileList" with the new path. */ |
851 | nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); |
852 | if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } |
853 | } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) |
854 | || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) |
855 | || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { |
856 | if (*bufStart + *pos + pathLength >= *bufEnd) { |
857 | ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; |
858 | *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); |
859 | if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } |
860 | *bufEnd = *bufStart + newListSize; |
861 | } |
862 | if (*bufStart + *pos + pathLength < *bufEnd) { |
863 | memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); |
864 | *pos += pathLength + 1; |
865 | nbFiles++; |
866 | } } |
867 | free(path); |
868 | } while (FindNextFileA(hFile, &cFile)); |
869 | |
870 | FindClose(hFile); |
871 | return nbFiles; |
872 | } |
873 | |
874 | #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ |
875 | |
876 | static int UTIL_prepareFileList(const char *dirName, |
877 | char** bufStart, size_t* pos, |
878 | char** bufEnd, int followLinks) |
879 | { |
880 | DIR* dir; |
881 | struct dirent * entry; |
882 | size_t dirLength; |
883 | int nbFiles = 0; |
884 | |
885 | if (!(dir = opendir(dirName))) { |
886 | UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); |
887 | return 0; |
888 | } |
889 | |
890 | dirLength = strlen(dirName); |
891 | errno = 0; |
892 | while ((entry = readdir(dir)) != NULL) { |
893 | char* path; |
894 | size_t fnameLength, pathLength; |
895 | if (strcmp (entry->d_name, "..") == 0 || |
896 | strcmp (entry->d_name, ".") == 0) continue; |
897 | fnameLength = strlen(entry->d_name); |
898 | path = (char*) malloc(dirLength + fnameLength + 2); |
899 | if (!path) { closedir(dir); return 0; } |
900 | memcpy(path, dirName, dirLength); |
901 | |
902 | path[dirLength] = '/'; |
903 | memcpy(path+dirLength+1, entry->d_name, fnameLength); |
904 | pathLength = dirLength+1+fnameLength; |
905 | path[pathLength] = 0; |
906 | |
907 | if (!followLinks && UTIL_isLink(path)) { |
908 | UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); |
909 | free(path); |
910 | continue; |
911 | } |
912 | |
913 | if (UTIL_isDirectory(path)) { |
914 | nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ |
915 | if (*bufStart == NULL) { free(path); closedir(dir); return 0; } |
916 | } else { |
917 | if (*bufStart + *pos + pathLength >= *bufEnd) { |
918 | ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; |
919 | assert(newListSize >= 0); |
920 | *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); |
921 | if (*bufStart != NULL) { |
922 | *bufEnd = *bufStart + newListSize; |
923 | } else { |
924 | free(path); closedir(dir); return 0; |
925 | } |
926 | } |
927 | if (*bufStart + *pos + pathLength < *bufEnd) { |
928 | memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ |
929 | *pos += pathLength + 1; |
930 | nbFiles++; |
931 | } } |
932 | free(path); |
933 | errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ |
934 | } |
935 | |
936 | if (errno != 0) { |
937 | UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); |
938 | free(*bufStart); |
939 | *bufStart = NULL; |
940 | } |
941 | closedir(dir); |
942 | return nbFiles; |
943 | } |
944 | |
945 | #else |
946 | |
947 | static int UTIL_prepareFileList(const char *dirName, |
948 | char** bufStart, size_t* pos, |
949 | char** bufEnd, int followLinks) |
950 | { |
951 | (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; |
952 | UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); |
953 | return 0; |
954 | } |
955 | |
956 | #endif /* #ifdef _WIN32 */ |
957 | |
958 | int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) |
959 | { |
960 | const char* ext = UTIL_getFileExtension(inputName); |
961 | while(*extensionList!=NULL) |
962 | { |
963 | const int isCompressedExtension = strcmp(ext,*extensionList); |
964 | if(isCompressedExtension==0) |
965 | return 1; |
966 | ++extensionList; |
967 | } |
968 | return 0; |
969 | } |
970 | |
971 | /*Utility function to get file extension from file */ |
972 | const char* UTIL_getFileExtension(const char* infilename) |
973 | { |
974 | const char* extension = strrchr(infilename, '.'); |
975 | if(!extension || extension==infilename) return ""; |
976 | return extension; |
977 | } |
978 | |
979 | static int pathnameHas2Dots(const char *pathname) |
980 | { |
981 | /* We need to figure out whether any ".." present in the path is a whole |
982 | * path token, which is the case if it is bordered on both sides by either |
983 | * the beginning/end of the path or by a directory separator. |
984 | */ |
985 | const char *needle = pathname; |
986 | while (1) { |
987 | needle = strstr(needle, ".."); |
988 | |
989 | if (needle == NULL) { |
990 | return 0; |
991 | } |
992 | |
993 | if ((needle == pathname || needle[-1] == PATH_SEP) |
994 | && (needle[2] == '\0' || needle[2] == PATH_SEP)) { |
995 | return 1; |
996 | } |
997 | |
998 | /* increment so we search for the next match */ |
999 | needle++; |
1000 | }; |
1001 | return 0; |
1002 | } |
1003 | |
1004 | static int isFileNameValidForMirroredOutput(const char *filename) |
1005 | { |
1006 | return !pathnameHas2Dots(filename); |
1007 | } |
1008 | |
1009 | |
1010 | #define DIR_DEFAULT_MODE 0755 |
1011 | static mode_t getDirMode(const char *dirName) |
1012 | { |
1013 | stat_t st; |
1014 | if (!UTIL_stat(dirName, &st)) { |
1015 | UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); |
1016 | return DIR_DEFAULT_MODE; |
1017 | } |
1018 | if (!UTIL_isDirectoryStat(&st)) { |
1019 | UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); |
1020 | return DIR_DEFAULT_MODE; |
1021 | } |
1022 | return st.st_mode; |
1023 | } |
1024 | |
1025 | static int makeDir(const char *dir, mode_t mode) |
1026 | { |
1027 | #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) |
1028 | int ret = _mkdir(dir); |
1029 | (void) mode; |
1030 | #else |
1031 | int ret = mkdir(dir, mode); |
1032 | #endif |
1033 | if (ret != 0) { |
1034 | if (errno == EEXIST) |
1035 | return 0; |
1036 | UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); |
1037 | } |
1038 | return ret; |
1039 | } |
1040 | |
1041 | /* this function requires a mutable input string */ |
1042 | static void convertPathnameToDirName(char *pathname) |
1043 | { |
1044 | size_t len = 0; |
1045 | char* pos = NULL; |
1046 | /* get dir name from pathname similar to 'dirname()' */ |
1047 | assert(pathname != NULL); |
1048 | |
1049 | /* remove trailing '/' chars */ |
1050 | len = strlen(pathname); |
1051 | assert(len > 0); |
1052 | while (pathname[len] == PATH_SEP) { |
1053 | pathname[len] = '\0'; |
1054 | len--; |
1055 | } |
1056 | if (len == 0) return; |
1057 | |
1058 | /* if input is a single file, return '.' instead. i.e. |
1059 | * "xyz/abc/file.txt" => "xyz/abc" |
1060 | "./file.txt" => "." |
1061 | "file.txt" => "." |
1062 | */ |
1063 | pos = strrchr(pathname, PATH_SEP); |
1064 | if (pos == NULL) { |
1065 | pathname[0] = '.'; |
1066 | pathname[1] = '\0'; |
1067 | } else { |
1068 | *pos = '\0'; |
1069 | } |
1070 | } |
1071 | |
1072 | /* pathname must be valid */ |
1073 | static const char* trimLeadingRootChar(const char *pathname) |
1074 | { |
1075 | assert(pathname != NULL); |
1076 | if (pathname[0] == PATH_SEP) |
1077 | return pathname + 1; |
1078 | return pathname; |
1079 | } |
1080 | |
1081 | /* pathname must be valid */ |
1082 | static const char* trimLeadingCurrentDirConst(const char *pathname) |
1083 | { |
1084 | assert(pathname != NULL); |
1085 | if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) |
1086 | return pathname + 2; |
1087 | return pathname; |
1088 | } |
1089 | |
1090 | static char* |
1091 | trimLeadingCurrentDir(char *pathname) |
1092 | { |
1093 | /* 'union charunion' can do const-cast without compiler warning */ |
1094 | union charunion { |
1095 | char *chr; |
1096 | const char* cchr; |
1097 | } ptr; |
1098 | ptr.cchr = trimLeadingCurrentDirConst(pathname); |
1099 | return ptr.chr; |
1100 | } |
1101 | |
1102 | /* remove leading './' or '/' chars here */ |
1103 | static const char * trimPath(const char *pathname) |
1104 | { |
1105 | return trimLeadingRootChar( |
1106 | trimLeadingCurrentDirConst(pathname)); |
1107 | } |
1108 | |
1109 | static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) |
1110 | { |
1111 | assert(dir1 != NULL && dir2 != NULL); |
1112 | { const size_t dir1Size = strlen(dir1); |
1113 | const size_t dir2Size = strlen(dir2); |
1114 | char *outDirBuffer, *buffer; |
1115 | |
1116 | outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); |
1117 | CONTROL(outDirBuffer != NULL); |
1118 | |
1119 | memcpy(outDirBuffer, dir1, dir1Size); |
1120 | outDirBuffer[dir1Size] = '\0'; |
1121 | |
1122 | if (dir2[0] == '.') |
1123 | return outDirBuffer; |
1124 | |
1125 | buffer = outDirBuffer + dir1Size; |
1126 | if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { |
1127 | *buffer = PATH_SEP; |
1128 | buffer++; |
1129 | } |
1130 | memcpy(buffer, dir2, dir2Size); |
1131 | buffer[dir2Size] = '\0'; |
1132 | |
1133 | return outDirBuffer; |
1134 | } |
1135 | } |
1136 | |
1137 | /* this function will return NULL if input srcFileName is not valid name for mirrored output path */ |
1138 | char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) |
1139 | { |
1140 | char* pathname = NULL; |
1141 | if (!isFileNameValidForMirroredOutput(srcFileName)) |
1142 | return NULL; |
1143 | |
1144 | pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); |
1145 | |
1146 | convertPathnameToDirName(pathname); |
1147 | return pathname; |
1148 | } |
1149 | |
1150 | static int |
1151 | mirrorSrcDir(char* srcDirName, const char* outDirName) |
1152 | { |
1153 | mode_t srcMode; |
1154 | int status = 0; |
1155 | char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); |
1156 | if (!newDir) |
1157 | return -ENOMEM; |
1158 | |
1159 | srcMode = getDirMode(srcDirName); |
1160 | status = makeDir(newDir, srcMode); |
1161 | free(newDir); |
1162 | return status; |
1163 | } |
1164 | |
1165 | static int |
1166 | mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) |
1167 | { |
1168 | int status = 0; |
1169 | char* pp = trimLeadingCurrentDir(srcDirName); |
1170 | char* sp = NULL; |
1171 | |
1172 | while ((sp = strchr(pp, PATH_SEP)) != NULL) { |
1173 | if (sp != pp) { |
1174 | *sp = '\0'; |
1175 | status = mirrorSrcDir(srcDirName, outDirName); |
1176 | if (status != 0) |
1177 | return status; |
1178 | *sp = PATH_SEP; |
1179 | } |
1180 | pp = sp + 1; |
1181 | } |
1182 | status = mirrorSrcDir(srcDirName, outDirName); |
1183 | return status; |
1184 | } |
1185 | |
1186 | static void |
1187 | makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) |
1188 | { |
1189 | unsigned int i = 0; |
1190 | for (i = 0; i < nbFile; i++) |
1191 | mirrorSrcDirRecursive(srcDirNames[i], outDirName); |
1192 | } |
1193 | |
1194 | static int |
1195 | firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) |
1196 | { |
1197 | size_t firstDirLen = strlen(firstDir), |
1198 | secondDirLen = strlen(secondDir); |
1199 | return firstDirLen <= secondDirLen && |
1200 | (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && |
1201 | 0 == strncmp(firstDir, secondDir, firstDirLen); |
1202 | } |
1203 | |
1204 | static int compareDir(const void* pathname1, const void* pathname2) { |
1205 | /* sort it after remove the leading '/' or './'*/ |
1206 | const char* s1 = trimPath(*(char * const *) pathname1); |
1207 | const char* s2 = trimPath(*(char * const *) pathname2); |
1208 | return strcmp(s1, s2); |
1209 | } |
1210 | |
1211 | static void |
1212 | makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) |
1213 | { |
1214 | unsigned int i = 0, uniqueDirNr = 0; |
1215 | char** uniqueDirNames = NULL; |
1216 | |
1217 | if (nbFile == 0) |
1218 | return; |
1219 | |
1220 | uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); |
1221 | CONTROL(uniqueDirNames != NULL); |
1222 | |
1223 | /* if dirs is "a/b/c" and "a/b/c/d", we only need call: |
1224 | * we just need "a/b/c/d" */ |
1225 | qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); |
1226 | |
1227 | uniqueDirNr = 1; |
1228 | uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; |
1229 | for (i = 1; i < nbFile; i++) { |
1230 | char* prevDirName = srcDirNames[i - 1]; |
1231 | char* currDirName = srcDirNames[i]; |
1232 | |
1233 | /* note: we always compare trimmed path, i.e.: |
1234 | * src dir of "./foo" and "/foo" will be both saved into: |
1235 | * "outDirName/foo/" */ |
1236 | if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), |
1237 | trimPath(currDirName))) |
1238 | uniqueDirNr++; |
1239 | |
1240 | /* we need to maintain original src dir name instead of trimmed |
1241 | * dir, so we can retrieve the original src dir's mode_t */ |
1242 | uniqueDirNames[uniqueDirNr - 1] = currDirName; |
1243 | } |
1244 | |
1245 | makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); |
1246 | |
1247 | free(uniqueDirNames); |
1248 | } |
1249 | |
1250 | static void |
1251 | makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) |
1252 | { |
1253 | unsigned int i = 0; |
1254 | for (i = 0; i < nbFile; ++i) |
1255 | convertPathnameToDirName(srcFileNames[i]); |
1256 | makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); |
1257 | } |
1258 | |
1259 | void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) |
1260 | { |
1261 | unsigned int i = 0, validFilenamesNr = 0; |
1262 | char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); |
1263 | CONTROL(srcFileNames != NULL); |
1264 | |
1265 | /* check input filenames is valid */ |
1266 | for (i = 0; i < nbFile; ++i) { |
1267 | if (isFileNameValidForMirroredOutput(inFileNames[i])) { |
1268 | char* fname = STRDUP(inFileNames[i]); |
1269 | CONTROL(fname != NULL); |
1270 | srcFileNames[validFilenamesNr++] = fname; |
1271 | } |
1272 | } |
1273 | |
1274 | if (validFilenamesNr > 0) { |
1275 | makeDir(outDirName, DIR_DEFAULT_MODE); |
1276 | makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); |
1277 | } |
1278 | |
1279 | for (i = 0; i < validFilenamesNr; i++) |
1280 | free(srcFileNames[i]); |
1281 | free(srcFileNames); |
1282 | } |
1283 | |
1284 | FileNamesTable* |
1285 | UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) |
1286 | { |
1287 | unsigned nbFiles; |
1288 | char* buf = (char*)malloc(LIST_SIZE_INCREASE); |
1289 | char* bufend = buf + LIST_SIZE_INCREASE; |
1290 | |
1291 | if (!buf) return NULL; |
1292 | |
1293 | { size_t ifnNb, pos; |
1294 | for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) { |
1295 | if (!UTIL_isDirectory(inputNames[ifnNb])) { |
1296 | size_t const len = strlen(inputNames[ifnNb]); |
1297 | if (buf + pos + len >= bufend) { |
1298 | ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; |
1299 | assert(newListSize >= 0); |
1300 | buf = (char*)UTIL_realloc(buf, (size_t)newListSize); |
1301 | if (!buf) return NULL; |
1302 | bufend = buf + newListSize; |
1303 | } |
1304 | if (buf + pos + len < bufend) { |
1305 | memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ |
1306 | pos += len + 1; |
1307 | nbFiles++; |
1308 | } |
1309 | } else { |
1310 | nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); |
1311 | if (buf == NULL) return NULL; |
1312 | } } } |
1313 | |
1314 | /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ |
1315 | |
1316 | { size_t ifnNb, pos; |
1317 | size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ |
1318 | const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); |
1319 | if (!fileNamesTable) { free(buf); return NULL; } |
1320 | |
1321 | for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { |
1322 | fileNamesTable[ifnNb] = buf + pos; |
1323 | if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } |
1324 | pos += strlen(fileNamesTable[ifnNb]) + 1; |
1325 | } |
1326 | return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); |
1327 | } |
1328 | } |
1329 | |
1330 | |
1331 | void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) |
1332 | { |
1333 | FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); |
1334 | CONTROL(newFNT != NULL); |
1335 | UTIL_freeFileNamesTable(*fnt); |
1336 | *fnt = newFNT; |
1337 | } |
1338 | |
1339 | FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) |
1340 | { |
1341 | size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); |
1342 | const char** const newFNTable = (const char**)malloc(sizeof_FNTable); |
1343 | if (newFNTable==NULL) return NULL; |
1344 | memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ |
1345 | return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); |
1346 | } |
1347 | |
1348 | |
1349 | /*-**************************************** |
1350 | * count the number of cores |
1351 | ******************************************/ |
1352 | |
1353 | #if defined(_WIN32) || defined(WIN32) |
1354 | |
1355 | #include <windows.h> |
1356 | |
1357 | typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); |
1358 | |
1359 | DWORD CountSetBits(ULONG_PTR bitMask) |
1360 | { |
1361 | DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; |
1362 | DWORD bitSetCount = 0; |
1363 | ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; |
1364 | DWORD i; |
1365 | |
1366 | for (i = 0; i <= LSHIFT; ++i) |
1367 | { |
1368 | bitSetCount += ((bitMask & bitTest)?1:0); |
1369 | bitTest/=2; |
1370 | } |
1371 | |
1372 | return bitSetCount; |
1373 | } |
1374 | |
1375 | int UTIL_countCores(int logical) |
1376 | { |
1377 | static int numCores = 0; |
1378 | if (numCores != 0) return numCores; |
1379 | |
1380 | { LPFN_GLPI glpi; |
1381 | BOOL done = FALSE; |
1382 | PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; |
1383 | PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; |
1384 | DWORD returnLength = 0; |
1385 | size_t byteOffset = 0; |
1386 | |
1387 | #if defined(_MSC_VER) |
1388 | /* Visual Studio does not like the following cast */ |
1389 | # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ |
1390 | # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ |
1391 | #endif |
1392 | glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), |
1393 | "GetLogicalProcessorInformation"); |
1394 | |
1395 | if (glpi == NULL) { |
1396 | goto failed; |
1397 | } |
1398 | |
1399 | while(!done) { |
1400 | DWORD rc = glpi(buffer, &returnLength); |
1401 | if (FALSE == rc) { |
1402 | if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { |
1403 | if (buffer) |
1404 | free(buffer); |
1405 | buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); |
1406 | |
1407 | if (buffer == NULL) { |
1408 | perror("zstd"); |
1409 | exit(1); |
1410 | } |
1411 | } else { |
1412 | /* some other error */ |
1413 | goto failed; |
1414 | } |
1415 | } else { |
1416 | done = TRUE; |
1417 | } } |
1418 | |
1419 | ptr = buffer; |
1420 | |
1421 | while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { |
1422 | |
1423 | if (ptr->Relationship == RelationProcessorCore) { |
1424 | if (logical) |
1425 | numCores += CountSetBits(ptr->ProcessorMask); |
1426 | else |
1427 | numCores++; |
1428 | } |
1429 | |
1430 | ptr++; |
1431 | byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); |
1432 | } |
1433 | |
1434 | free(buffer); |
1435 | |
1436 | return numCores; |
1437 | } |
1438 | |
1439 | failed: |
1440 | /* try to fall back on GetSystemInfo */ |
1441 | { SYSTEM_INFO sysinfo; |
1442 | GetSystemInfo(&sysinfo); |
1443 | numCores = sysinfo.dwNumberOfProcessors; |
1444 | if (numCores == 0) numCores = 1; /* just in case */ |
1445 | } |
1446 | return numCores; |
1447 | } |
1448 | |
1449 | #elif defined(__APPLE__) |
1450 | |
1451 | #include <sys/sysctl.h> |
1452 | |
1453 | /* Use apple-provided syscall |
1454 | * see: man 3 sysctl */ |
1455 | int UTIL_countCores(int logical) |
1456 | { |
1457 | static S32 numCores = 0; /* apple specifies int32_t */ |
1458 | if (numCores != 0) return numCores; |
1459 | |
1460 | { size_t size = sizeof(S32); |
1461 | int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); |
1462 | if (ret != 0) { |
1463 | if (errno == ENOENT) { |
1464 | /* entry not present, fall back on 1 */ |
1465 | numCores = 1; |
1466 | } else { |
1467 | perror("zstd: can't get number of cpus"); |
1468 | exit(1); |
1469 | } |
1470 | } |
1471 | |
1472 | return numCores; |
1473 | } |
1474 | } |
1475 | |
1476 | #elif defined(__linux__) |
1477 | |
1478 | /* parse /proc/cpuinfo |
1479 | * siblings / cpu cores should give hyperthreading ratio |
1480 | * otherwise fall back on sysconf */ |
1481 | int UTIL_countCores(int logical) |
1482 | { |
1483 | static int numCores = 0; |
1484 | |
1485 | if (numCores != 0) return numCores; |
1486 | |
1487 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1488 | if (numCores == -1) { |
1489 | /* value not queryable, fall back on 1 */ |
1490 | return numCores = 1; |
1491 | } |
1492 | |
1493 | /* try to determine if there's hyperthreading */ |
1494 | { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); |
1495 | #define BUF_SIZE 80 |
1496 | char buff[BUF_SIZE]; |
1497 | |
1498 | int siblings = 0; |
1499 | int cpu_cores = 0; |
1500 | int ratio = 1; |
1501 | |
1502 | if (cpuinfo == NULL) { |
1503 | /* fall back on the sysconf value */ |
1504 | return numCores; |
1505 | } |
1506 | |
1507 | /* assume the cpu cores/siblings values will be constant across all |
1508 | * present processors */ |
1509 | while (!feof(cpuinfo)) { |
1510 | if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { |
1511 | if (strncmp(buff, "siblings", 8) == 0) { |
1512 | const char* const sep = strchr(buff, ':'); |
1513 | if (sep == NULL || *sep == '\0') { |
1514 | /* formatting was broken? */ |
1515 | goto failed; |
1516 | } |
1517 | |
1518 | siblings = atoi(sep + 1); |
1519 | } |
1520 | if (strncmp(buff, "cpu cores", 9) == 0) { |
1521 | const char* const sep = strchr(buff, ':'); |
1522 | if (sep == NULL || *sep == '\0') { |
1523 | /* formatting was broken? */ |
1524 | goto failed; |
1525 | } |
1526 | |
1527 | cpu_cores = atoi(sep + 1); |
1528 | } |
1529 | } else if (ferror(cpuinfo)) { |
1530 | /* fall back on the sysconf value */ |
1531 | goto failed; |
1532 | } } |
1533 | if (siblings && cpu_cores && siblings > cpu_cores) { |
1534 | ratio = siblings / cpu_cores; |
1535 | } |
1536 | |
1537 | if (ratio && numCores > ratio && !logical) { |
1538 | numCores = numCores / ratio; |
1539 | } |
1540 | |
1541 | failed: |
1542 | fclose(cpuinfo); |
1543 | return numCores; |
1544 | } |
1545 | } |
1546 | |
1547 | #elif defined(__FreeBSD__) |
1548 | |
1549 | #include <sys/param.h> |
1550 | #include <sys/sysctl.h> |
1551 | |
1552 | /* Use physical core sysctl when available |
1553 | * see: man 4 smp, man 3 sysctl */ |
1554 | int UTIL_countCores(int logical) |
1555 | { |
1556 | static int numCores = 0; /* freebsd sysctl is native int sized */ |
1557 | #if __FreeBSD_version >= 1300008 |
1558 | static int perCore = 1; |
1559 | #endif |
1560 | if (numCores != 0) return numCores; |
1561 | |
1562 | #if __FreeBSD_version >= 1300008 |
1563 | { size_t size = sizeof(numCores); |
1564 | int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); |
1565 | if (ret == 0) { |
1566 | if (logical) { |
1567 | ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); |
1568 | /* default to physical cores if logical cannot be read */ |
1569 | if (ret == 0) |
1570 | numCores *= perCore; |
1571 | } |
1572 | |
1573 | return numCores; |
1574 | } |
1575 | if (errno != ENOENT) { |
1576 | perror("zstd: can't get number of cpus"); |
1577 | exit(1); |
1578 | } |
1579 | /* sysctl not present, fall through to older sysconf method */ |
1580 | } |
1581 | #else |
1582 | /* suppress unused parameter warning */ |
1583 | (void) logical; |
1584 | #endif |
1585 | |
1586 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1587 | if (numCores == -1) { |
1588 | /* value not queryable, fall back on 1 */ |
1589 | numCores = 1; |
1590 | } |
1591 | return numCores; |
1592 | } |
1593 | |
1594 | #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) |
1595 | |
1596 | /* Use POSIX sysconf |
1597 | * see: man 3 sysconf */ |
1598 | int UTIL_countCores(int logical) |
1599 | { |
1600 | static int numCores = 0; |
1601 | |
1602 | /* suppress unused parameter warning */ |
1603 | (void)logical; |
1604 | |
1605 | if (numCores != 0) return numCores; |
1606 | |
1607 | numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); |
1608 | if (numCores == -1) { |
1609 | /* value not queryable, fall back on 1 */ |
1610 | return numCores = 1; |
1611 | } |
1612 | return numCores; |
1613 | } |
1614 | |
1615 | #else |
1616 | |
1617 | int UTIL_countCores(int logical) |
1618 | { |
1619 | /* suppress unused parameter warning */ |
1620 | (void)logical; |
1621 | |
1622 | /* assume 1 */ |
1623 | return 1; |
1624 | } |
1625 | |
1626 | #endif |
1627 | |
1628 | int UTIL_countPhysicalCores(void) |
1629 | { |
1630 | return UTIL_countCores(0); |
1631 | } |
1632 | |
1633 | int UTIL_countLogicalCores(void) |
1634 | { |
1635 | return UTIL_countCores(1); |
1636 | } |
1637 | |
1638 | #if defined (__cplusplus) |
1639 | } |
1640 | #endif |