648db22b |
1 | /* |
2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | */ |
9 | #include "Pzstd.h" |
10 | extern "C" { |
11 | #include "datagen.h" |
12 | } |
13 | #include "test/RoundTrip.h" |
14 | #include "utils/ScopeGuard.h" |
15 | |
16 | #include <cstddef> |
17 | #include <cstdio> |
18 | #include <gtest/gtest.h> |
19 | #include <memory> |
20 | #include <random> |
21 | |
22 | using namespace std; |
23 | using namespace pzstd; |
24 | |
25 | TEST(Pzstd, SmallSizes) { |
26 | unsigned seed = std::random_device{}(); |
27 | std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); |
28 | std::mt19937 gen(seed); |
29 | |
30 | for (unsigned len = 1; len < 256; ++len) { |
31 | if (len % 16 == 0) { |
32 | std::fprintf(stderr, "%u / 16\n", len / 16); |
33 | } |
34 | std::string inputFile = std::tmpnam(nullptr); |
35 | auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); |
36 | { |
37 | static uint8_t buf[256]; |
38 | RDG_genBuffer(buf, len, 0.5, 0.0, gen()); |
39 | auto fd = std::fopen(inputFile.c_str(), "wb"); |
40 | auto written = std::fwrite(buf, 1, len, fd); |
41 | std::fclose(fd); |
42 | ASSERT_EQ(written, len); |
43 | } |
44 | for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { |
45 | for (unsigned level = 1; level <= 4; level *= 4) { |
46 | auto errorGuard = makeScopeGuard([&] { |
47 | std::fprintf(stderr, "# threads: %u\n", numThreads); |
48 | std::fprintf(stderr, "compression level: %u\n", level); |
49 | }); |
50 | Options options; |
51 | options.overwrite = true; |
52 | options.inputFiles = {inputFile}; |
53 | options.numThreads = numThreads; |
54 | options.compressionLevel = level; |
55 | options.verbosity = 1; |
56 | ASSERT_TRUE(roundTrip(options)); |
57 | errorGuard.dismiss(); |
58 | } |
59 | } |
60 | } |
61 | } |
62 | |
63 | TEST(Pzstd, LargeSizes) { |
64 | unsigned seed = std::random_device{}(); |
65 | std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed); |
66 | std::mt19937 gen(seed); |
67 | |
68 | for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) { |
69 | std::string inputFile = std::tmpnam(nullptr); |
70 | auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); |
71 | { |
72 | std::unique_ptr<uint8_t[]> buf(new uint8_t[len]); |
73 | RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen()); |
74 | auto fd = std::fopen(inputFile.c_str(), "wb"); |
75 | auto written = std::fwrite(buf.get(), 1, len, fd); |
76 | std::fclose(fd); |
77 | ASSERT_EQ(written, len); |
78 | } |
79 | for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { |
80 | for (unsigned level = 1; level <= 4; level *= 4) { |
81 | auto errorGuard = makeScopeGuard([&] { |
82 | std::fprintf(stderr, "# threads: %u\n", numThreads); |
83 | std::fprintf(stderr, "compression level: %u\n", level); |
84 | }); |
85 | Options options; |
86 | options.overwrite = true; |
87 | options.inputFiles = {inputFile}; |
88 | options.numThreads = std::min(numThreads, options.numThreads); |
89 | options.compressionLevel = level; |
90 | options.verbosity = 1; |
91 | ASSERT_TRUE(roundTrip(options)); |
92 | errorGuard.dismiss(); |
93 | } |
94 | } |
95 | } |
96 | } |
97 | |
98 | TEST(Pzstd, DISABLED_ExtremelyLargeSize) { |
99 | unsigned seed = std::random_device{}(); |
100 | std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); |
101 | std::mt19937 gen(seed); |
102 | |
103 | std::string inputFile = std::tmpnam(nullptr); |
104 | auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); |
105 | |
106 | { |
107 | // Write 4GB + 64 MB |
108 | constexpr size_t kLength = 1 << 26; |
109 | std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]); |
110 | auto fd = std::fopen(inputFile.c_str(), "wb"); |
111 | auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); |
112 | for (size_t i = 0; i < (1 << 6) + 1; ++i) { |
113 | RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); |
114 | auto written = std::fwrite(buf.get(), 1, kLength, fd); |
115 | if (written != kLength) { |
116 | std::fprintf(stderr, "Failed to write file, skipping test\n"); |
117 | return; |
118 | } |
119 | } |
120 | } |
121 | |
122 | Options options; |
123 | options.overwrite = true; |
124 | options.inputFiles = {inputFile}; |
125 | options.compressionLevel = 1; |
126 | if (options.numThreads == 0) { |
127 | options.numThreads = 1; |
128 | } |
129 | ASSERT_TRUE(roundTrip(options)); |
130 | } |
131 | |
132 | TEST(Pzstd, ExtremelyCompressible) { |
133 | std::string inputFile = std::tmpnam(nullptr); |
134 | auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); |
135 | { |
136 | std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]); |
137 | std::memset(buf.get(), 'a', 10000); |
138 | auto fd = std::fopen(inputFile.c_str(), "wb"); |
139 | auto written = std::fwrite(buf.get(), 1, 10000, fd); |
140 | std::fclose(fd); |
141 | ASSERT_EQ(written, 10000); |
142 | } |
143 | Options options; |
144 | options.overwrite = true; |
145 | options.inputFiles = {inputFile}; |
146 | options.numThreads = 1; |
147 | options.compressionLevel = 1; |
148 | ASSERT_TRUE(roundTrip(options)); |
149 | } |