1 // Copyright 2017 The Chromium OS Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <unistd.h> 6 7 #include <vector> 8 9 #include "gtest/gtest.h" 10 11 #include "puffin/src/file_stream.h" 12 #include "puffin/src/include/puffin/common.h" 13 #include "puffin/src/include/puffin/utils.h" 14 #include "puffin/src/memory_stream.h" 15 #include "puffin/src/unittest_common.h" 16 17 using std::string; 18 using std::vector; 19 20 namespace puffin { 21 22 namespace { 23 const uint8_t kZipEntries[] = { 24 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x02, 0x00, 0x08, 0x00, 0xfc, 0x88, 25 0x28, 0x4c, 0xcb, 0x86, 0xe1, 0x80, 0x06, 0x00, 0x00, 0x00, 0x09, 0x00, 26 0x00, 0x00, 0x01, 0x00, 0x1c, 0x00, 0x31, 0x55, 0x54, 0x09, 0x00, 0x03, 27 0xec, 0x15, 0x54, 0x5a, 0x49, 0x10, 0x54, 0x5a, 0x75, 0x78, 0x0b, 0x00, 28 0x01, 0x04, 0x8f, 0x66, 0x05, 0x00, 0x04, 0x88, 0x13, 0x00, 0x00, 0x33, 29 0x34, 0x84, 0x00, 0x2e, 0x00, 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x02, 30 0x00, 0x08, 0x00, 0x01, 0x89, 0x28, 0x4c, 0xe0, 0xe8, 0x6f, 0x6d, 0x06, 31 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x1c, 0x00, 0x32, 32 0x55, 0x54, 0x09, 0x00, 0x03, 0xf1, 0x15, 0x54, 0x5a, 0x38, 0x10, 0x54, 33 0x5a, 0x75, 0x78, 0x0b, 0x00, 0x01, 0x04, 0x8f, 0x66, 0x05, 0x00, 0x04, 34 0x88, 0x13, 0x00, 0x00, 0x33, 0x32, 0x82, 0x01, 0x2e, 0x00}; 35 36 // (echo "666666" > 2 && zip -fd test.zip 2 && 37 // cat test.zip | hexdump -v -e '10/1 "0x%02x, " "\n"') 38 const uint8_t kZipEntryWithDataDescriptor[] = { 39 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x00, 0x08, 0x00, 0x0b, 0x74, 40 0x2b, 0x4c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 41 0x00, 0x00, 0x01, 0x00, 0x1c, 0x00, 0x32, 0x55, 0x54, 0x09, 0x00, 0x03, 42 0xf5, 0xe5, 0x57, 0x5a, 0xf2, 0xe5, 0x57, 0x5a, 0x75, 0x78, 0x0b, 0x00, 43 0x01, 0x04, 0x8f, 0x66, 0x05, 0x00, 0x04, 0x88, 0x13, 0x00, 0x00, 0x33, 44 0x33, 0x03, 0x01, 0x2e, 0x00, 0x50, 0x4b, 0x07, 0x08, 0xb4, 0xa0, 0xf2, 45 0x36, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x50, 0x4b, 0x03, 46 0x04, 0x14, 0x00, 0x08, 0x00, 0x08, 0x00, 0x0b, 0x74, 0x2b, 0x4c, 0x00, 47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x01, 48 0x00, 0x1c, 0x00, 0x32, 0x55, 0x54, 0x09, 0x00, 0x03, 0xf5, 0xe5, 0x57, 49 0x5a, 0xf2, 0xe5, 0x57, 0x5a, 0x75, 0x78, 0x0b, 0x00, 0x01, 0x04, 0x8f, 50 0x66, 0x05, 0x00, 0x04, 0x88, 0x13, 0x00, 0x00, 0x33, 0x33, 0x03, 0x01, 51 0x2e, 0x00, 0xb4, 0xa0, 0xf2, 0x36, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 52 0x00, 0x00}; 53 54 // echo "0123456789" > test1.txt && echo "9876543210" > test2.txt && 55 // gzip -kf test1.txt test2.txt && cat test1.txt.gz test2.txt.gz | 56 // hexdump -v -e '12/1 "0x%02x, " "\n"' 57 const uint8_t kGzipEntryWithMultipleMembers[] = { 58 0x1f, 0x8b, 0x08, 0x08, 0x77, 0xd5, 0x84, 0x5a, 0x00, 0x03, 0x74, 0x65, 59 0x73, 0x74, 0x31, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x33, 0x30, 0x34, 0x32, 60 0x36, 0x31, 0x35, 0x33, 0xb7, 0xb0, 0xe4, 0x02, 0x00, 0xd1, 0xe5, 0x76, 61 0x40, 0x0b, 0x00, 0x00, 0x00, 0x1f, 0x8b, 0x08, 0x08, 0x77, 0xd5, 0x84, 62 0x5a, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x32, 0x2e, 0x74, 0x78, 0x74, 63 0x00, 0xb3, 0xb4, 0x30, 0x37, 0x33, 0x35, 0x31, 0x36, 0x32, 0x34, 0xe0, 64 0x02, 0x00, 0x20, 0x9c, 0x5f, 0x89, 0x0b, 0x00, 0x00, 0x00}; 65 66 // echo "0123456789" > test1.txt && gzip -kf test1.txt && cat test1.txt.gz | 67 // hexdump -v -e '12/1 "0x%02x, " "\n"' 68 // And manually insert extra field with two byte length (10) followed by: 69 // echo "extrafield" | hexdump -v -e '12/1 "0x%02x, " "\n"' 70 // Then change the forth byte of array to -x0c to enable the extra field. 71 const uint8_t kGzipEntryWithExtraField[] = { 72 0x1f, 0x8b, 0x08, 0x0c, 0xcf, 0x0e, 0x86, 0x5a, 0x00, 0x03, 73 // Extra field begin 74 0x0A, 0x00, 0x65, 0x78, 0x74, 0x72, 0x61, 0x66, 0x69, 0x65, 0x6c, 0x64, 75 // Extra field end 76 0x74, 0x65, 0x73, 0x74, 0x31, 0x2e, 0x74, 0x78, 0x74, 0x00, 0x33, 0x30, 77 0x34, 0x32, 0x36, 0x31, 0x35, 0x33, 0xb7, 0xb0, 0xe4, 0x02, 0x00, 0xd1, 78 0xe5, 0x76, 0x40, 0x0b, 0x00, 0x00, 0x00}; 79 80 // echo "0123456789" | zlib-flate -compress | 81 // hexdump -v -e '12/1 "0x%02x, " "\n"' 82 const uint8_t kZlibEntry[] = {0x78, 0x9c, 0x33, 0x30, 0x34, 0x32, 0x36, 83 0x31, 0x35, 0x33, 0xb7, 0xb0, 0xe4, 0x02, 84 0x00, 0x0d, 0x17, 0x02, 0x18}; 85 86 void FindDeflatesInZlibBlocks(const Buffer& src, 87 const vector<ByteExtent>& zlibs, 88 const vector<BitExtent>& deflates) { 89 string tmp_file; 90 ASSERT_TRUE(MakeTempFile(&tmp_file, nullptr)); 91 ScopedPathUnlinker unlinker(tmp_file); 92 auto src_stream = FileStream::Open(tmp_file, false, true); 93 ASSERT_TRUE(src_stream); 94 ASSERT_TRUE(src_stream->Write(src.data(), src.size())); 95 ASSERT_TRUE(src_stream->Close()); 96 97 vector<BitExtent> deflates_out; 98 ASSERT_TRUE(LocateDeflatesInZlibBlocks(tmp_file, zlibs, &deflates_out)); 99 ASSERT_EQ(deflates, deflates_out); 100 } 101 102 void CheckFindPuffLocation(const Buffer& compressed, 103 const vector<BitExtent>& deflates, 104 const vector<ByteExtent>& expected_puffs, 105 uint64_t expected_puff_size) { 106 auto src = MemoryStream::CreateForRead(compressed); 107 vector<ByteExtent> puffs; 108 uint64_t puff_size; 109 ASSERT_TRUE(FindPuffLocations(src, deflates, &puffs, &puff_size)); 110 EXPECT_EQ(puffs, expected_puffs); 111 EXPECT_EQ(puff_size, expected_puff_size); 112 } 113 } // namespace 114 115 // Test Simple Puffing of the source. 116 TEST(UtilsTest, FindPuffLocations1Test) { 117 CheckFindPuffLocation(kDeflatesSample1, kSubblockDeflateExtentsSample1, 118 kPuffExtentsSample1, kPuffsSample1.size()); 119 } 120 121 TEST(UtilsTest, FindPuffLocations2Test) { 122 CheckFindPuffLocation(kDeflatesSample2, kSubblockDeflateExtentsSample2, 123 kPuffExtentsSample2, kPuffsSample2.size()); 124 } 125 126 TEST(UtilsTest, LocateDeflatesInZlib) { 127 Buffer zlib_data(kZlibEntry, std::end(kZlibEntry)); 128 vector<BitExtent> deflates; 129 vector<BitExtent> expected_deflates = {{16, 98}}; 130 EXPECT_TRUE(LocateDeflatesInZlib(zlib_data, &deflates)); 131 EXPECT_EQ(deflates, expected_deflates); 132 } 133 134 TEST(UtilsTest, LocateDeflatesInEmptyZlib) { 135 Buffer empty; 136 vector<ByteExtent> empty_zlibs; 137 vector<BitExtent> empty_deflates; 138 FindDeflatesInZlibBlocks(empty, empty_zlibs, empty_deflates); 139 } 140 141 TEST(UtilsTest, LocateDeflatesInZlibWithInvalidFields) { 142 Buffer zlib_data(kZlibEntry, std::end(kZlibEntry)); 143 auto cmf = zlib_data[0]; 144 auto flag = zlib_data[1]; 145 146 vector<BitExtent> deflates; 147 zlib_data[0] = cmf & 0xF0; 148 EXPECT_FALSE(LocateDeflatesInZlib(zlib_data, &deflates)); 149 zlib_data[0] = cmf | (8 << 4); 150 EXPECT_FALSE(LocateDeflatesInZlib(zlib_data, &deflates)); 151 zlib_data[0] = cmf; // Correct it. 152 153 zlib_data[1] = flag & 0xF0; 154 EXPECT_FALSE(LocateDeflatesInZlib(zlib_data, &deflates)); 155 } 156 157 TEST(UtilsTest, LocateDeflatesInZipArchiveSmoke) { 158 Buffer zip_entries(kZipEntries, std::end(kZipEntries)); 159 vector<BitExtent> deflates; 160 vector<BitExtent> expected_deflates = {{472, 46}, {992, 46}}; 161 EXPECT_TRUE(LocateDeflatesInZipArchive(zip_entries, &deflates)); 162 EXPECT_EQ(deflates, expected_deflates); 163 } 164 165 TEST(UtilsTest, LocateDeflatesInZipArchiveWithDataDescriptor) { 166 Buffer zip_entries(kZipEntryWithDataDescriptor, 167 std::end(kZipEntryWithDataDescriptor)); 168 vector<BitExtent> deflates; 169 vector<BitExtent> expected_deflates = {{472, 46}, {1120, 46}}; 170 EXPECT_TRUE(LocateDeflatesInZipArchive(zip_entries, &deflates)); 171 EXPECT_EQ(deflates, expected_deflates); 172 } 173 174 TEST(UtilsTest, LocateDeflatesInZipArchiveErrorChecks) { 175 Buffer zip_entries(kZipEntries, std::end(kZipEntries)); 176 // Construct a invalid zip entry whose size overflows. 177 zip_entries[29] = 0xff; 178 vector<BitExtent> deflates_overflow; 179 vector<BitExtent> expected_deflates = {{992, 46}}; 180 EXPECT_TRUE(LocateDeflatesInZipArchive(zip_entries, &deflates_overflow)); 181 EXPECT_EQ(deflates_overflow, expected_deflates); 182 183 zip_entries.resize(128); 184 vector<BitExtent> deflates_incomplete; 185 EXPECT_TRUE(LocateDeflatesInZipArchive(zip_entries, &deflates_incomplete)); 186 EXPECT_TRUE(deflates_incomplete.empty()); 187 } 188 189 TEST(UtilsTest, LocateDeflatesInGzip) { 190 Buffer gzip_data(kGzipEntryWithMultipleMembers, 191 std::end(kGzipEntryWithMultipleMembers)); 192 vector<BitExtent> deflates; 193 vector<BitExtent> expected_deflates = {{160, 98}, {488, 98}}; 194 EXPECT_TRUE(LocateDeflatesInGzip(gzip_data, &deflates)); 195 EXPECT_EQ(deflates, expected_deflates); 196 } 197 198 TEST(UtilsTest, LocateDeflatesInGzipFail) { 199 Buffer gzip_data(kGzipEntryWithMultipleMembers, 200 std::end(kGzipEntryWithMultipleMembers)); 201 gzip_data[0] ^= 1; 202 vector<BitExtent> deflates; 203 EXPECT_FALSE(LocateDeflatesInGzip(gzip_data, &deflates)); 204 } 205 206 TEST(UtilsTest, LocateDeflatesInGzipWithPadding) { 207 Buffer gzip_data(kGzipEntryWithMultipleMembers, 208 std::end(kGzipEntryWithMultipleMembers)); 209 gzip_data.resize(gzip_data.size() + 100); 210 vector<BitExtent> deflates; 211 vector<BitExtent> expected_deflates = {{160, 98}, {488, 98}}; 212 EXPECT_TRUE(LocateDeflatesInGzip(gzip_data, &deflates)); 213 EXPECT_EQ(deflates, expected_deflates); 214 } 215 216 TEST(UtilsTest, LocateDeflatesInGzipWithExtraField) { 217 Buffer gzip_data(kGzipEntryWithExtraField, 218 std::end(kGzipEntryWithExtraField)); 219 vector<BitExtent> deflates; 220 vector<BitExtent> expected_deflates = {{256, 98}}; 221 EXPECT_TRUE(LocateDeflatesInGzip(gzip_data, &deflates)); 222 EXPECT_EQ(deflates, expected_deflates); 223 } 224 225 TEST(UtilsTest, RemoveEqualBitExtents) { 226 Buffer data1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; 227 Buffer data2 = {1, 2, 3, 4, 5, 5, 6, 7, 8, 9}; 228 vector<BitExtent> ext1 = {{0, 10}, {10, 14}, {25, 15}, {40, 8}, {50, 23}}; 229 vector<BitExtent> ext2 = {{0, 10}, {17, 15}, {32, 8}, {40, 8}, {50, 23}}; 230 RemoveEqualBitExtents(data1, data2, &ext1, &ext2); 231 vector<BitExtent> expected_ext1 = {{0, 10}, {10, 14}}; 232 EXPECT_EQ(expected_ext1, ext1); 233 vector<BitExtent> expected_ext2 = {{0, 10}}; 234 EXPECT_EQ(expected_ext2, ext2); 235 RemoveEqualBitExtents(data1, data2, &ext1, &ext1); 236 EXPECT_EQ(expected_ext1, ext1); 237 RemoveEqualBitExtents(data1, data1, &ext1, &ext1); 238 EXPECT_TRUE(ext1.empty()); 239 expected_ext1 = ext1 = {{0, 0}, {1, 1}, {2, 7}}; 240 RemoveEqualBitExtents(data1, data2, &ext1, &ext2); 241 EXPECT_EQ(expected_ext1, ext1); 242 EXPECT_EQ(expected_ext2, ext2); 243 } 244 245 TEST(UtilsTest, RemoveDeflatesWithBadDistanceCaches) { 246 vector<BitExtent> deflates(kProblematicCacheDeflateExtents), empty; 247 EXPECT_TRUE( 248 RemoveDeflatesWithBadDistanceCaches(kProblematicCache, &deflates)); 249 EXPECT_EQ(deflates, empty); 250 251 // Just a sanity check to make sure this function is not removing anything 252 // else. 253 deflates = kSubblockDeflateExtentsSample1; 254 EXPECT_TRUE(RemoveDeflatesWithBadDistanceCaches(kDeflatesSample1, &deflates)); 255 EXPECT_EQ(deflates, kSubblockDeflateExtentsSample1); 256 257 // Now combine three deflates and make sure it is doing the right job. 258 Buffer data; 259 data.insert(data.end(), kDeflatesSample1.begin(), kDeflatesSample1.end()); 260 data.insert(data.end(), kProblematicCache.begin(), kProblematicCache.end()); 261 data.insert(data.end(), kDeflatesSample1.begin(), kDeflatesSample1.end()); 262 263 deflates = kSubblockDeflateExtentsSample1; 264 size_t offset = kDeflatesSample1.size() * 8; 265 for (const auto& deflate : kProblematicCacheDeflateExtents) { 266 deflates.emplace_back(deflate.offset + offset, deflate.length); 267 } 268 offset += kProblematicCache.size() * 8; 269 for (const auto& deflate : kSubblockDeflateExtentsSample1) { 270 deflates.emplace_back(deflate.offset + offset, deflate.length); 271 } 272 273 auto expected_deflates(deflates); 274 expected_deflates.erase(expected_deflates.begin() + 275 kSubblockDeflateExtentsSample1.size()); 276 277 EXPECT_TRUE(RemoveDeflatesWithBadDistanceCaches(data, &deflates)); 278 EXPECT_EQ(deflates, expected_deflates); 279 } 280 281 } // namespace puffin 282
