1 // 2 // Copyright (C) 2017 The Android Open Source Project 3 // 4 // Licensed under the Apache License, Version 2.0 (the "License"); 5 // you may not use this file except in compliance with the License. 6 // You may obtain a copy of the License at 7 // 8 // http://www.apache.org/licenses/LICENSE-2.0 9 // 10 // Unless required by applicable law or agreed to in writing, software 11 // distributed under the License is distributed on an "AS IS" BASIS, 12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 // See the License for the specific language governing permissions and 14 // limitations under the License. 15 // 16 17 #include "update_engine/payload_generator/squashfs_filesystem.h" 18 19 #include <fcntl.h> 20 21 #include <algorithm> 22 #include <string> 23 #include <utility> 24 25 #include <base/files/file_util.h> 26 #include <base/logging.h> 27 #include <base/strings/string_number_conversions.h> 28 #include <base/strings/string_split.h> 29 #include <brillo/streams/file_stream.h> 30 31 #include "update_engine/common/subprocess.h" 32 #include "update_engine/common/utils.h" 33 #include "update_engine/payload_generator/deflate_utils.h" 34 #include "update_engine/payload_generator/delta_diff_generator.h" 35 #include "update_engine/payload_generator/extent_ranges.h" 36 #include "update_engine/payload_generator/extent_utils.h" 37 #include "update_engine/update_metadata.pb.h" 38 39 using std::string; 40 using std::unique_ptr; 41 using std::vector; 42 43 namespace chromeos_update_engine { 44 45 namespace { 46 47 Extent ExtentForBytes(uint64_t block_size, 48 uint64_t start_bytes, 49 uint64_t size_bytes) { 50 uint64_t start_block = start_bytes / block_size; 51 uint64_t end_block = (start_bytes + size_bytes + block_size - 1) / block_size; 52 return ExtentForRange(start_block, end_block - start_block); 53 } 54 55 // The size of the squashfs super block. 56 constexpr size_t kSquashfsSuperBlockSize = 96; 57 constexpr uint64_t kSquashfsCompressedBit = 1 << 24; 58 constexpr uint32_t kSquashfsZlibCompression = 1; 59 60 bool ReadSquashfsHeader(const brillo::Blob blob, 61 SquashfsFilesystem::SquashfsHeader* header) { 62 if (blob.size() < kSquashfsSuperBlockSize) { 63 return false; 64 } 65 66 memcpy(&header->magic, blob.data(), 4); 67 memcpy(&header->block_size, blob.data() + 12, 4); 68 memcpy(&header->compression_type, blob.data() + 20, 2); 69 memcpy(&header->major_version, blob.data() + 28, 2); 70 return true; 71 } 72 73 bool CheckHeader(const SquashfsFilesystem::SquashfsHeader& header) { 74 return header.magic == 0x73717368 && header.major_version == 4; 75 } 76 77 bool GetFileMapContent(const string& sqfs_path, string* map) { 78 // Create a tmp file 79 string map_file; 80 TEST_AND_RETURN_FALSE( 81 utils::MakeTempFile("squashfs_file_map.XXXXXX", &map_file, nullptr)); 82 ScopedPathUnlinker map_unlinker(map_file); 83 84 // Run unsquashfs to get the system file map. 85 // unsquashfs -m <map-file> <squashfs-file> 86 vector<string> cmd = {"unsquashfs", "-m", map_file, sqfs_path}; 87 string stdout; 88 int exit_code; 89 if (!Subprocess::SynchronousExec(cmd, &exit_code, &stdout) || 90 exit_code != 0) { 91 LOG(ERROR) << "Failed to run unsquashfs -m. The stdout content was: " 92 << stdout; 93 return false; 94 } 95 TEST_AND_RETURN_FALSE(utils::ReadFile(map_file, map)); 96 return true; 97 } 98 99 } // namespace 100 101 bool SquashfsFilesystem::Init(const string& map, 102 const string& sqfs_path, 103 size_t size, 104 const SquashfsHeader& header, 105 bool extract_deflates) { 106 size_ = size; 107 108 bool is_zlib = header.compression_type == kSquashfsZlibCompression; 109 if (!is_zlib) { 110 LOG(WARNING) << "Filesystem is not Gzipped. Not filling deflates!"; 111 } 112 vector<puffin::ByteExtent> zlib_blks; 113 114 // Reading files map. For the format of the file map look at the comments for 115 // |CreateFromFileMap()|. 116 auto lines = base::SplitStringPiece(map, 117 "\n", 118 base::WhitespaceHandling::KEEP_WHITESPACE, 119 base::SplitResult::SPLIT_WANT_NONEMPTY); 120 for (const auto& line : lines) { 121 auto splits = 122 base::SplitStringPiece(line, 123 " \t", 124 base::WhitespaceHandling::TRIM_WHITESPACE, 125 base::SplitResult::SPLIT_WANT_NONEMPTY); 126 // Only filename is invalid. 127 TEST_AND_RETURN_FALSE(splits.size() > 1); 128 uint64_t start; 129 TEST_AND_RETURN_FALSE(base::StringToUint64(splits[1], &start)); 130 uint64_t cur_offset = start; 131 for (size_t i = 2; i < splits.size(); ++i) { 132 uint64_t blk_size; 133 TEST_AND_RETURN_FALSE(base::StringToUint64(splits[i], &blk_size)); 134 // TODO(ahassani): For puffin push it into a proper list if uncompressed. 135 auto new_blk_size = blk_size & ~kSquashfsCompressedBit; 136 TEST_AND_RETURN_FALSE(new_blk_size <= header.block_size); 137 if (new_blk_size > 0 && !(blk_size & kSquashfsCompressedBit)) { 138 // Compressed block 139 if (is_zlib && extract_deflates) { 140 zlib_blks.emplace_back(cur_offset, new_blk_size); 141 } 142 } 143 cur_offset += new_blk_size; 144 } 145 146 // If size is zero do not add the file. 147 if (cur_offset - start > 0) { 148 File file; 149 file.name = splits[0].as_string(); 150 file.extents = {ExtentForBytes(kBlockSize, start, cur_offset - start)}; 151 files_.emplace_back(file); 152 } 153 } 154 155 // Sort all files by their offset in the squashfs. 156 std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) { 157 return a.extents[0].start_block() < b.extents[0].start_block(); 158 }); 159 // If there is any overlap between two consecutive extents, remove them. Here 160 // we are assuming all files have exactly one extent. If this assumption 161 // changes then this implementation needs to change too. 162 for (auto first = files_.begin(), second = first + 1; 163 first != files_.end() && second != files_.end(); 164 second = first + 1) { 165 auto first_begin = first->extents[0].start_block(); 166 auto first_end = first_begin + first->extents[0].num_blocks(); 167 auto second_begin = second->extents[0].start_block(); 168 auto second_end = second_begin + second->extents[0].num_blocks(); 169 // Remove the first file if the size is zero. 170 if (first_end == first_begin) { 171 first = files_.erase(first); 172 } else if (first_end > second_begin) { // We found a collision. 173 if (second_end <= first_end) { 174 // Second file is inside the first file, remove the second file. 175 second = files_.erase(second); 176 } else if (first_begin == second_begin) { 177 // First file is inside the second file, remove the first file. 178 first = files_.erase(first); 179 } else { 180 // Remove overlapping extents from the first file. 181 first->extents[0].set_num_blocks(second_begin - first_begin); 182 ++first; 183 } 184 } else { 185 ++first; 186 } 187 } 188 189 // Find all the metadata including superblock and add them to the list of 190 // files. 191 ExtentRanges file_extents; 192 for (const auto& file : files_) { 193 file_extents.AddExtents(file.extents); 194 } 195 vector<Extent> full = { 196 ExtentForRange(0, (size_ + kBlockSize - 1) / kBlockSize)}; 197 auto metadata_extents = FilterExtentRanges(full, file_extents); 198 // For now there should be at most two extents. One for superblock and one for 199 // metadata at the end. Just create appropriate files with <metadata-i> name. 200 // We can add all these extents as one metadata too, but that violates the 201 // contiguous write optimization. 202 for (size_t i = 0; i < metadata_extents.size(); i++) { 203 File file; 204 file.name = "<metadata-" + std::to_string(i) + ">"; 205 file.extents = {metadata_extents[i]}; 206 files_.emplace_back(file); 207 } 208 209 // Do one last sort before returning. 210 std::sort(files_.begin(), files_.end(), [](const File& a, const File& b) { 211 return a.extents[0].start_block() < b.extents[0].start_block(); 212 }); 213 214 if (is_zlib && extract_deflates) { 215 // If it is infact gzipped, then the sqfs_path should be valid to read its 216 // content. 217 TEST_AND_RETURN_FALSE(!sqfs_path.empty()); 218 if (zlib_blks.empty()) { 219 return true; 220 } 221 222 // Sort zlib blocks. 223 std::sort(zlib_blks.begin(), 224 zlib_blks.end(), 225 [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) { 226 return a.offset < b.offset; 227 }); 228 229 // Sanity check. Make sure zlib blocks are not overlapping. 230 auto result = std::adjacent_find( 231 zlib_blks.begin(), 232 zlib_blks.end(), 233 [](const puffin::ByteExtent& a, const puffin::ByteExtent& b) { 234 return (a.offset + a.length) > b.offset; 235 }); 236 TEST_AND_RETURN_FALSE(result == zlib_blks.end()); 237 238 vector<puffin::BitExtent> deflates; 239 TEST_AND_RETURN_FALSE( 240 puffin::LocateDeflatesInZlibBlocks(sqfs_path, zlib_blks, &deflates)); 241 242 // Add deflates for each file. 243 for (auto& file : files_) { 244 file.deflates = deflate_utils::FindDeflates(file.extents, deflates); 245 } 246 } 247 return true; 248 } 249 250 unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFile( 251 const string& sqfs_path, bool extract_deflates) { 252 if (sqfs_path.empty()) 253 return nullptr; 254 255 brillo::StreamPtr sqfs_file = 256 brillo::FileStream::Open(base::FilePath(sqfs_path), 257 brillo::Stream::AccessMode::READ, 258 brillo::FileStream::Disposition::OPEN_EXISTING, 259 nullptr); 260 if (!sqfs_file) { 261 LOG(ERROR) << "Unable to open " << sqfs_path << " for reading."; 262 return nullptr; 263 } 264 265 SquashfsHeader header; 266 brillo::Blob blob(kSquashfsSuperBlockSize); 267 if (!sqfs_file->ReadAllBlocking(blob.data(), blob.size(), nullptr)) { 268 LOG(ERROR) << "Unable to read from file: " << sqfs_path; 269 return nullptr; 270 } 271 if (!ReadSquashfsHeader(blob, &header) || !CheckHeader(header)) { 272 // This is not necessary an error. 273 return nullptr; 274 } 275 276 // Read the map file. 277 string filemap; 278 if (!GetFileMapContent(sqfs_path, &filemap)) { 279 LOG(ERROR) << "Failed to produce squashfs map file: " << sqfs_path; 280 return nullptr; 281 } 282 283 unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem()); 284 if (!sqfs->Init( 285 filemap, sqfs_path, sqfs_file->GetSize(), header, extract_deflates)) { 286 LOG(ERROR) << "Failed to initialized the Squashfs file system"; 287 return nullptr; 288 } 289 290 return sqfs; 291 } 292 293 unique_ptr<SquashfsFilesystem> SquashfsFilesystem::CreateFromFileMap( 294 const string& filemap, size_t size, const SquashfsHeader& header) { 295 if (!CheckHeader(header)) { 296 LOG(ERROR) << "Invalid Squashfs super block!"; 297 return nullptr; 298 } 299 300 unique_ptr<SquashfsFilesystem> sqfs(new SquashfsFilesystem()); 301 if (!sqfs->Init(filemap, "", size, header, false)) { 302 LOG(ERROR) << "Failed to initialize the Squashfs file system using filemap"; 303 return nullptr; 304 } 305 // TODO(ahassani): Add a function that initializes the puffin related extents. 306 return sqfs; 307 } 308 309 size_t SquashfsFilesystem::GetBlockSize() const { 310 return kBlockSize; 311 } 312 313 size_t SquashfsFilesystem::GetBlockCount() const { 314 return size_ / kBlockSize; 315 } 316 317 bool SquashfsFilesystem::GetFiles(vector<File>* files) const { 318 files->insert(files->end(), files_.begin(), files_.end()); 319 return true; 320 } 321 322 bool SquashfsFilesystem::LoadSettings(brillo::KeyValueStore* store) const { 323 // Settings not supported in squashfs. 324 LOG(ERROR) << "squashfs doesn't support LoadSettings()."; 325 return false; 326 } 327 328 bool SquashfsFilesystem::IsSquashfsImage(const brillo::Blob& blob) { 329 SquashfsHeader header; 330 return ReadSquashfsHeader(blob, &header) && CheckHeader(header); 331 } 332 } // namespace chromeos_update_engine 333