1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Applying a Chrome OS update payload. 6 7 This module is used internally by the main Payload class for applying an update 8 payload. The interface for invoking the applier is as follows: 9 10 applier = PayloadApplier(payload) 11 applier.Run(...) 12 13 """ 14 15 from __future__ import print_function 16 17 import array 18 import bz2 19 import hashlib 20 import itertools 21 import os 22 import shutil 23 import subprocess 24 import sys 25 import tempfile 26 27 from update_payload import common 28 from update_payload.error import PayloadError 29 30 31 # 32 # Helper functions. 33 # 34 def _VerifySha256(file_obj, expected_hash, name, length=-1): 35 """Verifies the SHA256 hash of a file. 36 37 Args: 38 file_obj: file object to read 39 expected_hash: the hash digest we expect to be getting 40 name: name string of this hash, for error reporting 41 length: precise length of data to verify (optional) 42 43 Raises: 44 PayloadError if computed hash doesn't match expected one, or if fails to 45 read the specified length of data. 46 """ 47 hasher = hashlib.sha256() 48 block_length = 1024 * 1024 49 max_length = length if length >= 0 else sys.maxint 50 51 while max_length > 0: 52 read_length = min(max_length, block_length) 53 data = file_obj.read(read_length) 54 if not data: 55 break 56 max_length -= len(data) 57 hasher.update(data) 58 59 if length >= 0 and max_length > 0: 60 raise PayloadError( 61 'insufficient data (%d instead of %d) when verifying %s' % 62 (length - max_length, length, name)) 63 64 actual_hash = hasher.digest() 65 if actual_hash != expected_hash: 66 raise PayloadError('%s hash (%s) not as expected (%s)' % 67 (name, common.FormatSha256(actual_hash), 68 common.FormatSha256(expected_hash))) 69 70 71 def _ReadExtents(file_obj, extents, block_size, max_length=-1): 72 """Reads data from file as defined by extent sequence. 73 74 This tries to be efficient by not copying data as it is read in chunks. 75 76 Args: 77 file_obj: file object 78 extents: sequence of block extents (offset and length) 79 block_size: size of each block 80 max_length: maximum length to read (optional) 81 82 Returns: 83 A character array containing the concatenated read data. 84 """ 85 data = array.array('c') 86 if max_length < 0: 87 max_length = sys.maxint 88 for ex in extents: 89 if max_length == 0: 90 break 91 read_length = min(max_length, ex.num_blocks * block_size) 92 93 # Fill with zeros or read from file, depending on the type of extent. 94 if ex.start_block == common.PSEUDO_EXTENT_MARKER: 95 data.extend(itertools.repeat('\0', read_length)) 96 else: 97 file_obj.seek(ex.start_block * block_size) 98 data.fromfile(file_obj, read_length) 99 100 max_length -= read_length 101 102 return data 103 104 105 def _WriteExtents(file_obj, data, extents, block_size, base_name): 106 """Writes data to file as defined by extent sequence. 107 108 This tries to be efficient by not copy data as it is written in chunks. 109 110 Args: 111 file_obj: file object 112 data: data to write 113 extents: sequence of block extents (offset and length) 114 block_size: size of each block 115 base_name: name string of extent sequence for error reporting 116 117 Raises: 118 PayloadError when things don't add up. 119 """ 120 data_offset = 0 121 data_length = len(data) 122 for ex, ex_name in common.ExtentIter(extents, base_name): 123 if not data_length: 124 raise PayloadError('%s: more write extents than data' % ex_name) 125 write_length = min(data_length, ex.num_blocks * block_size) 126 127 # Only do actual writing if this is not a pseudo-extent. 128 if ex.start_block != common.PSEUDO_EXTENT_MARKER: 129 file_obj.seek(ex.start_block * block_size) 130 data_view = buffer(data, data_offset, write_length) 131 file_obj.write(data_view) 132 133 data_offset += write_length 134 data_length -= write_length 135 136 if data_length: 137 raise PayloadError('%s: more data than write extents' % base_name) 138 139 140 def _ExtentsToBspatchArg(extents, block_size, base_name, data_length=-1): 141 """Translates an extent sequence into a bspatch-compatible string argument. 142 143 Args: 144 extents: sequence of block extents (offset and length) 145 block_size: size of each block 146 base_name: name string of extent sequence for error reporting 147 data_length: the actual total length of the data in bytes (optional) 148 149 Returns: 150 A tuple consisting of (i) a string of the form 151 "off_1:len_1,...,off_n:len_n", (ii) an offset where zero padding is needed 152 for filling the last extent, (iii) the length of the padding (zero means no 153 padding is needed and the extents cover the full length of data). 154 155 Raises: 156 PayloadError if data_length is too short or too long. 157 """ 158 arg = '' 159 pad_off = pad_len = 0 160 if data_length < 0: 161 data_length = sys.maxint 162 for ex, ex_name in common.ExtentIter(extents, base_name): 163 if not data_length: 164 raise PayloadError('%s: more extents than total data length' % ex_name) 165 166 is_pseudo = ex.start_block == common.PSEUDO_EXTENT_MARKER 167 start_byte = -1 if is_pseudo else ex.start_block * block_size 168 num_bytes = ex.num_blocks * block_size 169 if data_length < num_bytes: 170 # We're only padding a real extent. 171 if not is_pseudo: 172 pad_off = start_byte + data_length 173 pad_len = num_bytes - data_length 174 175 num_bytes = data_length 176 177 arg += '%s%d:%d' % (arg and ',', start_byte, num_bytes) 178 data_length -= num_bytes 179 180 if data_length: 181 raise PayloadError('%s: extents not covering full data length' % base_name) 182 183 return arg, pad_off, pad_len 184 185 186 # 187 # Payload application. 188 # 189 class PayloadApplier(object): 190 """Applying an update payload. 191 192 This is a short-lived object whose purpose is to isolate the logic used for 193 applying an update payload. 194 """ 195 196 def __init__(self, payload, bsdiff_in_place=True, bspatch_path=None, 197 puffpatch_path=None, truncate_to_expected_size=True): 198 """Initialize the applier. 199 200 Args: 201 payload: the payload object to check 202 bsdiff_in_place: whether to perform BSDIFF operation in-place (optional) 203 bspatch_path: path to the bspatch binary (optional) 204 puffpatch_path: path to the puffpatch binary (optional) 205 truncate_to_expected_size: whether to truncate the resulting partitions 206 to their expected sizes, as specified in the 207 payload (optional) 208 """ 209 assert payload.is_init, 'uninitialized update payload' 210 self.payload = payload 211 self.block_size = payload.manifest.block_size 212 self.minor_version = payload.manifest.minor_version 213 self.bsdiff_in_place = bsdiff_in_place 214 self.bspatch_path = bspatch_path or 'bspatch' 215 self.puffpatch_path = puffpatch_path or 'puffin' 216 self.truncate_to_expected_size = truncate_to_expected_size 217 218 def _ApplyReplaceOperation(self, op, op_name, out_data, part_file, part_size): 219 """Applies a REPLACE{,_BZ} operation. 220 221 Args: 222 op: the operation object 223 op_name: name string for error reporting 224 out_data: the data to be written 225 part_file: the partition file object 226 part_size: the size of the partition 227 228 Raises: 229 PayloadError if something goes wrong. 230 """ 231 block_size = self.block_size 232 data_length = len(out_data) 233 234 # Decompress data if needed. 235 if op.type == common.OpType.REPLACE_BZ: 236 out_data = bz2.decompress(out_data) 237 data_length = len(out_data) 238 239 # Write data to blocks specified in dst extents. 240 data_start = 0 241 for ex, ex_name in common.ExtentIter(op.dst_extents, 242 '%s.dst_extents' % op_name): 243 start_block = ex.start_block 244 num_blocks = ex.num_blocks 245 count = num_blocks * block_size 246 247 # Make sure it's not a fake (signature) operation. 248 if start_block != common.PSEUDO_EXTENT_MARKER: 249 data_end = data_start + count 250 251 # Make sure we're not running past partition boundary. 252 if (start_block + num_blocks) * block_size > part_size: 253 raise PayloadError( 254 '%s: extent (%s) exceeds partition size (%d)' % 255 (ex_name, common.FormatExtent(ex, block_size), 256 part_size)) 257 258 # Make sure that we have enough data to write. 259 if data_end >= data_length + block_size: 260 raise PayloadError( 261 '%s: more dst blocks than data (even with padding)') 262 263 # Pad with zeros if necessary. 264 if data_end > data_length: 265 padding = data_end - data_length 266 out_data += '\0' * padding 267 268 self.payload.payload_file.seek(start_block * block_size) 269 part_file.seek(start_block * block_size) 270 part_file.write(out_data[data_start:data_end]) 271 272 data_start += count 273 274 # Make sure we wrote all data. 275 if data_start < data_length: 276 raise PayloadError('%s: wrote fewer bytes (%d) than expected (%d)' % 277 (op_name, data_start, data_length)) 278 279 def _ApplyMoveOperation(self, op, op_name, part_file): 280 """Applies a MOVE operation. 281 282 Note that this operation must read the whole block data from the input and 283 only then dump it, due to our in-place update semantics; otherwise, it 284 might clobber data midway through. 285 286 Args: 287 op: the operation object 288 op_name: name string for error reporting 289 part_file: the partition file object 290 291 Raises: 292 PayloadError if something goes wrong. 293 """ 294 block_size = self.block_size 295 296 # Gather input raw data from src extents. 297 in_data = _ReadExtents(part_file, op.src_extents, block_size) 298 299 # Dump extracted data to dst extents. 300 _WriteExtents(part_file, in_data, op.dst_extents, block_size, 301 '%s.dst_extents' % op_name) 302 303 def _ApplyZeroOperation(self, op, op_name, part_file): 304 """Applies a ZERO operation. 305 306 Args: 307 op: the operation object 308 op_name: name string for error reporting 309 part_file: the partition file object 310 311 Raises: 312 PayloadError if something goes wrong. 313 """ 314 block_size = self.block_size 315 base_name = '%s.dst_extents' % op_name 316 317 # Iterate over the extents and write zero. 318 # pylint: disable=unused-variable 319 for ex, ex_name in common.ExtentIter(op.dst_extents, base_name): 320 # Only do actual writing if this is not a pseudo-extent. 321 if ex.start_block != common.PSEUDO_EXTENT_MARKER: 322 part_file.seek(ex.start_block * block_size) 323 part_file.write('\0' * (ex.num_blocks * block_size)) 324 325 def _ApplySourceCopyOperation(self, op, op_name, old_part_file, 326 new_part_file): 327 """Applies a SOURCE_COPY operation. 328 329 Args: 330 op: the operation object 331 op_name: name string for error reporting 332 old_part_file: the old partition file object 333 new_part_file: the new partition file object 334 335 Raises: 336 PayloadError if something goes wrong. 337 """ 338 if not old_part_file: 339 raise PayloadError( 340 '%s: no source partition file provided for operation type (%d)' % 341 (op_name, op.type)) 342 343 block_size = self.block_size 344 345 # Gather input raw data from src extents. 346 in_data = _ReadExtents(old_part_file, op.src_extents, block_size) 347 348 # Dump extracted data to dst extents. 349 _WriteExtents(new_part_file, in_data, op.dst_extents, block_size, 350 '%s.dst_extents' % op_name) 351 352 def _BytesInExtents(self, extents, base_name): 353 """Counts the length of extents in bytes. 354 355 Args: 356 extents: The list of Extents. 357 base_name: For error reporting. 358 359 Returns: 360 The number of bytes in extents. 361 """ 362 363 length = 0 364 # pylint: disable=unused-variable 365 for ex, ex_name in common.ExtentIter(extents, base_name): 366 length += ex.num_blocks * self.block_size 367 return length 368 369 def _ApplyDiffOperation(self, op, op_name, patch_data, old_part_file, 370 new_part_file): 371 """Applies a SOURCE_BSDIFF, BROTLI_BSDIFF or PUFFDIFF operation. 372 373 Args: 374 op: the operation object 375 op_name: name string for error reporting 376 patch_data: the binary patch content 377 old_part_file: the source partition file object 378 new_part_file: the target partition file object 379 380 Raises: 381 PayloadError if something goes wrong. 382 """ 383 if not old_part_file: 384 raise PayloadError( 385 '%s: no source partition file provided for operation type (%d)' % 386 (op_name, op.type)) 387 388 block_size = self.block_size 389 390 # Dump patch data to file. 391 with tempfile.NamedTemporaryFile(delete=False) as patch_file: 392 patch_file_name = patch_file.name 393 patch_file.write(patch_data) 394 395 if (hasattr(new_part_file, 'fileno') and 396 ((not old_part_file) or hasattr(old_part_file, 'fileno'))): 397 # Construct input and output extents argument for bspatch. 398 399 in_extents_arg, _, _ = _ExtentsToBspatchArg( 400 op.src_extents, block_size, '%s.src_extents' % op_name, 401 data_length=op.src_length if op.src_length else 402 self._BytesInExtents(op.src_extents, "%s.src_extents")) 403 out_extents_arg, pad_off, pad_len = _ExtentsToBspatchArg( 404 op.dst_extents, block_size, '%s.dst_extents' % op_name, 405 data_length=op.dst_length if op.dst_length else 406 self._BytesInExtents(op.dst_extents, "%s.dst_extents")) 407 408 new_file_name = '/dev/fd/%d' % new_part_file.fileno() 409 # Diff from source partition. 410 old_file_name = '/dev/fd/%d' % old_part_file.fileno() 411 412 if op.type in (common.OpType.BSDIFF, common.OpType.SOURCE_BSDIFF, 413 common.OpType.BROTLI_BSDIFF): 414 # Invoke bspatch on partition file with extents args. 415 bspatch_cmd = [self.bspatch_path, old_file_name, new_file_name, 416 patch_file_name, in_extents_arg, out_extents_arg] 417 subprocess.check_call(bspatch_cmd) 418 elif op.type == common.OpType.PUFFDIFF: 419 # Invoke puffpatch on partition file with extents args. 420 puffpatch_cmd = [self.puffpatch_path, 421 "--operation=puffpatch", 422 "--src_file=%s" % old_file_name, 423 "--dst_file=%s" % new_file_name, 424 "--patch_file=%s" % patch_file_name, 425 "--src_extents=%s" % in_extents_arg, 426 "--dst_extents=%s" % out_extents_arg] 427 subprocess.check_call(puffpatch_cmd) 428 else: 429 raise PayloadError("Unknown operation %s", op.type) 430 431 # Pad with zeros past the total output length. 432 if pad_len: 433 new_part_file.seek(pad_off) 434 new_part_file.write('\0' * pad_len) 435 else: 436 # Gather input raw data and write to a temp file. 437 input_part_file = old_part_file if old_part_file else new_part_file 438 in_data = _ReadExtents(input_part_file, op.src_extents, block_size, 439 max_length=op.src_length if op.src_length else 440 self._BytesInExtents(op.src_extents, 441 "%s.src_extents")) 442 with tempfile.NamedTemporaryFile(delete=False) as in_file: 443 in_file_name = in_file.name 444 in_file.write(in_data) 445 446 # Allocate temporary output file. 447 with tempfile.NamedTemporaryFile(delete=False) as out_file: 448 out_file_name = out_file.name 449 450 if op.type in (common.OpType.BSDIFF, common.OpType.SOURCE_BSDIFF, 451 common.OpType.BROTLI_BSDIFF): 452 # Invoke bspatch. 453 bspatch_cmd = [self.bspatch_path, in_file_name, out_file_name, 454 patch_file_name] 455 subprocess.check_call(bspatch_cmd) 456 elif op.type == common.OpType.PUFFDIFF: 457 # Invoke puffpatch. 458 puffpatch_cmd = [self.puffpatch_path, 459 "--operation=puffpatch", 460 "--src_file=%s" % in_file_name, 461 "--dst_file=%s" % out_file_name, 462 "--patch_file=%s" % patch_file_name] 463 subprocess.check_call(puffpatch_cmd) 464 else: 465 raise PayloadError("Unknown operation %s", op.type) 466 467 # Read output. 468 with open(out_file_name, 'rb') as out_file: 469 out_data = out_file.read() 470 if len(out_data) != op.dst_length: 471 raise PayloadError( 472 '%s: actual patched data length (%d) not as expected (%d)' % 473 (op_name, len(out_data), op.dst_length)) 474 475 # Write output back to partition, with padding. 476 unaligned_out_len = len(out_data) % block_size 477 if unaligned_out_len: 478 out_data += '\0' * (block_size - unaligned_out_len) 479 _WriteExtents(new_part_file, out_data, op.dst_extents, block_size, 480 '%s.dst_extents' % op_name) 481 482 # Delete input/output files. 483 os.remove(in_file_name) 484 os.remove(out_file_name) 485 486 # Delete patch file. 487 os.remove(patch_file_name) 488 489 def _ApplyOperations(self, operations, base_name, old_part_file, 490 new_part_file, part_size): 491 """Applies a sequence of update operations to a partition. 492 493 This assumes an in-place update semantics for MOVE and BSDIFF, namely all 494 reads are performed first, then the data is processed and written back to 495 the same file. 496 497 Args: 498 operations: the sequence of operations 499 base_name: the name of the operation sequence 500 old_part_file: the old partition file object, open for reading/writing 501 new_part_file: the new partition file object, open for reading/writing 502 part_size: the partition size 503 504 Raises: 505 PayloadError if anything goes wrong while processing the payload. 506 """ 507 for op, op_name in common.OperationIter(operations, base_name): 508 # Read data blob. 509 data = self.payload.ReadDataBlob(op.data_offset, op.data_length) 510 511 if op.type in (common.OpType.REPLACE, common.OpType.REPLACE_BZ): 512 self._ApplyReplaceOperation(op, op_name, data, new_part_file, part_size) 513 elif op.type == common.OpType.MOVE: 514 self._ApplyMoveOperation(op, op_name, new_part_file) 515 elif op.type == common.OpType.ZERO: 516 self._ApplyZeroOperation(op, op_name, new_part_file) 517 elif op.type == common.OpType.BSDIFF: 518 self._ApplyDiffOperation(op, op_name, data, new_part_file, 519 new_part_file) 520 elif op.type == common.OpType.SOURCE_COPY: 521 self._ApplySourceCopyOperation(op, op_name, old_part_file, 522 new_part_file) 523 elif op.type in (common.OpType.SOURCE_BSDIFF, common.OpType.PUFFDIFF, 524 common.OpType.BROTLI_BSDIFF): 525 self._ApplyDiffOperation(op, op_name, data, old_part_file, 526 new_part_file) 527 else: 528 raise PayloadError('%s: unknown operation type (%d)' % 529 (op_name, op.type)) 530 531 def _ApplyToPartition(self, operations, part_name, base_name, 532 new_part_file_name, new_part_info, 533 old_part_file_name=None, old_part_info=None): 534 """Applies an update to a partition. 535 536 Args: 537 operations: the sequence of update operations to apply 538 part_name: the name of the partition, for error reporting 539 base_name: the name of the operation sequence 540 new_part_file_name: file name to write partition data to 541 new_part_info: size and expected hash of dest partition 542 old_part_file_name: file name of source partition (optional) 543 old_part_info: size and expected hash of source partition (optional) 544 545 Raises: 546 PayloadError if anything goes wrong with the update. 547 """ 548 # Do we have a source partition? 549 if old_part_file_name: 550 # Verify the source partition. 551 with open(old_part_file_name, 'rb') as old_part_file: 552 _VerifySha256(old_part_file, old_part_info.hash, 553 'old ' + part_name, length=old_part_info.size) 554 new_part_file_mode = 'r+b' 555 if self.minor_version == common.INPLACE_MINOR_PAYLOAD_VERSION: 556 # Copy the src partition to the dst one; make sure we don't truncate it. 557 shutil.copyfile(old_part_file_name, new_part_file_name) 558 elif (self.minor_version == common.SOURCE_MINOR_PAYLOAD_VERSION or 559 self.minor_version == common.OPSRCHASH_MINOR_PAYLOAD_VERSION or 560 self.minor_version == common.BROTLI_BSDIFF_MINOR_PAYLOAD_VERSION or 561 self.minor_version == common.PUFFDIFF_MINOR_PAYLOAD_VERSION): 562 # In minor version >= 2, we don't want to copy the partitions, so 563 # instead just make the new partition file. 564 open(new_part_file_name, 'w').close() 565 else: 566 raise PayloadError("Unknown minor version: %d" % self.minor_version) 567 else: 568 # We need to create/truncate the dst partition file. 569 new_part_file_mode = 'w+b' 570 571 # Apply operations. 572 with open(new_part_file_name, new_part_file_mode) as new_part_file: 573 old_part_file = (open(old_part_file_name, 'r+b') 574 if old_part_file_name else None) 575 try: 576 self._ApplyOperations(operations, base_name, old_part_file, 577 new_part_file, new_part_info.size) 578 finally: 579 if old_part_file: 580 old_part_file.close() 581 582 # Truncate the result, if so instructed. 583 if self.truncate_to_expected_size: 584 new_part_file.seek(0, 2) 585 if new_part_file.tell() > new_part_info.size: 586 new_part_file.seek(new_part_info.size) 587 new_part_file.truncate() 588 589 # Verify the resulting partition. 590 with open(new_part_file_name, 'rb') as new_part_file: 591 _VerifySha256(new_part_file, new_part_info.hash, 592 'new ' + part_name, length=new_part_info.size) 593 594 def Run(self, new_kernel_part, new_rootfs_part, old_kernel_part=None, 595 old_rootfs_part=None): 596 """Applier entry point, invoking all update operations. 597 598 Args: 599 new_kernel_part: name of dest kernel partition file 600 new_rootfs_part: name of dest rootfs partition file 601 old_kernel_part: name of source kernel partition file (optional) 602 old_rootfs_part: name of source rootfs partition file (optional) 603 604 Raises: 605 PayloadError if payload application failed. 606 """ 607 self.payload.ResetFile() 608 609 # Make sure the arguments are sane and match the payload. 610 if not (new_kernel_part and new_rootfs_part): 611 raise PayloadError('missing dst {kernel,rootfs} partitions') 612 613 if not (old_kernel_part or old_rootfs_part): 614 if not self.payload.IsFull(): 615 raise PayloadError('trying to apply a non-full update without src ' 616 '{kernel,rootfs} partitions') 617 elif old_kernel_part and old_rootfs_part: 618 if not self.payload.IsDelta(): 619 raise PayloadError('trying to apply a non-delta update onto src ' 620 '{kernel,rootfs} partitions') 621 else: 622 raise PayloadError('not all src partitions provided') 623 624 # Apply update to rootfs. 625 self._ApplyToPartition( 626 self.payload.manifest.install_operations, 'rootfs', 627 'install_operations', new_rootfs_part, 628 self.payload.manifest.new_rootfs_info, old_rootfs_part, 629 self.payload.manifest.old_rootfs_info) 630 631 # Apply update to kernel update. 632 self._ApplyToPartition( 633 self.payload.manifest.kernel_install_operations, 'kernel', 634 'kernel_install_operations', new_kernel_part, 635 self.payload.manifest.new_kernel_info, old_kernel_part, 636 self.payload.manifest.old_kernel_info) 637