1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2011 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 Build image output_image_file from input_directory and properties_file. 19 20 Usage: build_image input_directory properties_file output_image_file 21 22 """ 23 import os 24 import os.path 25 import re 26 import subprocess 27 import sys 28 import commands 29 import common 30 import shutil 31 import sparse_img 32 import tempfile 33 34 OPTIONS = common.OPTIONS 35 36 FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7" 37 BLOCK_SIZE = 4096 38 39 def RunCommand(cmd): 40 """Echo and run the given command. 41 42 Args: 43 cmd: the command represented as a list of strings. 44 Returns: 45 A tuple of the output and the exit code. 46 """ 47 print "Running: ", " ".join(cmd) 48 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 49 output, _ = p.communicate() 50 print "%s" % (output.rstrip(),) 51 return (output, p.returncode) 52 53 def GetVerityFECSize(partition_size): 54 cmd = "fec -s %d" % partition_size 55 status, output = commands.getstatusoutput(cmd) 56 if status: 57 print output 58 return False, 0 59 return True, int(output) 60 61 def GetVerityTreeSize(partition_size): 62 cmd = "build_verity_tree -s %d" 63 cmd %= partition_size 64 status, output = commands.getstatusoutput(cmd) 65 if status: 66 print output 67 return False, 0 68 return True, int(output) 69 70 def GetVerityMetadataSize(partition_size): 71 cmd = "system/extras/verity/build_verity_metadata.py -s %d" 72 cmd %= partition_size 73 74 status, output = commands.getstatusoutput(cmd) 75 if status: 76 print output 77 return False, 0 78 return True, int(output) 79 80 def GetVeritySize(partition_size, fec_supported): 81 success, verity_tree_size = GetVerityTreeSize(partition_size) 82 if not success: 83 return 0 84 success, verity_metadata_size = GetVerityMetadataSize(partition_size) 85 if not success: 86 return 0 87 verity_size = verity_tree_size + verity_metadata_size 88 if fec_supported: 89 success, fec_size = GetVerityFECSize(partition_size + verity_size) 90 if not success: 91 return 0 92 return verity_size + fec_size 93 return verity_size 94 95 def GetSimgSize(image_file): 96 simg = sparse_img.SparseImage(image_file, build_map=False) 97 return simg.blocksize * simg.total_blocks 98 99 def ZeroPadSimg(image_file, pad_size): 100 blocks = pad_size // BLOCK_SIZE 101 print("Padding %d blocks (%d bytes)" % (blocks, pad_size)) 102 simg = sparse_img.SparseImage(image_file, mode="r+b", build_map=False) 103 simg.AppendFillChunk(0, blocks) 104 105 def AdjustPartitionSizeForVerity(partition_size, fec_supported): 106 """Modifies the provided partition size to account for the verity metadata. 107 108 This information is used to size the created image appropriately. 109 Args: 110 partition_size: the size of the partition to be verified. 111 Returns: 112 The size of the partition adjusted for verity metadata. 113 """ 114 key = "%d %d" % (partition_size, fec_supported) 115 if key in AdjustPartitionSizeForVerity.results: 116 return AdjustPartitionSizeForVerity.results[key] 117 118 hi = partition_size 119 if hi % BLOCK_SIZE != 0: 120 hi = (hi // BLOCK_SIZE) * BLOCK_SIZE 121 122 # verity tree and fec sizes depend on the partition size, which 123 # means this estimate is always going to be unnecessarily small 124 lo = partition_size - GetVeritySize(hi, fec_supported) 125 result = lo 126 127 # do a binary search for the optimal size 128 while lo < hi: 129 i = ((lo + hi) // (2 * BLOCK_SIZE)) * BLOCK_SIZE 130 size = i + GetVeritySize(i, fec_supported) 131 if size <= partition_size: 132 if result < i: 133 result = i 134 lo = i + BLOCK_SIZE 135 else: 136 hi = i 137 138 AdjustPartitionSizeForVerity.results[key] = result 139 return result 140 141 AdjustPartitionSizeForVerity.results = {} 142 143 def BuildVerityFEC(sparse_image_path, verity_path, verity_fec_path): 144 cmd = "fec -e %s %s %s" % (sparse_image_path, verity_path, verity_fec_path) 145 print cmd 146 status, output = commands.getstatusoutput(cmd) 147 if status: 148 print "Could not build FEC data! Error: %s" % output 149 return False 150 return True 151 152 def BuildVerityTree(sparse_image_path, verity_image_path, prop_dict): 153 cmd = "build_verity_tree -A %s %s %s" % ( 154 FIXED_SALT, sparse_image_path, verity_image_path) 155 print cmd 156 status, output = commands.getstatusoutput(cmd) 157 if status: 158 print "Could not build verity tree! Error: %s" % output 159 return False 160 root, salt = output.split() 161 prop_dict["verity_root_hash"] = root 162 prop_dict["verity_salt"] = salt 163 return True 164 165 def BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt, 166 block_device, signer_path, key): 167 cmd_template = ( 168 "system/extras/verity/build_verity_metadata.py %s %s %s %s %s %s %s") 169 cmd = cmd_template % (image_size, verity_metadata_path, root_hash, salt, 170 block_device, signer_path, key) 171 print cmd 172 status, output = commands.getstatusoutput(cmd) 173 if status: 174 print "Could not build verity metadata! Error: %s" % output 175 return False 176 return True 177 178 def Append2Simg(sparse_image_path, unsparse_image_path, error_message): 179 """Appends the unsparse image to the given sparse image. 180 181 Args: 182 sparse_image_path: the path to the (sparse) image 183 unsparse_image_path: the path to the (unsparse) image 184 Returns: 185 True on success, False on failure. 186 """ 187 cmd = "append2simg %s %s" 188 cmd %= (sparse_image_path, unsparse_image_path) 189 print cmd 190 status, output = commands.getstatusoutput(cmd) 191 if status: 192 print "%s: %s" % (error_message, output) 193 return False 194 return True 195 196 def Append(target, file_to_append, error_message): 197 cmd = 'cat %s >> %s' % (file_to_append, target) 198 print cmd 199 status, output = commands.getstatusoutput(cmd) 200 if status: 201 print "%s: %s" % (error_message, output) 202 return False 203 return True 204 205 def BuildVerifiedImage(data_image_path, verity_image_path, 206 verity_metadata_path, verity_fec_path, 207 fec_supported): 208 if not Append(verity_image_path, verity_metadata_path, 209 "Could not append verity metadata!"): 210 return False 211 212 if fec_supported: 213 # build FEC for the entire partition, including metadata 214 if not BuildVerityFEC(data_image_path, verity_image_path, 215 verity_fec_path): 216 return False 217 218 if not Append(verity_image_path, verity_fec_path, "Could not append FEC!"): 219 return False 220 221 if not Append2Simg(data_image_path, verity_image_path, 222 "Could not append verity data!"): 223 return False 224 return True 225 226 def UnsparseImage(sparse_image_path, replace=True): 227 img_dir = os.path.dirname(sparse_image_path) 228 unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path) 229 unsparse_image_path = os.path.join(img_dir, unsparse_image_path) 230 if os.path.exists(unsparse_image_path): 231 if replace: 232 os.unlink(unsparse_image_path) 233 else: 234 return True, unsparse_image_path 235 inflate_command = ["simg2img", sparse_image_path, unsparse_image_path] 236 (_, exit_code) = RunCommand(inflate_command) 237 if exit_code != 0: 238 os.remove(unsparse_image_path) 239 return False, None 240 return True, unsparse_image_path 241 242 def MakeVerityEnabledImage(out_file, fec_supported, prop_dict): 243 """Creates an image that is verifiable using dm-verity. 244 245 Args: 246 out_file: the location to write the verifiable image at 247 prop_dict: a dictionary of properties required for image creation and 248 verification 249 Returns: 250 True on success, False otherwise. 251 """ 252 # get properties 253 image_size = prop_dict["partition_size"] 254 block_dev = prop_dict["verity_block_device"] 255 signer_key = prop_dict["verity_key"] + ".pk8" 256 if OPTIONS.verity_signer_path is not None: 257 signer_path = OPTIONS.verity_signer_path + ' ' 258 signer_path += ' '.join(OPTIONS.verity_signer_args) 259 else: 260 signer_path = prop_dict["verity_signer_cmd"] 261 262 # make a tempdir 263 tempdir_name = tempfile.mkdtemp(suffix="_verity_images") 264 265 # get partial image paths 266 verity_image_path = os.path.join(tempdir_name, "verity.img") 267 verity_metadata_path = os.path.join(tempdir_name, "verity_metadata.img") 268 verity_fec_path = os.path.join(tempdir_name, "verity_fec.img") 269 270 # build the verity tree and get the root hash and salt 271 if not BuildVerityTree(out_file, verity_image_path, prop_dict): 272 shutil.rmtree(tempdir_name, ignore_errors=True) 273 return False 274 275 # build the metadata blocks 276 root_hash = prop_dict["verity_root_hash"] 277 salt = prop_dict["verity_salt"] 278 if not BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt, 279 block_dev, signer_path, signer_key): 280 shutil.rmtree(tempdir_name, ignore_errors=True) 281 return False 282 283 # build the full verified image 284 if not BuildVerifiedImage(out_file, 285 verity_image_path, 286 verity_metadata_path, 287 verity_fec_path, 288 fec_supported): 289 shutil.rmtree(tempdir_name, ignore_errors=True) 290 return False 291 292 shutil.rmtree(tempdir_name, ignore_errors=True) 293 return True 294 295 def ConvertBlockMapToBaseFs(block_map_file): 296 fd, base_fs_file = tempfile.mkstemp(prefix="script_gen_", 297 suffix=".base_fs") 298 os.close(fd) 299 300 convert_command = ["blk_alloc_to_base_fs", block_map_file, base_fs_file] 301 (_, exit_code) = RunCommand(convert_command) 302 if exit_code != 0: 303 os.remove(base_fs_file) 304 return None 305 return base_fs_file 306 307 def BuildImage(in_dir, prop_dict, out_file, target_out=None): 308 """Build an image to out_file from in_dir with property prop_dict. 309 310 Args: 311 in_dir: path of input directory. 312 prop_dict: property dictionary. 313 out_file: path of the output image file. 314 target_out: path of the product out directory to read device specific FS config files. 315 316 Returns: 317 True iff the image is built successfully. 318 """ 319 # system_root_image=true: build a system.img that combines the contents of 320 # /system and the ramdisk, and can be mounted at the root of the file system. 321 origin_in = in_dir 322 fs_config = prop_dict.get("fs_config") 323 base_fs_file = None 324 if (prop_dict.get("system_root_image") == "true" 325 and prop_dict["mount_point"] == "system"): 326 in_dir = tempfile.mkdtemp() 327 # Change the mount point to "/" 328 prop_dict["mount_point"] = "/" 329 if fs_config: 330 # We need to merge the fs_config files of system and ramdisk. 331 fd, merged_fs_config = tempfile.mkstemp(prefix="root_fs_config", 332 suffix=".txt") 333 os.close(fd) 334 with open(merged_fs_config, "w") as fw: 335 if "ramdisk_fs_config" in prop_dict: 336 with open(prop_dict["ramdisk_fs_config"]) as fr: 337 fw.writelines(fr.readlines()) 338 with open(fs_config) as fr: 339 fw.writelines(fr.readlines()) 340 fs_config = merged_fs_config 341 342 build_command = [] 343 fs_type = prop_dict.get("fs_type", "") 344 run_fsck = False 345 346 fs_spans_partition = True 347 if fs_type.startswith("squash"): 348 fs_spans_partition = False 349 350 is_verity_partition = "verity_block_device" in prop_dict 351 verity_supported = prop_dict.get("verity") == "true" 352 verity_fec_supported = prop_dict.get("verity_fec") == "true" 353 354 # Adjust the partition size to make room for the hashes if this is to be 355 # verified. 356 if verity_supported and is_verity_partition: 357 partition_size = int(prop_dict.get("partition_size")) 358 adjusted_size = AdjustPartitionSizeForVerity(partition_size, 359 verity_fec_supported) 360 if not adjusted_size: 361 return False 362 prop_dict["partition_size"] = str(adjusted_size) 363 prop_dict["original_partition_size"] = str(partition_size) 364 365 if fs_type.startswith("ext"): 366 build_command = ["mkuserimg.sh"] 367 if "extfs_sparse_flag" in prop_dict: 368 build_command.append(prop_dict["extfs_sparse_flag"]) 369 run_fsck = True 370 build_command.extend([in_dir, out_file, fs_type, 371 prop_dict["mount_point"]]) 372 build_command.append(prop_dict["partition_size"]) 373 if "journal_size" in prop_dict: 374 build_command.extend(["-j", prop_dict["journal_size"]]) 375 if "timestamp" in prop_dict: 376 build_command.extend(["-T", str(prop_dict["timestamp"])]) 377 if fs_config: 378 build_command.extend(["-C", fs_config]) 379 if target_out: 380 build_command.extend(["-D", target_out]) 381 if "block_list" in prop_dict: 382 build_command.extend(["-B", prop_dict["block_list"]]) 383 if "base_fs_file" in prop_dict: 384 base_fs_file = ConvertBlockMapToBaseFs(prop_dict["base_fs_file"]) 385 if base_fs_file is None: 386 return False 387 build_command.extend(["-d", base_fs_file]) 388 build_command.extend(["-L", prop_dict["mount_point"]]) 389 if "selinux_fc" in prop_dict: 390 build_command.append(prop_dict["selinux_fc"]) 391 elif fs_type.startswith("squash"): 392 build_command = ["mksquashfsimage.sh"] 393 build_command.extend([in_dir, out_file]) 394 if "squashfs_sparse_flag" in prop_dict: 395 build_command.extend([prop_dict["squashfs_sparse_flag"]]) 396 build_command.extend(["-m", prop_dict["mount_point"]]) 397 if target_out: 398 build_command.extend(["-d", target_out]) 399 if fs_config: 400 build_command.extend(["-C", fs_config]) 401 if "selinux_fc" in prop_dict: 402 build_command.extend(["-c", prop_dict["selinux_fc"]]) 403 if "block_list" in prop_dict: 404 build_command.extend(["-B", prop_dict["block_list"]]) 405 if "squashfs_compressor" in prop_dict: 406 build_command.extend(["-z", prop_dict["squashfs_compressor"]]) 407 if "squashfs_compressor_opt" in prop_dict: 408 build_command.extend(["-zo", prop_dict["squashfs_compressor_opt"]]) 409 if "squashfs_disable_4k_align" in prop_dict and prop_dict.get("squashfs_disable_4k_align") == "true": 410 build_command.extend(["-a"]) 411 elif fs_type.startswith("f2fs"): 412 build_command = ["mkf2fsuserimg.sh"] 413 build_command.extend([out_file, prop_dict["partition_size"]]) 414 else: 415 build_command = ["mkyaffs2image", "-f"] 416 if prop_dict.get("mkyaffs2_extra_flags", None): 417 build_command.extend(prop_dict["mkyaffs2_extra_flags"].split()) 418 build_command.append(in_dir) 419 build_command.append(out_file) 420 if "selinux_fc" in prop_dict: 421 build_command.append(prop_dict["selinux_fc"]) 422 build_command.append(prop_dict["mount_point"]) 423 424 if in_dir != origin_in: 425 # Construct a staging directory of the root file system. 426 ramdisk_dir = prop_dict.get("ramdisk_dir") 427 if ramdisk_dir: 428 shutil.rmtree(in_dir) 429 shutil.copytree(ramdisk_dir, in_dir, symlinks=True) 430 staging_system = os.path.join(in_dir, "system") 431 shutil.rmtree(staging_system, ignore_errors=True) 432 shutil.copytree(origin_in, staging_system, symlinks=True) 433 434 reserved_blocks = prop_dict.get("has_ext4_reserved_blocks") == "true" 435 ext4fs_output = None 436 437 try: 438 if reserved_blocks and fs_type.startswith("ext4"): 439 (ext4fs_output, exit_code) = RunCommand(build_command) 440 else: 441 (_, exit_code) = RunCommand(build_command) 442 finally: 443 if in_dir != origin_in: 444 # Clean up temporary directories and files. 445 shutil.rmtree(in_dir, ignore_errors=True) 446 if fs_config: 447 os.remove(fs_config) 448 if base_fs_file is not None: 449 os.remove(base_fs_file) 450 if exit_code != 0: 451 return False 452 453 # Bug: 21522719, 22023465 454 # There are some reserved blocks on ext4 FS (lesser of 4096 blocks and 2%). 455 # We need to deduct those blocks from the available space, since they are 456 # not writable even with root privilege. It only affects devices using 457 # file-based OTA and a kernel version of 3.10 or greater (currently just 458 # sprout). 459 if reserved_blocks and fs_type.startswith("ext4"): 460 assert ext4fs_output is not None 461 ext4fs_stats = re.compile( 462 r'Created filesystem with .* (?P<used_blocks>[0-9]+)/' 463 r'(?P<total_blocks>[0-9]+) blocks') 464 m = ext4fs_stats.match(ext4fs_output.strip().split('\n')[-1]) 465 used_blocks = int(m.groupdict().get('used_blocks')) 466 total_blocks = int(m.groupdict().get('total_blocks')) 467 reserved_blocks = min(4096, int(total_blocks * 0.02)) 468 adjusted_blocks = total_blocks - reserved_blocks 469 if used_blocks > adjusted_blocks: 470 mount_point = prop_dict.get("mount_point") 471 print("Error: Not enough room on %s (total: %d blocks, used: %d blocks, " 472 "reserved: %d blocks, available: %d blocks)" % ( 473 mount_point, total_blocks, used_blocks, reserved_blocks, 474 adjusted_blocks)) 475 return False 476 477 if not fs_spans_partition: 478 mount_point = prop_dict.get("mount_point") 479 partition_size = int(prop_dict.get("partition_size")) 480 image_size = GetSimgSize(out_file) 481 if image_size > partition_size: 482 print("Error: %s image size of %d is larger than partition size of " 483 "%d" % (mount_point, image_size, partition_size)) 484 return False 485 if verity_supported and is_verity_partition: 486 ZeroPadSimg(out_file, partition_size - image_size) 487 488 # create the verified image if this is to be verified 489 if verity_supported and is_verity_partition: 490 if not MakeVerityEnabledImage(out_file, verity_fec_supported, prop_dict): 491 return False 492 493 if run_fsck and prop_dict.get("skip_fsck") != "true": 494 success, unsparse_image = UnsparseImage(out_file, replace=False) 495 if not success: 496 return False 497 498 # Run e2fsck on the inflated image file 499 e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image] 500 (_, exit_code) = RunCommand(e2fsck_command) 501 502 os.remove(unsparse_image) 503 504 return exit_code == 0 505 506 507 def ImagePropFromGlobalDict(glob_dict, mount_point): 508 """Build an image property dictionary from the global dictionary. 509 510 Args: 511 glob_dict: the global dictionary from the build system. 512 mount_point: such as "system", "data" etc. 513 """ 514 d = {} 515 516 if "build.prop" in glob_dict: 517 bp = glob_dict["build.prop"] 518 if "ro.build.date.utc" in bp: 519 d["timestamp"] = bp["ro.build.date.utc"] 520 521 def copy_prop(src_p, dest_p): 522 if src_p in glob_dict: 523 d[dest_p] = str(glob_dict[src_p]) 524 525 common_props = ( 526 "extfs_sparse_flag", 527 "squashfs_sparse_flag", 528 "mkyaffs2_extra_flags", 529 "selinux_fc", 530 "skip_fsck", 531 "verity", 532 "verity_key", 533 "verity_signer_cmd", 534 "verity_fec" 535 ) 536 for p in common_props: 537 copy_prop(p, p) 538 539 d["mount_point"] = mount_point 540 if mount_point == "system": 541 copy_prop("fs_type", "fs_type") 542 # Copy the generic sysetem fs type first, override with specific one if 543 # available. 544 copy_prop("system_fs_type", "fs_type") 545 copy_prop("system_size", "partition_size") 546 copy_prop("system_journal_size", "journal_size") 547 copy_prop("system_verity_block_device", "verity_block_device") 548 copy_prop("system_root_image", "system_root_image") 549 copy_prop("ramdisk_dir", "ramdisk_dir") 550 copy_prop("ramdisk_fs_config", "ramdisk_fs_config") 551 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 552 copy_prop("system_squashfs_compressor", "squashfs_compressor") 553 copy_prop("system_squashfs_compressor_opt", "squashfs_compressor_opt") 554 copy_prop("system_squashfs_disable_4k_align", "squashfs_disable_4k_align") 555 copy_prop("system_base_fs_file", "base_fs_file") 556 elif mount_point == "data": 557 # Copy the generic fs type first, override with specific one if available. 558 copy_prop("fs_type", "fs_type") 559 copy_prop("userdata_fs_type", "fs_type") 560 copy_prop("userdata_size", "partition_size") 561 elif mount_point == "cache": 562 copy_prop("cache_fs_type", "fs_type") 563 copy_prop("cache_size", "partition_size") 564 elif mount_point == "vendor": 565 copy_prop("vendor_fs_type", "fs_type") 566 copy_prop("vendor_size", "partition_size") 567 copy_prop("vendor_journal_size", "journal_size") 568 copy_prop("vendor_verity_block_device", "verity_block_device") 569 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 570 copy_prop("vendor_squashfs_compressor", "squashfs_compressor") 571 copy_prop("vendor_squashfs_compressor_opt", "squashfs_compressor_opt") 572 copy_prop("vendor_squashfs_disable_4k_align", "squashfs_disable_4k_align") 573 copy_prop("vendor_base_fs_file", "base_fs_file") 574 elif mount_point == "oem": 575 copy_prop("fs_type", "fs_type") 576 copy_prop("oem_size", "partition_size") 577 copy_prop("oem_journal_size", "journal_size") 578 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 579 580 return d 581 582 583 def LoadGlobalDict(filename): 584 """Load "name=value" pairs from filename""" 585 d = {} 586 f = open(filename) 587 for line in f: 588 line = line.strip() 589 if not line or line.startswith("#"): 590 continue 591 k, v = line.split("=", 1) 592 d[k] = v 593 f.close() 594 return d 595 596 597 def main(argv): 598 if len(argv) != 4: 599 print __doc__ 600 sys.exit(1) 601 602 in_dir = argv[0] 603 glob_dict_file = argv[1] 604 out_file = argv[2] 605 target_out = argv[3] 606 607 glob_dict = LoadGlobalDict(glob_dict_file) 608 if "mount_point" in glob_dict: 609 # The caller knows the mount point and provides a dictionay needed by 610 # BuildImage(). 611 image_properties = glob_dict 612 else: 613 image_filename = os.path.basename(out_file) 614 mount_point = "" 615 if image_filename == "system.img": 616 mount_point = "system" 617 elif image_filename == "userdata.img": 618 mount_point = "data" 619 elif image_filename == "cache.img": 620 mount_point = "cache" 621 elif image_filename == "vendor.img": 622 mount_point = "vendor" 623 elif image_filename == "oem.img": 624 mount_point = "oem" 625 else: 626 print >> sys.stderr, "error: unknown image file name ", image_filename 627 exit(1) 628 629 image_properties = ImagePropFromGlobalDict(glob_dict, mount_point) 630 631 if not BuildImage(in_dir, image_properties, out_file, target_out): 632 print >> sys.stderr, "error: failed to build %s from %s" % (out_file, 633 in_dir) 634 exit(1) 635 636 637 if __name__ == '__main__': 638 main(sys.argv[1:]) 639
