1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2011 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 Build image output_image_file from input_directory and properties_file. 19 20 Usage: build_image input_directory properties_file output_image_file 21 22 """ 23 import os 24 import os.path 25 import re 26 import subprocess 27 import sys 28 import commands 29 import common 30 import shutil 31 import tempfile 32 33 OPTIONS = common.OPTIONS 34 35 FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7" 36 37 def RunCommand(cmd): 38 """Echo and run the given command. 39 40 Args: 41 cmd: the command represented as a list of strings. 42 Returns: 43 A tuple of the output and the exit code. 44 """ 45 print "Running: ", " ".join(cmd) 46 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 47 output, _ = p.communicate() 48 print "%s" % (output.rstrip(),) 49 return (output, p.returncode) 50 51 def GetVerityTreeSize(partition_size): 52 cmd = "build_verity_tree -s %d" 53 cmd %= partition_size 54 status, output = commands.getstatusoutput(cmd) 55 if status: 56 print output 57 return False, 0 58 return True, int(output) 59 60 def GetVerityMetadataSize(partition_size): 61 cmd = "system/extras/verity/build_verity_metadata.py -s %d" 62 cmd %= partition_size 63 64 status, output = commands.getstatusoutput(cmd) 65 if status: 66 print output 67 return False, 0 68 return True, int(output) 69 70 def AdjustPartitionSizeForVerity(partition_size): 71 """Modifies the provided partition size to account for the verity metadata. 72 73 This information is used to size the created image appropriately. 74 Args: 75 partition_size: the size of the partition to be verified. 76 Returns: 77 The size of the partition adjusted for verity metadata. 78 """ 79 success, verity_tree_size = GetVerityTreeSize(partition_size) 80 if not success: 81 return 0 82 success, verity_metadata_size = GetVerityMetadataSize(partition_size) 83 if not success: 84 return 0 85 return partition_size - verity_tree_size - verity_metadata_size 86 87 def BuildVerityTree(sparse_image_path, verity_image_path, prop_dict): 88 cmd = "build_verity_tree -A %s %s %s" % ( 89 FIXED_SALT, sparse_image_path, verity_image_path) 90 print cmd 91 status, output = commands.getstatusoutput(cmd) 92 if status: 93 print "Could not build verity tree! Error: %s" % output 94 return False 95 root, salt = output.split() 96 prop_dict["verity_root_hash"] = root 97 prop_dict["verity_salt"] = salt 98 return True 99 100 def BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt, 101 block_device, signer_path, key): 102 cmd_template = ( 103 "system/extras/verity/build_verity_metadata.py %s %s %s %s %s %s %s") 104 cmd = cmd_template % (image_size, verity_metadata_path, root_hash, salt, 105 block_device, signer_path, key) 106 print cmd 107 status, output = commands.getstatusoutput(cmd) 108 if status: 109 print "Could not build verity metadata! Error: %s" % output 110 return False 111 return True 112 113 def Append2Simg(sparse_image_path, unsparse_image_path, error_message): 114 """Appends the unsparse image to the given sparse image. 115 116 Args: 117 sparse_image_path: the path to the (sparse) image 118 unsparse_image_path: the path to the (unsparse) image 119 Returns: 120 True on success, False on failure. 121 """ 122 cmd = "append2simg %s %s" 123 cmd %= (sparse_image_path, unsparse_image_path) 124 print cmd 125 status, output = commands.getstatusoutput(cmd) 126 if status: 127 print "%s: %s" % (error_message, output) 128 return False 129 return True 130 131 def BuildVerifiedImage(data_image_path, verity_image_path, 132 verity_metadata_path): 133 if not Append2Simg(data_image_path, verity_metadata_path, 134 "Could not append verity metadata!"): 135 return False 136 if not Append2Simg(data_image_path, verity_image_path, 137 "Could not append verity tree!"): 138 return False 139 return True 140 141 def UnsparseImage(sparse_image_path, replace=True): 142 img_dir = os.path.dirname(sparse_image_path) 143 unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path) 144 unsparse_image_path = os.path.join(img_dir, unsparse_image_path) 145 if os.path.exists(unsparse_image_path): 146 if replace: 147 os.unlink(unsparse_image_path) 148 else: 149 return True, unsparse_image_path 150 inflate_command = ["simg2img", sparse_image_path, unsparse_image_path] 151 (_, exit_code) = RunCommand(inflate_command) 152 if exit_code != 0: 153 os.remove(unsparse_image_path) 154 return False, None 155 return True, unsparse_image_path 156 157 def MakeVerityEnabledImage(out_file, prop_dict): 158 """Creates an image that is verifiable using dm-verity. 159 160 Args: 161 out_file: the location to write the verifiable image at 162 prop_dict: a dictionary of properties required for image creation and 163 verification 164 Returns: 165 True on success, False otherwise. 166 """ 167 # get properties 168 image_size = prop_dict["partition_size"] 169 block_dev = prop_dict["verity_block_device"] 170 signer_key = prop_dict["verity_key"] + ".pk8" 171 if OPTIONS.verity_signer_path is not None: 172 signer_path = OPTIONS.verity_signer_path + ' ' 173 signer_path += ' '.join(OPTIONS.verity_signer_args) 174 else: 175 signer_path = prop_dict["verity_signer_cmd"] 176 177 # make a tempdir 178 tempdir_name = tempfile.mkdtemp(suffix="_verity_images") 179 180 # get partial image paths 181 verity_image_path = os.path.join(tempdir_name, "verity.img") 182 verity_metadata_path = os.path.join(tempdir_name, "verity_metadata.img") 183 184 # build the verity tree and get the root hash and salt 185 if not BuildVerityTree(out_file, verity_image_path, prop_dict): 186 shutil.rmtree(tempdir_name, ignore_errors=True) 187 return False 188 189 # build the metadata blocks 190 root_hash = prop_dict["verity_root_hash"] 191 salt = prop_dict["verity_salt"] 192 if not BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt, 193 block_dev, signer_path, signer_key): 194 shutil.rmtree(tempdir_name, ignore_errors=True) 195 return False 196 197 # build the full verified image 198 if not BuildVerifiedImage(out_file, 199 verity_image_path, 200 verity_metadata_path): 201 shutil.rmtree(tempdir_name, ignore_errors=True) 202 return False 203 204 shutil.rmtree(tempdir_name, ignore_errors=True) 205 return True 206 207 def BuildImage(in_dir, prop_dict, out_file, target_out=None): 208 """Build an image to out_file from in_dir with property prop_dict. 209 210 Args: 211 in_dir: path of input directory. 212 prop_dict: property dictionary. 213 out_file: path of the output image file. 214 target_out: path of the product out directory to read device specific FS config files. 215 216 Returns: 217 True iff the image is built successfully. 218 """ 219 # system_root_image=true: build a system.img that combines the contents of 220 # /system and the ramdisk, and can be mounted at the root of the file system. 221 origin_in = in_dir 222 fs_config = prop_dict.get("fs_config") 223 if (prop_dict.get("system_root_image") == "true" 224 and prop_dict["mount_point"] == "system"): 225 in_dir = tempfile.mkdtemp() 226 # Change the mount point to "/" 227 prop_dict["mount_point"] = "/" 228 if fs_config: 229 # We need to merge the fs_config files of system and ramdisk. 230 fd, merged_fs_config = tempfile.mkstemp(prefix="root_fs_config", 231 suffix=".txt") 232 os.close(fd) 233 with open(merged_fs_config, "w") as fw: 234 if "ramdisk_fs_config" in prop_dict: 235 with open(prop_dict["ramdisk_fs_config"]) as fr: 236 fw.writelines(fr.readlines()) 237 with open(fs_config) as fr: 238 fw.writelines(fr.readlines()) 239 fs_config = merged_fs_config 240 241 build_command = [] 242 fs_type = prop_dict.get("fs_type", "") 243 run_fsck = False 244 245 fs_spans_partition = True 246 if fs_type.startswith("squash"): 247 fs_spans_partition = False 248 249 is_verity_partition = "verity_block_device" in prop_dict 250 verity_supported = prop_dict.get("verity") == "true" 251 # Adjust the partition size to make room for the hashes if this is to be 252 # verified. 253 if verity_supported and is_verity_partition and fs_spans_partition: 254 partition_size = int(prop_dict.get("partition_size")) 255 256 adjusted_size = AdjustPartitionSizeForVerity(partition_size) 257 if not adjusted_size: 258 return False 259 prop_dict["partition_size"] = str(adjusted_size) 260 prop_dict["original_partition_size"] = str(partition_size) 261 262 if fs_type.startswith("ext"): 263 build_command = ["mkuserimg.sh"] 264 if "extfs_sparse_flag" in prop_dict: 265 build_command.append(prop_dict["extfs_sparse_flag"]) 266 run_fsck = True 267 build_command.extend([in_dir, out_file, fs_type, 268 prop_dict["mount_point"]]) 269 build_command.append(prop_dict["partition_size"]) 270 if "journal_size" in prop_dict: 271 build_command.extend(["-j", prop_dict["journal_size"]]) 272 if "timestamp" in prop_dict: 273 build_command.extend(["-T", str(prop_dict["timestamp"])]) 274 if fs_config: 275 build_command.extend(["-C", fs_config]) 276 if target_out: 277 build_command.extend(["-D", target_out]) 278 if "block_list" in prop_dict: 279 build_command.extend(["-B", prop_dict["block_list"]]) 280 build_command.extend(["-L", prop_dict["mount_point"]]) 281 if "selinux_fc" in prop_dict: 282 build_command.append(prop_dict["selinux_fc"]) 283 elif fs_type.startswith("squash"): 284 build_command = ["mksquashfsimage.sh"] 285 build_command.extend([in_dir, out_file]) 286 build_command.extend(["-s"]) 287 build_command.extend(["-m", prop_dict["mount_point"]]) 288 if target_out: 289 build_command.extend(["-d", target_out]) 290 if "selinux_fc" in prop_dict: 291 build_command.extend(["-c", prop_dict["selinux_fc"]]) 292 if "squashfs_compressor" in prop_dict: 293 build_command.extend(["-z", prop_dict["squashfs_compressor"]]) 294 if "squashfs_compressor_opt" in prop_dict: 295 build_command.extend(["-zo", prop_dict["squashfs_compressor_opt"]]) 296 elif fs_type.startswith("f2fs"): 297 build_command = ["mkf2fsuserimg.sh"] 298 build_command.extend([out_file, prop_dict["partition_size"]]) 299 else: 300 build_command = ["mkyaffs2image", "-f"] 301 if prop_dict.get("mkyaffs2_extra_flags", None): 302 build_command.extend(prop_dict["mkyaffs2_extra_flags"].split()) 303 build_command.append(in_dir) 304 build_command.append(out_file) 305 if "selinux_fc" in prop_dict: 306 build_command.append(prop_dict["selinux_fc"]) 307 build_command.append(prop_dict["mount_point"]) 308 309 if in_dir != origin_in: 310 # Construct a staging directory of the root file system. 311 ramdisk_dir = prop_dict.get("ramdisk_dir") 312 if ramdisk_dir: 313 shutil.rmtree(in_dir) 314 shutil.copytree(ramdisk_dir, in_dir, symlinks=True) 315 staging_system = os.path.join(in_dir, "system") 316 shutil.rmtree(staging_system, ignore_errors=True) 317 shutil.copytree(origin_in, staging_system, symlinks=True) 318 319 reserved_blocks = prop_dict.get("has_ext4_reserved_blocks") == "true" 320 ext4fs_output = None 321 322 try: 323 if reserved_blocks and fs_type.startswith("ext4"): 324 (ext4fs_output, exit_code) = RunCommand(build_command) 325 else: 326 (_, exit_code) = RunCommand(build_command) 327 finally: 328 if in_dir != origin_in: 329 # Clean up temporary directories and files. 330 shutil.rmtree(in_dir, ignore_errors=True) 331 if fs_config: 332 os.remove(fs_config) 333 if exit_code != 0: 334 return False 335 336 # Bug: 21522719, 22023465 337 # There are some reserved blocks on ext4 FS (lesser of 4096 blocks and 2%). 338 # We need to deduct those blocks from the available space, since they are 339 # not writable even with root privilege. It only affects devices using 340 # file-based OTA and a kernel version of 3.10 or greater (currently just 341 # sprout). 342 if reserved_blocks and fs_type.startswith("ext4"): 343 assert ext4fs_output is not None 344 ext4fs_stats = re.compile( 345 r'Created filesystem with .* (?P<used_blocks>[0-9]+)/' 346 r'(?P<total_blocks>[0-9]+) blocks') 347 m = ext4fs_stats.match(ext4fs_output.strip().split('\n')[-1]) 348 used_blocks = int(m.groupdict().get('used_blocks')) 349 total_blocks = int(m.groupdict().get('total_blocks')) 350 reserved_blocks = min(4096, int(total_blocks * 0.02)) 351 adjusted_blocks = total_blocks - reserved_blocks 352 if used_blocks > adjusted_blocks: 353 mount_point = prop_dict.get("mount_point") 354 print("Error: Not enough room on %s (total: %d blocks, used: %d blocks, " 355 "reserved: %d blocks, available: %d blocks)" % ( 356 mount_point, total_blocks, used_blocks, reserved_blocks, 357 adjusted_blocks)) 358 return False 359 360 if not fs_spans_partition: 361 mount_point = prop_dict.get("mount_point") 362 partition_size = int(prop_dict.get("partition_size")) 363 image_size = os.stat(out_file).st_size 364 if image_size > partition_size: 365 print("Error: %s image size of %d is larger than partition size of " 366 "%d" % (mount_point, image_size, partition_size)) 367 return False 368 if verity_supported and is_verity_partition: 369 if 2 * image_size - AdjustPartitionSizeForVerity(image_size) > partition_size: 370 print "Error: No more room on %s to fit verity data" % mount_point 371 return False 372 prop_dict["original_partition_size"] = prop_dict["partition_size"] 373 prop_dict["partition_size"] = str(image_size) 374 375 # create the verified image if this is to be verified 376 if verity_supported and is_verity_partition: 377 if not MakeVerityEnabledImage(out_file, prop_dict): 378 return False 379 380 if run_fsck and prop_dict.get("skip_fsck") != "true": 381 success, unsparse_image = UnsparseImage(out_file, replace=False) 382 if not success: 383 return False 384 385 # Run e2fsck on the inflated image file 386 e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image] 387 (_, exit_code) = RunCommand(e2fsck_command) 388 389 os.remove(unsparse_image) 390 391 return exit_code == 0 392 393 394 def ImagePropFromGlobalDict(glob_dict, mount_point): 395 """Build an image property dictionary from the global dictionary. 396 397 Args: 398 glob_dict: the global dictionary from the build system. 399 mount_point: such as "system", "data" etc. 400 """ 401 d = {} 402 if "build.prop" in glob_dict: 403 bp = glob_dict["build.prop"] 404 if "ro.build.date.utc" in bp: 405 d["timestamp"] = bp["ro.build.date.utc"] 406 407 def copy_prop(src_p, dest_p): 408 if src_p in glob_dict: 409 d[dest_p] = str(glob_dict[src_p]) 410 411 common_props = ( 412 "extfs_sparse_flag", 413 "mkyaffs2_extra_flags", 414 "selinux_fc", 415 "skip_fsck", 416 "verity", 417 "verity_key", 418 "verity_signer_cmd" 419 ) 420 for p in common_props: 421 copy_prop(p, p) 422 423 d["mount_point"] = mount_point 424 if mount_point == "system": 425 copy_prop("fs_type", "fs_type") 426 # Copy the generic sysetem fs type first, override with specific one if 427 # available. 428 copy_prop("system_fs_type", "fs_type") 429 copy_prop("system_size", "partition_size") 430 copy_prop("system_journal_size", "journal_size") 431 copy_prop("system_verity_block_device", "verity_block_device") 432 copy_prop("system_root_image", "system_root_image") 433 copy_prop("ramdisk_dir", "ramdisk_dir") 434 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 435 copy_prop("system_squashfs_compressor", "squashfs_compressor") 436 copy_prop("system_squashfs_compressor_opt", "squashfs_compressor_opt") 437 elif mount_point == "data": 438 # Copy the generic fs type first, override with specific one if available. 439 copy_prop("fs_type", "fs_type") 440 copy_prop("userdata_fs_type", "fs_type") 441 copy_prop("userdata_size", "partition_size") 442 elif mount_point == "cache": 443 copy_prop("cache_fs_type", "fs_type") 444 copy_prop("cache_size", "partition_size") 445 elif mount_point == "vendor": 446 copy_prop("vendor_fs_type", "fs_type") 447 copy_prop("vendor_size", "partition_size") 448 copy_prop("vendor_journal_size", "journal_size") 449 copy_prop("vendor_verity_block_device", "verity_block_device") 450 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 451 elif mount_point == "oem": 452 copy_prop("fs_type", "fs_type") 453 copy_prop("oem_size", "partition_size") 454 copy_prop("oem_journal_size", "journal_size") 455 copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks") 456 457 return d 458 459 460 def LoadGlobalDict(filename): 461 """Load "name=value" pairs from filename""" 462 d = {} 463 f = open(filename) 464 for line in f: 465 line = line.strip() 466 if not line or line.startswith("#"): 467 continue 468 k, v = line.split("=", 1) 469 d[k] = v 470 f.close() 471 return d 472 473 474 def main(argv): 475 if len(argv) != 4: 476 print __doc__ 477 sys.exit(1) 478 479 in_dir = argv[0] 480 glob_dict_file = argv[1] 481 out_file = argv[2] 482 target_out = argv[3] 483 484 glob_dict = LoadGlobalDict(glob_dict_file) 485 if "mount_point" in glob_dict: 486 # The caller knows the mount point and provides a dictionay needed by 487 # BuildImage(). 488 image_properties = glob_dict 489 else: 490 image_filename = os.path.basename(out_file) 491 mount_point = "" 492 if image_filename == "system.img": 493 mount_point = "system" 494 elif image_filename == "userdata.img": 495 mount_point = "data" 496 elif image_filename == "cache.img": 497 mount_point = "cache" 498 elif image_filename == "vendor.img": 499 mount_point = "vendor" 500 elif image_filename == "oem.img": 501 mount_point = "oem" 502 else: 503 print >> sys.stderr, "error: unknown image file name ", image_filename 504 exit(1) 505 506 image_properties = ImagePropFromGlobalDict(glob_dict, mount_point) 507 508 if not BuildImage(in_dir, image_properties, out_file, target_out): 509 print >> sys.stderr, "error: failed to build %s from %s" % (out_file, 510 in_dir) 511 exit(1) 512 513 514 if __name__ == '__main__': 515 main(sys.argv[1:]) 516
