1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import fnmatch 6 import glob 7 import logging 8 import os 9 10 from autotest_lib.client.bin import test, utils 11 from autotest_lib.client.common_lib import error 12 from optparse import OptionParser 13 14 FILE_CMD="file -m /usr/local/share/misc/magic.mgc" 15 16 class ToolchainOptionSet: 17 """ 18 Handles a set of hits, along with potential whitelists to ignore. 19 """ 20 def __init__(self, description, bad_files, whitelist_file): 21 self.description = description 22 self.bad_set = set(bad_files.splitlines()) 23 self.whitelist_set = set([]) 24 self.process_whitelist_with_private(whitelist_file) 25 26 27 def process_whitelist_with_private(self, whitelist_file): 28 """ 29 Filter out hits found on non-comment lines in the whitelist and 30 and private whitelist. 31 32 @param whitelist_file: path to whitelist file 33 """ 34 whitelist_files = [whitelist_file] 35 private_file = os.path.join(os.path.dirname(whitelist_file), 36 "private_" + 37 os.path.basename(whitelist_file)) 38 whitelist_files.append(private_file) 39 self.process_whitelists(whitelist_files) 40 41 42 def process_whitelist(self, whitelist_file): 43 """ 44 Filter out hits found on non-comment lines in the whitelist. 45 46 @param whitelist_file: path to whitelist file 47 """ 48 if os.path.isfile(whitelist_file): 49 f = open(whitelist_file) 50 whitelist = [x for x in f.read().splitlines() 51 if not x.startswith('#')] 52 f.close() 53 self.whitelist_set = self.whitelist_set.union(set(whitelist)) 54 55 filtered_list = [] 56 for bad_file in self.bad_set: 57 # Does |bad_file| match any entry in the whitelist? 58 in_whitelist = any([fnmatch.fnmatch(bad_file, whitelist_entry) 59 for whitelist_entry in self.whitelist_set]) 60 if not in_whitelist: 61 filtered_list.append(bad_file) 62 63 self.filtered_set = set(filtered_list) 64 # TODO(jorgelo): remove glob patterns from |new_passes|. 65 self.new_passes = self.whitelist_set.difference(self.bad_set) 66 67 68 def process_whitelists(self, whitelist_files): 69 """ 70 Filter out hits found in a list of whitelist files. 71 72 @param whitelist_files: list of paths to whitelist files 73 """ 74 for whitelist_file in whitelist_files: 75 self.process_whitelist(whitelist_file) 76 77 78 def get_fail_summary_message(self): 79 m = "Test %s " % self.description 80 m += "%d failures" % len(self.filtered_set) 81 return m 82 83 84 def get_fail_message(self): 85 m = self.get_fail_summary_message() 86 sorted_list = list(self.filtered_set) 87 sorted_list.sort() 88 m += "\nFAILED:\n%s\n\n" % "\n".join(sorted_list) 89 return m 90 91 92 def __str__(self): 93 m = "Test %s " % self.description 94 m += ("%d failures, %d in whitelist, %d in filtered, %d new passes " % 95 (len(self.bad_set), 96 len(self.whitelist_set), 97 len(self.filtered_set), 98 len(self.new_passes))) 99 100 if len(self.filtered_set): 101 sorted_list = list(self.filtered_set) 102 sorted_list.sort() 103 m += "FAILED:\n%s" % "\n".join(sorted_list) 104 else: 105 m += "PASSED!" 106 107 if len(self.new_passes): 108 sorted_list = list(self.new_passes) 109 sorted_list.sort() 110 m += ("\nNew passes (remove these from the whitelist):\n%s" % 111 "\n".join(sorted_list)) 112 logging.debug(m) 113 return m 114 115 116 class platform_ToolchainOptions(test.test): 117 """ 118 Tests for various expected conditions on ELF binaries in the image. 119 """ 120 version = 2 121 122 def get_cmd(self, test_cmd, find_options=""): 123 base_cmd = ("find '%s' -wholename %s -prune -o " 124 " -wholename /proc -prune -o " 125 " -wholename /dev -prune -o " 126 " -wholename /sys -prune -o " 127 " -wholename /mnt/stateful_partition -prune -o " 128 " -wholename /usr/local -prune -o " 129 # There are files in /home/chronos that cause false 130 # positives, and since that's noexec anyways, it should 131 # be skipped. 132 " -wholename '/home/chronos' -prune -o " 133 " %s " 134 " -not -name 'libstdc++.so.*' " 135 " -not -name 'libgcc_s.so.*' " 136 " -type f -executable -exec " 137 "sh -c '%s " 138 "{} | grep -q ELF && " 139 "(%s || echo {})' ';'") 140 rootdir = "/" 141 cmd = base_cmd % (rootdir, self.autodir, find_options, FILE_CMD, 142 test_cmd) 143 return cmd 144 145 146 def create_and_filter(self, description, cmd, whitelist_file, 147 find_options=""): 148 """ 149 Runs a command, with "{}" replaced (via "find -exec") with the 150 target ELF binary. If the command fails, the file is marked as 151 failing the test. Results are filtered against the provided 152 whitelist file. 153 154 @param description: text name of the check being done 155 @param cmd: command to run via find's -exec option 156 @param whitelist_file: list of failures to ignore 157 @param find_options: additional options for find to limit the scope 158 """ 159 full_cmd = self.get_cmd(cmd, find_options) 160 bad_files = utils.system_output(full_cmd) 161 cso = ToolchainOptionSet(description, bad_files, whitelist_file) 162 cso.process_whitelist_with_private(whitelist_file) 163 return cso 164 165 166 def run_once(self, rootdir="/", args=[]): 167 """ 168 Do a find for all the ELF files on the system. 169 For each one, test for compiler options that should have been used 170 when compiling the file. 171 172 For missing compiler options, print the files. 173 """ 174 175 parser = OptionParser() 176 parser.add_option('--hardfp', 177 dest='enable_hardfp', 178 default=False, 179 action='store_true', 180 help='Whether to check for hardfp binaries.') 181 (options, args) = parser.parse_args(args) 182 183 option_sets = [] 184 185 libc_glob = "/lib/libc-[0-9]*" 186 187 readelf_cmd = glob.glob("/usr/local/*/binutils-bin/*/readelf")[0] 188 189 # We do not test binaries if they are built with Address Sanitizer 190 # because it is a separate testing tool. 191 no_asan_used = utils.system_output("%s -s " 192 "/opt/google/chrome/chrome | " 193 "egrep -q \"__asan_init\" || " 194 "echo no ASAN" % readelf_cmd) 195 if not no_asan_used: 196 logging.debug("ASAN detected on /opt/google/chrome/chrome. " 197 "Will skip all checks.") 198 return 199 200 # Check that gold was used to build binaries. 201 # TODO(jorgelo): re-enable this check once crbug.com/417912 is fixed. 202 # gold_cmd = ("%s -S {} 2>&1 | " 203 # "egrep -q \".note.gnu.gold-ve\"" % readelf_cmd) 204 # gold_find_options = "" 205 # if utils.get_cpu_arch() == "arm": 206 # # gold is only enabled for Chrome on ARM. 207 # gold_find_options = "-path \"/opt/google/chrome/chrome\"" 208 # gold_whitelist = os.path.join(self.bindir, "gold_whitelist") 209 # option_sets.append(self.create_and_filter("gold", 210 # gold_cmd, 211 # gold_whitelist, 212 # gold_find_options)) 213 214 # Verify non-static binaries have BIND_NOW in dynamic section. 215 now_cmd = ("(%s {} | grep -q statically) ||" 216 "%s -d {} 2>&1 | " 217 "egrep -q \"BIND_NOW\"" % (FILE_CMD, readelf_cmd)) 218 if utils.is_freon(): 219 now_whitelist = os.path.join(self.bindir, "now_whitelist") 220 else: 221 now_whitelist = os.path.join(self.bindir, "now_whitelist_x") 222 option_sets.append(self.create_and_filter("-Wl,-z,now", 223 now_cmd, 224 now_whitelist)) 225 226 # Verify non-static binaries have RELRO program header. 227 relro_cmd = ("(%s {} | grep -q statically) ||" 228 "%s -l {} 2>&1 | " 229 "egrep -q \"GNU_RELRO\"" % (FILE_CMD, readelf_cmd)) 230 relro_whitelist = os.path.join(self.bindir, "relro_whitelist") 231 option_sets.append(self.create_and_filter("-Wl,-z,relro", 232 relro_cmd, 233 relro_whitelist)) 234 235 # Verify non-static binaries are dynamic (built PIE). 236 pie_cmd = ("(%s {} | grep -q statically) ||" 237 "%s -l {} 2>&1 | " 238 "egrep -q \"Elf file type is DYN\"" % (FILE_CMD, 239 readelf_cmd)) 240 pie_whitelist = os.path.join(self.bindir, "pie_whitelist") 241 option_sets.append(self.create_and_filter("-fPIE", 242 pie_cmd, 243 pie_whitelist)) 244 245 # Verify all binaries have non-exec STACK program header. 246 stack_cmd = ("%s -lW {} 2>&1 | " 247 "egrep -q \"GNU_STACK.*RW \"" % readelf_cmd) 248 stack_whitelist = os.path.join(self.bindir, "stack_whitelist") 249 option_sets.append(self.create_and_filter("Executable Stack", 250 stack_cmd, 251 stack_whitelist)) 252 253 # Verify all binaries have W^X LOAD program headers. 254 loadwx_cmd = ("%s -lW {} 2>&1 | " 255 "grep \"LOAD\" | egrep -v \"(RW |R E)\" | " 256 "wc -l | grep -q \"^0$\"" % readelf_cmd) 257 if utils.is_freon(): 258 loadwx_whitelist = os.path.join(self.bindir, "loadwx_whitelist") 259 else: 260 loadwx_whitelist = os.path.join(self.bindir, "loadwx_whitelist_x") 261 option_sets.append(self.create_and_filter("LOAD Writable and Exec", 262 loadwx_cmd, 263 loadwx_whitelist)) 264 265 # Verify ARM binaries are all using VFP registers. 266 if (options.enable_hardfp and utils.get_cpu_arch() == 'arm'): 267 hardfp_cmd = ("%s -A {} 2>&1 | " 268 "egrep -q \"Tag_ABI_VFP_args: VFP registers\"" % 269 readelf_cmd) 270 hardfp_whitelist = os.path.join(self.bindir, "hardfp_whitelist") 271 option_sets.append(self.create_and_filter("hardfp", hardfp_cmd, 272 hardfp_whitelist)) 273 274 fail_msg = "" 275 276 # There is currently no way to clear binary prebuilts for all devs. 277 # Thus, when a new check is added to this test, the test might fail 278 # for users who have old prebuilts which have not been compiled 279 # in the correct manner. 280 fail_summaries = [] 281 full_msg = "Test results:" 282 num_fails = 0 283 for cos in option_sets: 284 if len(cos.filtered_set): 285 num_fails += 1 286 fail_msg += cos.get_fail_message() + "\n" 287 fail_summaries.append(cos.get_fail_summary_message()) 288 full_msg += str(cos) + "\n\n" 289 fail_summary_msg = ", ".join(fail_summaries) 290 291 logging.error(fail_msg) 292 logging.debug(full_msg) 293 if num_fails: 294 raise error.TestFail(fail_summary_msg) 295
