1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import fnmatch 6 import glob 7 import logging 8 import os 9 10 from autotest_lib.client.bin import test, utils 11 from autotest_lib.client.common_lib import error 12 from optparse import OptionParser 13 14 FILE_CMD="file -m /usr/local/share/misc/magic.mgc" 15 16 class ToolchainOptionSet: 17 """ 18 Handles a set of hits, along with potential whitelists to ignore. 19 """ 20 def __init__(self, description, bad_files, whitelist_file): 21 self.description = description 22 self.bad_set = set(bad_files.splitlines()) 23 self.whitelist_set = set([]) 24 self.process_whitelist_with_private(whitelist_file) 25 26 27 def process_whitelist_with_private(self, whitelist_file): 28 """ 29 Filter out hits found on non-comment lines in the whitelist and 30 and private whitelist. 31 32 @param whitelist_file: path to whitelist file 33 """ 34 whitelist_files = [whitelist_file] 35 private_file = os.path.join(os.path.dirname(whitelist_file), 36 "private_" + 37 os.path.basename(whitelist_file)) 38 whitelist_files.append(private_file) 39 self.process_whitelists(whitelist_files) 40 41 42 def process_whitelist(self, whitelist_file): 43 """ 44 Filter out hits found on non-comment lines in the whitelist. 45 46 @param whitelist_file: path to whitelist file 47 """ 48 if os.path.isfile(whitelist_file): 49 f = open(whitelist_file) 50 whitelist = [x for x in f.read().splitlines() 51 if not x.startswith('#')] 52 f.close() 53 self.whitelist_set = self.whitelist_set.union(set(whitelist)) 54 55 filtered_list = [] 56 for bad_file in self.bad_set: 57 # Does |bad_file| match any entry in the whitelist? 58 in_whitelist = any([fnmatch.fnmatch(bad_file, whitelist_entry) 59 for whitelist_entry in self.whitelist_set]) 60 if not in_whitelist: 61 filtered_list.append(bad_file) 62 63 self.filtered_set = set(filtered_list) 64 # TODO(jorgelo): remove glob patterns from |new_passes|. 65 self.new_passes = self.whitelist_set.difference(self.bad_set) 66 67 68 def process_whitelists(self, whitelist_files): 69 """ 70 Filter out hits found in a list of whitelist files. 71 72 @param whitelist_files: list of paths to whitelist files 73 """ 74 for whitelist_file in whitelist_files: 75 self.process_whitelist(whitelist_file) 76 77 78 def get_fail_summary_message(self): 79 m = "Test %s " % self.description 80 m += "%d failures" % len(self.filtered_set) 81 return m 82 83 84 def get_fail_message(self): 85 m = self.get_fail_summary_message() 86 sorted_list = list(self.filtered_set) 87 sorted_list.sort() 88 m += "\nFAILED:\n%s\n\n" % "\n".join(sorted_list) 89 return m 90 91 92 def __str__(self): 93 m = "Test %s " % self.description 94 m += ("%d failures, %d in whitelist, %d in filtered, %d new passes " % 95 (len(self.bad_set), 96 len(self.whitelist_set), 97 len(self.filtered_set), 98 len(self.new_passes))) 99 100 if len(self.filtered_set): 101 sorted_list = list(self.filtered_set) 102 sorted_list.sort() 103 m += "FAILED:\n%s" % "\n".join(sorted_list) 104 else: 105 m += "PASSED!" 106 107 if len(self.new_passes): 108 sorted_list = list(self.new_passes) 109 sorted_list.sort() 110 m += ("\nNew passes (remove these from the whitelist):\n%s" % 111 "\n".join(sorted_list)) 112 logging.debug(m) 113 return m 114 115 116 class platform_ToolchainOptions(test.test): 117 """ 118 Tests for various expected conditions on ELF binaries in the image. 119 """ 120 version = 2 121 122 def get_cmd(self, test_cmd, find_options=""): 123 base_cmd = ("find '%s' -wholename %s -prune -o " 124 " -wholename /proc -prune -o " 125 " -wholename /dev -prune -o " 126 " -wholename /sys -prune -o " 127 " -wholename /mnt/stateful_partition -prune -o " 128 " -wholename /usr/local -prune -o " 129 # There are files in /home/chronos that cause false 130 # positives, and since that's noexec anyways, it should 131 # be skipped. 132 " -wholename '/home/chronos' -prune -o " 133 " -wholename " 134 "/opt/google/containers/android/rootfs/root/vendor" 135 " -prune -o " 136 " -wholename " 137 "/run/containers/android_*/root/vendor" 138 " -prune -o " 139 " %s " 140 " -not -name 'libstdc++.so.*' " 141 " -not -name 'libgcc_s.so.*' " 142 " -type f -executable -exec " 143 "sh -c '%s " 144 "{} | grep -q ELF && " 145 "(%s || echo {})' ';'") 146 rootdir = "/" 147 cmd = base_cmd % (rootdir, self.autodir, find_options, FILE_CMD, 148 test_cmd) 149 return cmd 150 151 152 def create_and_filter(self, description, cmd, whitelist_file, 153 find_options=""): 154 """ 155 Runs a command, with "{}" replaced (via "find -exec") with the 156 target ELF binary. If the command fails, the file is marked as 157 failing the test. Results are filtered against the provided 158 whitelist file. 159 160 @param description: text name of the check being done 161 @param cmd: command to run via find's -exec option 162 @param whitelist_file: list of failures to ignore 163 @param find_options: additional options for find to limit the scope 164 """ 165 full_cmd = self.get_cmd(cmd, find_options) 166 bad_files = utils.system_output(full_cmd) 167 cso = ToolchainOptionSet(description, bad_files, whitelist_file) 168 cso.process_whitelist_with_private(whitelist_file) 169 return cso 170 171 172 def run_once(self, rootdir="/", args=[]): 173 """ 174 Do a find for all the ELF files on the system. 175 For each one, test for compiler options that should have been used 176 when compiling the file. 177 178 For missing compiler options, print the files. 179 """ 180 181 parser = OptionParser() 182 parser.add_option('--hardfp', 183 dest='enable_hardfp', 184 default=False, 185 action='store_true', 186 help='Whether to check for hardfp binaries.') 187 (options, args) = parser.parse_args(args) 188 189 option_sets = [] 190 191 libc_glob = "/lib/libc-[0-9]*" 192 193 readelf_cmd = glob.glob("/usr/local/*/binutils-bin/*/readelf")[0] 194 195 # We do not test binaries if they are built with Address Sanitizer 196 # because it is a separate testing tool. 197 no_asan_used = utils.system_output("%s -s " 198 "/opt/google/chrome/chrome | " 199 "egrep -q \"__asan_init\" || " 200 "echo no ASAN" % readelf_cmd) 201 if not no_asan_used: 202 logging.debug("ASAN detected on /opt/google/chrome/chrome. " 203 "Will skip all checks.") 204 return 205 206 # Check that gold was used to build binaries. 207 # TODO(jorgelo): re-enable this check once crbug.com/417912 is fixed. 208 # gold_cmd = ("%s -S {} 2>&1 | " 209 # "egrep -q \".note.gnu.gold-ve\"" % readelf_cmd) 210 # gold_find_options = "" 211 # if utils.get_cpu_arch() == "arm": 212 # # gold is only enabled for Chrome on ARM. 213 # gold_find_options = "-path \"/opt/google/chrome/chrome\"" 214 # gold_whitelist = os.path.join(self.bindir, "gold_whitelist") 215 # option_sets.append(self.create_and_filter("gold", 216 # gold_cmd, 217 # gold_whitelist, 218 # gold_find_options)) 219 220 # Verify non-static binaries have BIND_NOW in dynamic section. 221 now_cmd = ("(%s {} | grep -q statically) ||" 222 "%s -d {} 2>&1 | " 223 "egrep -q \"BIND_NOW\"" % (FILE_CMD, readelf_cmd)) 224 now_whitelist = os.path.join(self.bindir, "now_whitelist") 225 option_sets.append(self.create_and_filter("-Wl,-z,now", 226 now_cmd, 227 now_whitelist)) 228 229 # Verify non-static binaries have RELRO program header. 230 relro_cmd = ("(%s {} | grep -q statically) ||" 231 "%s -l {} 2>&1 | " 232 "egrep -q \"GNU_RELRO\"" % (FILE_CMD, readelf_cmd)) 233 relro_whitelist = os.path.join(self.bindir, "relro_whitelist") 234 option_sets.append(self.create_and_filter("-Wl,-z,relro", 235 relro_cmd, 236 relro_whitelist)) 237 238 # Verify non-static binaries are dynamic (built PIE). 239 pie_cmd = ("(%s {} | grep -q statically) ||" 240 "%s -l {} 2>&1 | " 241 "egrep -q \"Elf file type is DYN\"" % (FILE_CMD, 242 readelf_cmd)) 243 pie_whitelist = os.path.join(self.bindir, "pie_whitelist") 244 option_sets.append(self.create_and_filter("-fPIE", 245 pie_cmd, 246 pie_whitelist)) 247 248 # Verify ELFs don't include TEXTRELs. 249 # FIXME: Remove the i?86 filter after the bug is fixed. 250 # crbug.com/686926 251 if (utils.get_current_kernel_arch() not in 252 ('i%d86' % i for i in xrange(3,7))): 253 textrel_cmd = ("(%s {} | grep -q statically) ||" 254 "%s -d {} 2>&1 | " 255 "(egrep -q \"0x0+16..TEXTREL\"; [ $? -ne 0 ])" 256 % (FILE_CMD, readelf_cmd)) 257 textrel_whitelist = os.path.join(self.bindir, "textrel_whitelist") 258 option_sets.append(self.create_and_filter("TEXTREL", 259 textrel_cmd, 260 textrel_whitelist)) 261 262 # Verify all binaries have non-exec STACK program header. 263 stack_cmd = ("%s -lW {} 2>&1 | " 264 "egrep -q \"GNU_STACK.*RW \"" % readelf_cmd) 265 stack_whitelist = os.path.join(self.bindir, "stack_whitelist") 266 option_sets.append(self.create_and_filter("Executable Stack", 267 stack_cmd, 268 stack_whitelist)) 269 270 # Verify all binaries have W^X LOAD program headers. 271 loadwx_cmd = ("%s -lW {} 2>&1 | " 272 "grep \"LOAD\" | egrep -v \"(RW |R E)\" | " 273 "wc -l | grep -q \"^0$\"" % readelf_cmd) 274 loadwx_whitelist = os.path.join(self.bindir, "loadwx_whitelist") 275 option_sets.append(self.create_and_filter("LOAD Writable and Exec", 276 loadwx_cmd, 277 loadwx_whitelist)) 278 279 # Verify ARM binaries are all using VFP registers. 280 if (options.enable_hardfp and utils.get_cpu_arch() == 'arm'): 281 hardfp_cmd = ("%s -A {} 2>&1 | " 282 "egrep -q \"Tag_ABI_VFP_args: VFP registers\"" % 283 readelf_cmd) 284 hardfp_whitelist = os.path.join(self.bindir, "hardfp_whitelist") 285 option_sets.append(self.create_and_filter("hardfp", hardfp_cmd, 286 hardfp_whitelist)) 287 288 fail_msg = "" 289 290 # There is currently no way to clear binary prebuilts for all devs. 291 # Thus, when a new check is added to this test, the test might fail 292 # for users who have old prebuilts which have not been compiled 293 # in the correct manner. 294 fail_summaries = [] 295 full_msg = "Test results:" 296 num_fails = 0 297 for cos in option_sets: 298 if len(cos.filtered_set): 299 num_fails += 1 300 fail_msg += cos.get_fail_message() + "\n" 301 fail_summaries.append(cos.get_fail_summary_message()) 302 full_msg += str(cos) + "\n\n" 303 fail_summary_msg = ", ".join(fail_summaries) 304 305 logging.error(fail_msg) 306 logging.debug(full_msg) 307 if num_fails: 308 raise error.TestFail(fail_summary_msg) 309
