1 #!/usr/bin/python 2 # 3 # Copyright 2017 - The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 """Generates a report on CKI syscall coverage in VTS LTP. 19 20 This module generates a report on the syscalls in the Android CKI and 21 their coverage in VTS LTP. 22 23 The coverage report provides, for each syscall in the CKI, the number of 24 enabled and disabled LTP tests for the syscall in VTS. If VTS test output is 25 supplied, the report instead provides the number of disabled, skipped, failing, 26 and passing tests for each syscall. 27 28 Assumptions are made about the structure of files in LTP source 29 and the naming convention. 30 """ 31 32 import argparse 33 import os.path 34 import re 35 import sys 36 import xml.etree.ElementTree as ET 37 38 sys.path.append(os.path.join(os.environ["ANDROID_BUILD_TOP"], 39 "bionic/libc/tools")) 40 import gensyscalls 41 42 sys.path.append(os.path.join(os.environ["ANDROID_BUILD_TOP"], 43 "test/vts-testcase/kernel/ltp/configs")) 44 import disabled_tests as vts_disabled 45 import stable_tests as vts_stable 46 47 bionic_libc_root = os.path.join(os.environ["ANDROID_BUILD_TOP"], "bionic/libc") 48 49 50 class CKI_Coverage(object): 51 """Determines current test coverage of CKI system calls in LTP. 52 53 Many of the system calls in the CKI are tested by LTP. For a given 54 system call an LTP test may or may not exist, that LTP test may or may 55 not be currently compiling properly for Android, the test may not be 56 stable, the test may not be running due to environment issues or 57 passing. This class looks at various sources of information to determine 58 the current test coverage of system calls in the CKI from LTP. 59 60 Note that due to some deviations in LTP of tests from the common naming 61 convention there there may be tests that are flagged here as not having 62 coverage when in fact they do. 63 """ 64 65 LTP_SYSCALL_ROOT = os.path.join(os.environ["ANDROID_BUILD_TOP"], 66 "external/ltp/testcases/kernel/syscalls") 67 DISABLED_IN_LTP_PATH = os.path.join(os.environ["ANDROID_BUILD_TOP"], 68 "external/ltp/android/tools/disabled_tests.txt") 69 70 ltp_full_set = [] 71 72 cki_syscalls = [] 73 74 disabled_in_ltp = [] 75 disabled_in_vts_ltp = vts_disabled.DISABLED_TESTS 76 stable_in_vts_ltp = vts_stable.STABLE_TESTS 77 78 syscall_tests = {} 79 disabled_tests = {} 80 failing_tests = {} 81 skipped_tests = {} 82 passing_tests = {} 83 84 test_results = {} 85 86 def __init__(self, arch): 87 self.arch = arch 88 89 def load_ltp_tests(self): 90 """Load the list of LTP syscall tests. 91 92 Load the list of all syscall tests existing in LTP. 93 """ 94 for path, dirs, files in os.walk(self.LTP_SYSCALL_ROOT): 95 for filename in files: 96 basename, ext = os.path.splitext(filename) 97 if ext != ".c": continue 98 self.ltp_full_set.append(basename) 99 100 def load_ltp_disabled_tests(self): 101 """Load the list of LTP tests not being compiled. 102 103 The LTP repository in Android contains a list of tests which are not 104 compiled due to incompatibilities with Android. 105 """ 106 with open(self.DISABLED_IN_LTP_PATH) as fp: 107 for line in fp: 108 line = line.strip() 109 if not line: continue 110 test_re = re.compile(r"^(\w+)") 111 test_match = re.match(test_re, line) 112 if not test_match: continue 113 self.disabled_in_ltp.append(test_match.group(1)) 114 115 def parse_test_results(self, results): 116 """Parse xml from VTS output to collect LTP results. 117 118 Parse xml to collect pass/fail results for each LTP 119 test. A failure occurs if a test has a result other than pass or fail. 120 121 Args: 122 results: Path to VTS output XML file. 123 """ 124 tree = ET.parse(results) 125 root = tree.getroot() 126 127 # find LTP module 128 for module in root.findall("Module"): 129 if module.attrib["name"] != "VtsKernelLtp": continue 130 131 # find LTP testcase 132 for testcase in module.findall("TestCase"): 133 if testcase.attrib["name"] != "KernelLtpTest": continue 134 135 # iterate over each LTP test 136 for test in testcase.findall("Test"): 137 test_re = re.compile(r"^syscalls.(\w+)_((32bit)|(64bit))$") 138 test_match = re.match(test_re, test.attrib["name"]) 139 if not test_match: continue 140 141 test_name = test_match.group(1) 142 143 if test.attrib["result"] == "pass": 144 self.test_results[test_name] = "pass" 145 elif test.attrib["result"] == "fail": 146 self.test_results[test_name] = "fail" 147 else: 148 print ("Unknown VTS LTP test result for %s is %s" % 149 (test_name, test.attrib["result"])) 150 sys.exit(-1) 151 152 def ltp_test_special_cases(self, syscall, test): 153 """Detect special cases in syscall to LTP mapping. 154 155 Most syscall tests in LTP follow a predictable naming 156 convention, but some do not. Detect known special cases. 157 158 Args: 159 syscall: The name of a syscall. 160 test: The name of a testcase. 161 162 Returns: 163 A boolean indicating whether the given syscall is tested 164 by the given testcase. 165 """ 166 if syscall == "clock_nanosleep" and test == "clock_nanosleep2_01": 167 return True 168 if syscall == "fadvise" and test.startswith("posix_fadvise"): 169 return True 170 if syscall == "futex" and test.startswith("futex_"): 171 return True 172 if syscall == "inotify_add_watch" or syscall == "inotify_rm_watch": 173 test_re = re.compile(r"^inotify\d+$") 174 if re.match(test_re, test): 175 return True 176 if syscall == "newfstatat": 177 test_re = re.compile(r"^fstatat\d+$") 178 if re.match(test_re, test): 179 return True 180 181 return False 182 183 def match_syscalls_to_tests(self, syscalls): 184 """Match syscalls with tests in LTP. 185 186 Create a mapping from CKI syscalls and tests in LTP. This mapping can 187 largely be determined using a common naming convention in the LTP file 188 hierarchy but there are special cases that have to be taken care of. 189 190 Args: 191 syscalls: List of syscall structures containing all syscalls 192 in the CKI. 193 """ 194 for syscall in syscalls: 195 if self.arch not in syscall: 196 continue 197 self.cki_syscalls.append(syscall["name"]) 198 self.syscall_tests[syscall["name"]] = [] 199 # LTP does not use the 64 at the end of syscall names for testcases. 200 ltp_syscall_name = syscall["name"] 201 if ltp_syscall_name.endswith("64"): 202 ltp_syscall_name = ltp_syscall_name[0:-2] 203 # Most LTP syscalls have source files for the tests that follow 204 # a naming convention in the regexp below. Exceptions exist though. 205 # For now those are checked for specifically. 206 test_re = re.compile(r"^%s_?0?\d\d?$" % ltp_syscall_name) 207 for test in self.ltp_full_set: 208 if (re.match(test_re, test) or 209 self.ltp_test_special_cases(ltp_syscall_name, test)): 210 # The filenames of the ioctl tests in LTP do not match the name 211 # of the testcase defined in that source, which is what shows 212 # up in VTS. 213 if ltp_syscall_name == "ioctl": 214 test = "ioctl01_02" 215 self.syscall_tests[syscall["name"]].append(test) 216 self.cki_syscalls.sort() 217 218 def update_test_status(self): 219 """Populate test configuration and output for all CKI syscalls. 220 221 Go through VTS test configuration and test results (if provided) to 222 populate data for all CKI syscalls. 223 """ 224 for syscall in self.cki_syscalls: 225 self.disabled_tests[syscall] = [] 226 self.skipped_tests[syscall] = [] 227 self.failing_tests[syscall] = [] 228 self.passing_tests[syscall] = [] 229 if not self.syscall_tests[syscall]: 230 continue 231 for test in self.syscall_tests[syscall]: 232 if (test in self.disabled_in_ltp or 233 "syscalls.%s" % test in self.disabled_in_vts_ltp or 234 ("syscalls.%s_32bit" % test not in self.stable_in_vts_ltp and 235 "syscalls.%s_64bit" % test not in self.stable_in_vts_ltp)): 236 self.disabled_tests[syscall].append(test) 237 continue 238 239 if not self.test_results: 240 continue 241 242 if test not in self.test_results: 243 self.skipped_tests[syscall].append(test) 244 elif self.test_results[test] == "fail": 245 self.failing_tests[syscall].append(test) 246 elif self.test_results[test] == "pass": 247 self.passing_tests[syscall].append(test) 248 else: 249 print ("Warning - could not resolve test %s status for syscall %s" % 250 (test, syscall)) 251 252 def output_results(self): 253 """Pretty print the CKI syscall LTP coverage results. 254 255 Pretty prints a table of the CKI syscall LTP coverage, pointing out 256 syscalls which have no passing tests in VTS LTP. 257 """ 258 if not self.test_results: 259 self.output_limited_results() 260 return 261 count = 0 262 uncovered = 0 263 for syscall in self.cki_syscalls: 264 if not count % 20: 265 print ("%25s Disabled Skipped Failing Passing -------------" % 266 "-------------") 267 sys.stdout.write("%25s %s %s %s %s" % 268 (syscall, len(self.disabled_tests[syscall]), 269 len(self.skipped_tests[syscall]), 270 len(self.failing_tests[syscall]), 271 len(self.passing_tests[syscall]))) 272 if not self.passing_tests[syscall]: 273 print " <-- uncovered" 274 uncovered += 1 275 else: 276 print "" 277 count += 1 278 print "" 279 print ("Total uncovered syscalls: %s out of %s" % 280 (uncovered, len(self.cki_syscalls))) 281 282 def output_limited_results(self): 283 """Pretty print the CKI syscall LTP coverage without VTS test results. 284 285 When no VTS test results are supplied then only the count of enabled 286 and disabled LTP tests may be shown. 287 """ 288 count = 0 289 uncovered = 0 290 for syscall in self.cki_syscalls: 291 if not count % 20: 292 print ("%25s Disabled Enabled -------------" % 293 "-------------") 294 sys.stdout.write("%25s %s %s" % 295 (syscall, len(self.disabled_tests[syscall]), 296 len(self.syscall_tests[syscall]) - 297 len(self.disabled_tests[syscall]))) 298 if (len(self.syscall_tests[syscall]) - 299 len(self.disabled_tests[syscall]) <= 0): 300 print " <-- uncovered" 301 uncovered += 1 302 else: 303 print "" 304 count += 1 305 print "" 306 print ("Total uncovered syscalls: %s out of %s" % 307 (uncovered, len(self.cki_syscalls))) 308 309 if __name__ == "__main__": 310 parser = argparse.ArgumentParser(description="Output list of system calls " 311 "in the Common Kernel Interface and their VTS LTP coverage. If VTS " 312 "test output is supplied, output includes system calls which have " 313 "tests in VTS LTP, but the tests are skipped or are failing.") 314 parser.add_argument("arch", help="architecture of Android platform") 315 parser.add_argument("-l", action="store_true", 316 help="list CKI syscalls only, without coverage") 317 parser.add_argument("-r", "--results", help="path to VTS test_result.xml") 318 args = parser.parse_args() 319 320 cki = gensyscalls.SysCallsTxtParser() 321 cki.parse_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT")) 322 cki.parse_file(os.path.join(bionic_libc_root, "SECCOMP_WHITELIST.TXT")) 323 if args.l: 324 for syscall in cki.syscalls: 325 if args.arch in syscall: 326 print syscall["name"] 327 exit(0) 328 329 cki_cov = CKI_Coverage(args.arch) 330 331 cki_cov.load_ltp_tests() 332 cki_cov.load_ltp_disabled_tests() 333 if args.results: 334 cki_cov.parse_test_results(args.results) 335 cki_cov.match_syscalls_to_tests(cki.syscalls) 336 cki_cov.update_test_status() 337 338 beta_string = ("*** WARNING: This script is still in development and may\n" 339 "*** report both false positives and negatives.") 340 print beta_string 341 cki_cov.output_results() 342 print beta_string 343
