1 #!/usr/bin/env python3 2 """ 3 An LTP [execution and] parsing wrapper. 4 5 Used as a second layer for ease-of-use with users as many developers 6 complain about complexity involved with trying to use LTP in my 7 organization -_-. 8 9 Copyright (C) 2009-2012, Ngie Cooper 10 11 This program is free software; you can redistribute it and/or modify 12 it under the terms of the GNU General Public License as published by 13 the Free Software Foundation; either version 2 of the License, or 14 (at your option) any later version. 15 16 This program is distributed in the hope that it will be useful, 17 but WITHOUT ANY WARRANTY; without even the implied warranty of 18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 GNU General Public License for more details. 20 21 You should have received a copy of the GNU General Public License along 22 with this program; if not, write to the Free Software Foundation, Inc., 23 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 24 """ 25 26 27 from optparse import OptionGroup, OptionParser 28 import os 29 import re 30 import sys 31 32 33 class ResultsParseException(Exception): 34 """ Extended class for parsing LTP results. """ 35 36 37 def parse_ltp_results(exec_log, output_log, verbose=0): 38 """Function for parsing LTP results. 39 40 1. The exec log is the log with the results in summary form. 41 42 And now a note from our sponsors about exec logs... 43 44 startup='Thu Oct 1 06:42:07 2009' 45 tag=abort01 stime=1254379327 dur=2 exit=exited stat=0 core=no cu=0 cs=16 46 tag=accept01 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=1 cs=0 47 tag=access01 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=0 cs=0 48 tag=access02 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=0 cs=0 49 tag=access03 stime=1254379329 dur=1 exit=exited stat=0 core=no cu=0 cs=1 50 51 [...] 52 53 a. tag is the test tag name. 54 b. stime is the system time at the start of the exec. 55 c. dur is the total duration of the test. 56 d. exit tells you what the result was. Valid values are: 57 - exited 58 - signaled 59 - stopped 60 - unknown 61 See run_child in pan.c. 62 e. stat is the exit status. 63 f. core answers the question: `did I dump core?'. 64 g. cu is the cutime (cumulative user time). 65 h. cs is the cstime (cumulative system time). 66 67 2. The output log is the log with all of the terse results. 68 3. verbose tells us whether or not we need to include the passed results. 69 """ 70 71 if not os.access(exec_log, os.R_OK): 72 raise ResultsParseException("Exec log - %s - specified doesn't exist" 73 % exec_log) 74 elif 1 < verbose and not os.access(output_log, os.R_OK): 75 # Need the output log for context to the end user. 76 raise ResultsParseException("Output log - %s - specified doesn't exist" 77 % output_log) 78 79 context = None 80 81 failed = [] 82 passed = 0 83 84 if 2 <= verbose: 85 passed = [] 86 87 target_vals = ('exited', '0', 'no') 88 89 fd = open(exec_log, 'r') 90 91 try: 92 content = fd.read() 93 matches = re.finditer('tag=(?P<tag>\w+).+exit=(?P<exit>\w+) ' 94 'stat=(?P<stat>\d+) core=(?P<core>\w+)', content) 95 finally: 96 content = None 97 fd.close() 98 99 if not matches: 100 raise ResultsParseException("No parseable results were found in the " 101 "exec log - `%s'." % exec_log) 102 103 for match in matches: 104 105 if ((match.group('exit'), match.group('stat'), match.group('core')) != 106 target_vals): 107 failed.append(match.group('tag')) 108 elif 2 <= verbose: 109 passed.append(match.group('tag')) 110 else: 111 passed += 1 112 113 # Save memory on large files because lists can eat up a fair amount of 114 # memory. 115 matches = None 116 117 if 1 <= verbose: 118 119 context = {} 120 121 search_tags = failed[:] 122 123 if 2 <= verbose: 124 search_tags += passed 125 126 search_tags.sort() 127 128 fd = open(output_log, 'r') 129 130 try: 131 132 line_iterator = getattr(fd, 'xreadlines', getattr(fd, 'readlines')) 133 134 end_output = '<<<execution_status>>>' 135 output_start = '<<<test_output>>>' 136 137 tag_re = re.compile('tag=(\w+)') 138 139 grab_output = False 140 141 local_context = '' 142 143 search_tag = None 144 145 try: 146 147 while True: 148 149 line = next(line_iterator) 150 151 if line.startswith(end_output): 152 153 if search_tag: 154 context[search_tag] = local_context 155 156 grab_output = False 157 local_context = '' 158 search_tag = None 159 160 if not search_tag: 161 162 while True: 163 164 line = next(line_iterator) 165 166 match = tag_re.match(line) 167 168 if match and match.group(1) in search_tags: 169 search_tag = match.group(1) 170 break 171 172 elif line.startswith(output_start): 173 grab_output = True 174 elif grab_output: 175 local_context += line 176 177 except StopIteration: 178 pass 179 180 for k in list(context.keys()): 181 if k not in search_tags: 182 raise ResultsParseException('Leftover token in search ' 183 'keys: %s' % k) 184 185 except Exception as exc: 186 # XXX (garrcoop): change from Exception to soft error and print 187 # out warning with logging module. 188 raise ResultsParseException('Encountered exception reading output ' 189 'for context: %s' % str(exc)) 190 finally: 191 fd.close() 192 193 return failed, passed, context 194 195 196 def determine_context(output_log, testsuite, test_set, context): 197 """Return a set of context values mapping test_set -> context.""" 198 199 test_set_context = {} 200 201 for test in test_set: 202 203 if test in context: 204 test_context = context[test] 205 del context[test] 206 else: 207 test_context = ('Could not determine context for %s; please see ' 208 'output log - %s' % (test, output_log)) 209 210 test_set_context['%s : %s' % (testsuite, test)] = test_context 211 212 return test_set_context 213 214 215 def print_context(output_dest, header, testsuite_context): 216 """Print out testsuite_context to output_dest, heading it up with 217 header. 218 """ 219 220 output_dest.write('\n'.join(['', '=' * 40, header, '-' * 40, ''])) 221 222 for test, context in list(testsuite_context.items()): 223 output_dest.write('<output test="%s">\n%s\n</output>\n' % 224 (test, context.strip())) 225 226 227 def main(): 228 """main""" 229 230 parser = OptionParser(prog=os.path.basename(sys.argv[0]), 231 usage='usage: %prog [options] test ...', 232 version='0.0.2') 233 234 ltpdir = os.getenv('LTPROOT', '@prefix@') 235 236 parser.add_option('-l', '--ltp-dir', dest='ltp_dir', 237 default=ltpdir, help='LTP directory [default: %default]') 238 parser.add_option('-L', '--log-dir', dest='log_dir', 239 default=None, 240 help=('directory for [storing and] retrieving logs ' 241 '[default: %s/output]' % ltpdir), 242 metavar='DIR') 243 parser.add_option('-p', '--postprocess-only', dest='postprocess_only', 244 default=False, action='store_true', 245 help=("Don't execute runltp; just postprocess logs " 246 "[default: %default].")) 247 parser.add_option('-o', '--output-file', dest='output_file', 248 default=None, 249 help='File to output results') 250 parser.add_option('-r', '--runltp-opts', dest='runltp_opts', 251 default='', 252 help=('options to pass directly to runltp (will ' 253 'suppress -q).')) 254 255 group = OptionGroup(parser, 'Logging', 256 'If --summary-mode is 0, then the summary output is ' 257 'suppressed. ' 258 'If --summary-mode is 1 [the default], then summary ' 259 'output will be displayed for test execution' 260 'If --summary-mode is 2, then summary output will be ' 261 'provided on a per-test suite basis. If only ' 262 'one test suite is specified, this has the same net ' 263 "effect as `--summary-mode 1'" 264 'If --verbose is specified once, prints out failed ' 265 'test information with additional context. ' 266 'If --verbose is specified twice, prints out the ' 267 'failed and passed test context, as well as the ' 268 'summary.') 269 270 parser.add_option('-s', '--summary-mode', dest='summary_mode', default=1, 271 type='int', 272 help='See Logging.') 273 274 parser.add_option('-v', '--verbose', dest='verbose', default=0, 275 action='count', 276 help=('Increases context verbosity from tests. See ' 277 'Verbosity for more details.')) 278 parser.add_option_group(group) 279 280 group = OptionGroup(parser, 'Copyright', 281 '%(prog)s version %(version)s, Copyright (C) ' 282 '2009-2012, Ngie Cooper %(prog)s comes with ' 283 'ABSOLUTELY NO WARRANTY; ' 284 'This is free software, and you are welcome to ' 285 'redistribute it under certain conditions (See the ' 286 'license tort in %(file)s for more details).' 287 % { 'file' : os.path.abspath(__file__), 288 'prog' : parser.prog, 289 'version' : parser.version }) 290 291 parser.add_option_group(group) 292 293 opts, args = parser.parse_args() 294 295 # Remove -q from the opts string, as long as it's a standalone option. 296 runltp_opts = re.sub('^((?<!\S)+\-q\s+|\-q|\s+\-q(?!\S))$', '', 297 opts.runltp_opts) 298 299 if not opts.log_dir: 300 opts.log_dir = os.path.join(opts.ltp_dir, 'output') 301 302 if not opts.summary_mode and not opts.verbose: 303 parser.error('You cannot suppress summary output and disable ' 304 'verbosity.') 305 elif opts.summary_mode not in list(range(3)): 306 parser.error('--summary-mode must be a value between 0 and 2.') 307 308 if len(args) == 0: 309 # Default to scenarios also used by runltp. 310 fd = open(os.path.join(ltpdir, 'scenario_groups/default'), 'r') 311 try: 312 args = [l.strip() for l in fd.readlines()] 313 finally: 314 fd.close() 315 316 if opts.output_file: 317 318 output_dir = os.path.dirname(opts.output_file) 319 320 if output_dir: 321 # Not cwd; let's check to make sure that the directory does or 322 # does not exist. 323 324 if not os.path.exists(output_dir): 325 # We need to make the directory. 326 os.makedirs(os.path.dirname(opts.output_file)) 327 elif not os.path.isdir(os.path.abspath(output_dir)): 328 # Path exists, but isn't a file. Oops! 329 parser.error('Dirname for path specified - %s - is not valid' 330 % output_dir) 331 332 else: 333 # Current path (cwd) 334 opts.output_file = os.path.join(os.getcwd(), opts.output_file) 335 336 output_dest = open(opts.output_file, 'w') 337 338 else: 339 340 output_dest = sys.stdout 341 342 try: 343 344 failed_context = {} 345 passed_context = {} 346 347 failed_count = 0 348 passed_count = 0 349 350 if opts.summary_mode == 2 and len(args) == 1: 351 opts.summary_mode = 1 352 353 for testsuite in args: 354 355 # Iterate over the provided test list 356 357 context = {} 358 exec_log = os.path.join(opts.log_dir, '%s-exec.log' % testsuite) 359 output_log = os.path.join(opts.log_dir, ('%s-output.log' 360 % testsuite)) 361 362 failed_subset = {} 363 364 runtest_file = os.path.join(opts.ltp_dir, 'runtest', testsuite) 365 366 if not opts.postprocess_only: 367 368 for log in [exec_log, output_log]: 369 if os.path.isfile(log): 370 os.remove(log) 371 372 if not os.access(runtest_file, os.R_OK): 373 output_dest.write("%s doesn't exist; skipping " 374 "test\n" % runtest_file) 375 continue 376 377 os.system(' '.join([os.path.join(opts.ltp_dir, 'runltp'), 378 runltp_opts, '-f', testsuite, 379 '-l', exec_log, '-o', output_log])) 380 381 try: 382 383 failed_subset, passed_css, context = \ 384 parse_ltp_results(exec_log, output_log, 385 verbose=opts.verbose) 386 387 except ResultsParseException as rpe: 388 output_dest.write('Error encountered when parsing results for ' 389 'test - %s: %s\n' % (testsuite, str(rpe))) 390 continue 391 392 failed_count += len(failed_subset) 393 394 failed_subset_context = {} 395 passed_subset_context = {} 396 397 if opts.verbose: 398 failed_subset_context = determine_context(output_log, 399 testsuite, 400 failed_subset, 401 context) 402 if type(passed_css) == list: 403 404 passed_count += len(passed_css) 405 406 if opts.verbose == 2: 407 passed_subset_context = determine_context(output_log, 408 testsuite, 409 passed_css, 410 context) 411 412 else: 413 414 passed_count += passed_css 415 416 if opts.summary_mode == 1: 417 418 failed_context.update(failed_subset_context) 419 passed_context.update(passed_subset_context) 420 421 else: 422 423 if 1 <= opts.verbose: 424 # Print out failed testcases. 425 print_context(output_dest, 426 'FAILED TESTCASES for %s' % testsuite, 427 failed_subset_context) 428 429 if opts.verbose == 2: 430 # Print out passed testcases with context. 431 print_context(output_dest, 432 'PASSED TESTCASES for %s' % testsuite, 433 passed_subset_context) 434 435 if opts.summary_mode == 2: 436 output_dest.write(""" 437 ======================================== 438 SUMMARY for: %s 439 ---------------------------------------- 440 PASS - %d 441 FAIL - %d 442 ---------------------------------------- 443 """ % (testsuite, passed_count, len(failed_subset))) 444 445 if opts.summary_mode == 1: 446 447 # Print out overall results. 448 449 if 1 <= opts.verbose: 450 # Print out failed testcases with context. 451 print_context(output_dest, "FAILED TESTCASES", failed_context) 452 453 if opts.verbose == 2: 454 # Print out passed testcases with context. 455 print_context(output_dest, "PASSED TESTCASES", passed_context) 456 457 output_dest.write(""" 458 ======================================== 459 SUMMARY for tests: 460 %s 461 ---------------------------------------- 462 PASS - %d 463 FAIL - %d 464 ---------------------------------------- 465 """ % (' '.join(args), passed_count, failed_count)) 466 467 finally: 468 469 if output_dest != sys.stdout: 470 471 output_dest.close() 472 473 if __name__ == '__main__': 474 main() 475
