Home | History | Annotate | Download | only in kernel_LTP 
      1 #!/usr/bin/python
      2 # (C) Copyright IBM Corp. 2006
      3 # Author: Dustin Kirkland <dustin.kirkland (at] us.ibm.com>
      4 # Description:
      5 #  Input:  Two or more files containing results from different executions of
      6 #          the LTP. The input can either be file names or the url location
      7 #          of the ltp.results file.
      8 #  Output: A report on the following:
      9 #          - The total number of tests executed in each run
     10 #          - The testname, sequence number, and output of each run
     11 #            where the results of those runs differ
     12 #  Return:
     13 #          0 if all runs had identical results
     14 #          Non-zero if results differ, or bad input
     15 
     16 
     17 import sys, re
     18 from autotest_lib.client.common_lib import utils
     19 
     20 def usage():
     21     """Show script help text."""
     22     print "\nUsage: \n\
     23 ltp-diff results1 results2 ... locationN \n\
     24 Note: location[1,2,N] may be local files or URLs of LTP results\n"
     25     sys.exit(1)
     26 
     27 def get_results(results_files):
     28     """
     29     Download the results if needed.
     30     Return results of each run in a numerically-indexed dictionary
     31     of dictionaries keyed on testnames.
     32     Return dictionary keyed on unique testnames across all runs.
     33 
     34     @param results_files: list of result-file names (or url locations).
     35     """
     36     r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
     37     i = 0
     38     runs = {}
     39     testnames = {}
     40     for file in results_files:
     41         runs[i] = {}
     42         try:
     43             fh = utils.urlopen(file)
     44             results = fh.readlines()
     45             fh.close()
     46         except:
     47             print "ERROR: reading results resource [%s]" % (file)
     48             usage()
     49         for line in results:
     50             try:
     51                 s = r.match(line)
     52                 testname = s.group(1)
     53                 status = s.group(2)
     54                 runs[i][testname] = status
     55                 testnames[testname] = 1
     56             except:
     57                 pass
     58         i += 1
     59     return (runs, testnames)
     60 
     61 
     62 
     63 def compare_results(runs):
     64     """
     65     Loop through all testnames alpahbetically.
     66     Print any testnames with differing results across runs.
     67     Return 1 if any test results across runs differ.
     68     Return 0 if all test results match.
     69 
     70     @param runs: dict of test status from get_results().
     71     """
     72     rc = 0
     73     print "LTP Test Results to Compare"
     74     for i in range(len(runs)):
     75         print "  Run[%d]: %d" % (i, len(runs[i].keys()))
     76     print ""
     77     header = 0
     78     all_testnames = testnames.keys()
     79     all_testnames.sort()
     80     for testname in all_testnames:
     81         differ = 0
     82         for i in range(1,len(runs)):
     83             # Must handle testcases that executed in one run
     84             # but not another by setting status to "null"
     85             if not runs[i].has_key(testname):
     86                 runs[i][testname] = "null"
     87             if not runs[i-1].has_key(testname):
     88                 runs[i-1][testname] = "null"
     89             # Check for the results inconsistencies
     90             if runs[i][testname] != runs[i-1][testname]:
     91                 differ = 1
     92         if differ:
     93             if header == 0:
     94                 # Print the differences header only once
     95                 print "Tests with Inconsistent Results across Runs"
     96                 print "  %-35s:\t%s" % ("Testname,Sequence", "Run Results")
     97                 header = 1
     98 
     99             # Print info if results differ
    100             rc = 1
    101             testname_cleaned = re.sub('\s+', ',', testname)
    102             print "  %-35s:\t" % (testname_cleaned),
    103             all_results = ""
    104             for i in range(len(runs)):
    105                 all_results += runs[i][testname]
    106                 if i+1<len(runs):
    107                     all_results += "/"
    108             print all_results
    109     if rc == 0:
    110         print "All LTP results are identical"
    111     return rc
    112 
    113 
    114 ########
    115 # Main #
    116 ########
    117 sys.argv.pop(0)
    118 if (len(sys.argv) < 2):
    119     usage()
    120 (runs, testnames) = get_results(sys.argv)
    121 rc = compare_results(runs)
    122 sys.exit(rc)
    123