Home | History | Annotate | Download | only in ltp 
      1 #!/usr/bin/python
      2 # (C) Copyright IBM Corp. 2006
      3 # Author: Dustin Kirkland <dustin.kirkland (at] us.ibm.com>
      4 # Description:
      5 #  Input:  Two or more files containing results from different executions of
      6 #          the LTP. The input can either be file names or the url location
      7 #          of the ltp.results file.
      8 #  Output: A report on the following:
      9 #          - The total number of tests executed in each run
     10 #          - The testname, sequence number, and output of each run
     11 #            where the results of those runs differ
     12 #  Return:
     13 #          0 if all runs had identical results
     14 #          Non-zero if results differ, or bad input
     15 
     16 
     17 import sys, string, re
     18 from autotest_lib.client.common_lib import utils
     19 
     20 def usage():
     21     print "\nUsage: \n\
     22 ltp-diff results1 results2 ... locationN \n\
     23 Note: location[1,2,N] may be local files or URLs of LTP results\n"
     24     sys.exit(1)
     25 
     26 def get_results(results_files):
     27     """
     28     Download the results if needed.
     29     Return results of each run in a numerically-indexed dictionary
     30     of dictionaries keyed on testnames.
     31     Return dictionary keyed on unique testnames across all runs.
     32     """
     33     r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:')
     34     i = 0
     35     runs = {}
     36     testnames = {}
     37     for file in results_files:
     38         runs[i] = {}
     39         try:
     40             fh = utils.urlopen(file)
     41             results = fh.readlines()
     42             fh.close()
     43         except:
     44             print "ERROR: reading results resource [%s]" % (file)
     45             usage()
     46         for line in results:
     47             try:
     48                 s = r.match(line)
     49                 testname = s.group(1)
     50                 status = s.group(2)
     51                 runs[i][testname] = status
     52                 testnames[testname] = 1
     53             except:
     54                 pass
     55         i += 1
     56     return (runs, testnames)
     57 
     58 
     59 
     60 def compare_results(runs):
     61     """
     62     Loop through all testnames alpahbetically.
     63     Print any testnames with differing results across runs.
     64     Return 1 if any test results across runs differ.
     65     Return 0 if all test results match.
     66     """
     67     rc = 0
     68     print "LTP Test Results to Compare"
     69     for i in range(len(runs)):
     70         print "  Run[%d]: %d" % (i, len(runs[i].keys()))
     71     print ""
     72     header = 0
     73     all_testnames = testnames.keys()
     74     all_testnames.sort()
     75     for testname in all_testnames:
     76         differ = 0
     77         for i in range(1,len(runs)):
     78             # Must handle testcases that executed in one run
     79             # but not another by setting status to "null"
     80             if not runs[i].has_key(testname):
     81                 runs[i][testname] = "null"
     82             if not runs[i-1].has_key(testname):
     83                 runs[i-1][testname] = "null"
     84             # Check for the results inconsistencies
     85             if runs[i][testname] != runs[i-1][testname]:
     86                 differ = 1
     87         if differ:
     88             if header == 0:
     89                 # Print the differences header only once
     90                 print "Tests with Inconsistent Results across Runs"
     91                 print "  %-35s:\t%s" % ("Testname,Sequence", "Run Results")
     92                 header = 1
     93 
     94             # Print info if results differ
     95             rc = 1
     96             testname_cleaned = re.sub('\s+', ',', testname)
     97             print "  %-35s:\t" % (testname_cleaned),
     98             all_results = ""
     99             for i in range(len(runs)):
    100                 all_results += runs[i][testname]
    101                 if i+1<len(runs):
    102                     all_results += "/"
    103             print all_results
    104     if rc == 0:
    105         print "All LTP results are identical"
    106     return rc
    107 
    108 
    109 ########
    110 # Main #
    111 ########
    112 sys.argv.pop(0)
    113 if (len(sys.argv) < 2):
    114     usage()
    115 (runs, testnames) = get_results(sys.argv)
    116 rc = compare_results(runs)
    117 sys.exit(rc)
    118