Home | History | Annotate | Download | only in test
      1 from __future__ import print_function
      2 import sys
      3 import os
      4 import os.path
      5 from glob import glob
      6 import optparse
      7 
      8 VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes '
      9 
     10 def compareOutputs( expected, actual, message ):
     11     expected = expected.strip().replace('\r','').split('\n')
     12     actual = actual.strip().replace('\r','').split('\n')
     13     diff_line = 0
     14     max_line_to_compare = min( len(expected), len(actual) )
     15     for index in range(0,max_line_to_compare):
     16         if expected[index].strip() != actual[index].strip():
     17             diff_line = index + 1
     18             break
     19     if diff_line == 0 and len(expected) != len(actual):
     20         diff_line = max_line_to_compare+1
     21     if diff_line == 0:
     22         return None
     23     def safeGetLine( lines, index ):
     24         index += -1
     25         if index >= len(lines):
     26             return ''
     27         return lines[index].strip()
     28     return """  Difference in %s at line %d:
     29   Expected: '%s'
     30   Actual:   '%s'
     31 """ % (message, diff_line,
     32        safeGetLine(expected,diff_line),
     33        safeGetLine(actual,diff_line) )
     34         
     35 def safeReadFile( path ):
     36     try:
     37         return file( path, 'rt' ).read()
     38     except IOError as e:
     39         return '<File "%s" is missing: %s>' % (path,e)
     40 
     41 def runAllTests( jsontest_executable_path, input_dir = None,
     42                  use_valgrind=False, with_json_checker=False ):
     43     if not input_dir:
     44         input_dir = os.path.join( os.getcwd(), 'data' )
     45     tests = glob( os.path.join( input_dir, '*.json' ) )
     46     if with_json_checker:
     47         test_jsonchecker = glob( os.path.join( input_dir, '../jsonchecker', '*.json' ) )
     48     else:
     49         test_jsonchecker = []
     50     failed_tests = []
     51     valgrind_path = use_valgrind and VALGRIND_CMD or ''
     52     for input_path in tests + test_jsonchecker:
     53         expect_failure = os.path.basename( input_path ).startswith( 'fail' )
     54         is_json_checker_test = (input_path in test_jsonchecker) or expect_failure
     55         print('TESTING:', input_path, end=' ')
     56         options = is_json_checker_test and '--json-checker' or ''
     57         pipe = os.popen( "%s%s %s %s" % (
     58             valgrind_path, jsontest_executable_path, options,
     59             input_path) )
     60         process_output = pipe.read()
     61         status = pipe.close()
     62         if is_json_checker_test:
     63             if expect_failure:
     64                 if status is None:
     65                     print('FAILED')
     66                     failed_tests.append( (input_path, 'Parsing should have failed:\n%s' %
     67                                           safeReadFile(input_path)) )
     68                 else:
     69                     print('OK')
     70             else:
     71                 if status is not None:
     72                     print('FAILED')
     73                     failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) )
     74                 else:
     75                     print('OK')
     76         else:
     77             base_path = os.path.splitext(input_path)[0]
     78             actual_output = safeReadFile( base_path + '.actual' )
     79             actual_rewrite_output = safeReadFile( base_path + '.actual-rewrite' )
     80             file(base_path + '.process-output','wt').write( process_output )
     81             if status:
     82                 print('parsing failed')
     83                 failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) )
     84             else:
     85                 expected_output_path = os.path.splitext(input_path)[0] + '.expected'
     86                 expected_output = file( expected_output_path, 'rt' ).read()
     87                 detail = ( compareOutputs( expected_output, actual_output, 'input' )
     88                             or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) )
     89                 if detail:
     90                     print('FAILED')
     91                     failed_tests.append( (input_path, detail) )
     92                 else:
     93                     print('OK')
     94 
     95     if failed_tests:
     96         print()
     97         print('Failure details:')
     98         for failed_test in failed_tests:
     99             print('* Test', failed_test[0])
    100             print(failed_test[1])
    101             print()
    102         print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests),
    103                                                        len(failed_tests) ))
    104         return 1
    105     else:
    106         print('All %d tests passed.' % len(tests))
    107         return 0
    108 
    109 def main():
    110     from optparse import OptionParser
    111     parser = OptionParser( usage="%prog [options] <path to jsontestrunner.exe> [test case directory]" )
    112     parser.add_option("--valgrind",
    113                   action="store_true", dest="valgrind", default=False,
    114                   help="run all the tests using valgrind to detect memory leaks")
    115     parser.add_option("-c", "--with-json-checker",
    116                   action="store_true", dest="with_json_checker", default=False,
    117                   help="run all the tests from the official JSONChecker test suite of json.org")
    118     parser.enable_interspersed_args()
    119     options, args = parser.parse_args()
    120 
    121     if len(args) < 1 or len(args) > 2:
    122         parser.error( 'Must provides at least path to jsontestrunner executable.' )
    123         sys.exit( 1 )
    124 
    125     jsontest_executable_path = os.path.normpath( os.path.abspath( args[0] ) )
    126     if len(args) > 1:
    127         input_path = os.path.normpath( os.path.abspath( args[1] ) )
    128     else:
    129         input_path = None
    130     status = runAllTests( jsontest_executable_path, input_path,
    131                           use_valgrind=options.valgrind, with_json_checker=options.with_json_checker )
    132     sys.exit( status )
    133 
    134 if __name__ == '__main__':
    135     main()
    136