1 import sys 2 import os 3 import os.path 4 from glob import glob 5 import optparse 6 7 VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' 8 9 def compareOutputs( expected, actual, message ): 10 expected = expected.strip().replace('\r','').split('\n') 11 actual = actual.strip().replace('\r','').split('\n') 12 diff_line = 0 13 max_line_to_compare = min( len(expected), len(actual) ) 14 for index in xrange(0,max_line_to_compare): 15 if expected[index].strip() != actual[index].strip(): 16 diff_line = index + 1 17 break 18 if diff_line == 0 and len(expected) != len(actual): 19 diff_line = max_line_to_compare+1 20 if diff_line == 0: 21 return None 22 def safeGetLine( lines, index ): 23 index += -1 24 if index >= len(lines): 25 return '' 26 return lines[index].strip() 27 return """ Difference in %s at line %d: 28 Expected: '%s' 29 Actual: '%s' 30 """ % (message, diff_line, 31 safeGetLine(expected,diff_line), 32 safeGetLine(actual,diff_line) ) 33 34 def safeReadFile( path ): 35 try: 36 return file( path, 'rt' ).read() 37 except IOError, e: 38 return '<File "%s" is missing: %s>' % (path,e) 39 40 def runAllTests( jsontest_executable_path, input_dir = None, 41 use_valgrind=False, with_json_checker=False ): 42 if not input_dir: 43 input_dir = os.path.join( os.getcwd(), 'data' ) 44 tests = glob( os.path.join( input_dir, '*.json' ) ) 45 if with_json_checker: 46 test_jsonchecker = glob( os.path.join( input_dir, '../jsonchecker', '*.json' ) ) 47 else: 48 test_jsonchecker = [] 49 failed_tests = [] 50 valgrind_path = use_valgrind and VALGRIND_CMD or '' 51 for input_path in tests + test_jsonchecker: 52 expect_failure = os.path.basename( input_path ).startswith( 'fail' ) 53 is_json_checker_test = (input_path in test_jsonchecker) or expect_failure 54 print 'TESTING:', input_path, 55 options = is_json_checker_test and '--json-checker' or '' 56 pipe = os.popen( "%s%s %s %s" % ( 57 valgrind_path, jsontest_executable_path, options, 58 input_path) ) 59 process_output = pipe.read() 60 status = pipe.close() 61 if is_json_checker_test: 62 if expect_failure: 63 if status is None: 64 print 'FAILED' 65 failed_tests.append( (input_path, 'Parsing should have failed:\n%s' % 66 safeReadFile(input_path)) ) 67 else: 68 print 'OK' 69 else: 70 if status is not None: 71 print 'FAILED' 72 failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) 73 else: 74 print 'OK' 75 else: 76 base_path = os.path.splitext(input_path)[0] 77 actual_output = safeReadFile( base_path + '.actual' ) 78 actual_rewrite_output = safeReadFile( base_path + '.actual-rewrite' ) 79 file(base_path + '.process-output','wt').write( process_output ) 80 if status: 81 print 'parsing failed' 82 failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) 83 else: 84 expected_output_path = os.path.splitext(input_path)[0] + '.expected' 85 expected_output = file( expected_output_path, 'rt' ).read() 86 detail = ( compareOutputs( expected_output, actual_output, 'input' ) 87 or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) ) 88 if detail: 89 print 'FAILED' 90 failed_tests.append( (input_path, detail) ) 91 else: 92 print 'OK' 93 94 if failed_tests: 95 print 96 print 'Failure details:' 97 for failed_test in failed_tests: 98 print '* Test', failed_test[0] 99 print failed_test[1] 100 print 101 print 'Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), 102 len(failed_tests) ) 103 return 1 104 else: 105 print 'All %d tests passed.' % len(tests) 106 return 0 107 108 def main(): 109 from optparse import OptionParser 110 parser = OptionParser( usage="%prog [options] <path to jsontestrunner.exe> [test case directory]" ) 111 parser.add_option("--valgrind", 112 action="store_true", dest="valgrind", default=False, 113 help="run all the tests using valgrind to detect memory leaks") 114 parser.add_option("-c", "--with-json-checker", 115 action="store_true", dest="with_json_checker", default=False, 116 help="run all the tests from the official JSONChecker test suite of json.org") 117 parser.enable_interspersed_args() 118 options, args = parser.parse_args() 119 120 if len(args) < 1 or len(args) > 2: 121 parser.error( 'Must provides at least path to jsontestrunner executable.' ) 122 sys.exit( 1 ) 123 124 jsontest_executable_path = os.path.normpath( os.path.abspath( args[0] ) ) 125 if len(args) > 1: 126 input_path = os.path.normpath( os.path.abspath( args[1] ) ) 127 else: 128 input_path = None 129 status = runAllTests( jsontest_executable_path, input_path, 130 use_valgrind=options.valgrind, with_json_checker=options.with_json_checker ) 131 sys.exit( status ) 132 133 if __name__ == '__main__': 134 main() 135
