1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2012 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ 18 Executes WebView CTS tests and verifies results against known failures. 19 """ 20 21 import re 22 import signal 23 import subprocess 24 import sys 25 26 # Eventually this list will be empty! 27 # If you add or remove tests from this lists please update the CTS 28 # spreadsheet! 29 EXPECTED_FAILURES = set([ 30 'android.webkit.cts.WebViewClientTest#testOnScaleChanged', 31 'android.webkit.cts.WebViewTest#testCapturePicture', 32 # BUG=crbug.com/162967 33 'android.webkit.cts.WebViewTest#testPageScroll', 34 'android.webkit.cts.WebViewTest#testRequestChildRectangleOnScreen', 35 'android.webkit.cts.WebViewTest#testScrollBarOverlay', 36 'android.webkit.cts.WebViewTest#testSetInitialScale', 37 'android.webkit.cts.WebViewTest#testSetScrollBarStyle', 38 'android.webkit.cts.WebViewTest#testSetWebViewClient', 39 'android.webkit.cts.WebViewTest#testZoom', 40 # b/8763845 41 'android.webkit.cts.WebViewTest#testFindAll', 42 # b/8763944 43 'android.webkit.cts.WebViewTest#testGetContentHeight', 44 # See b/8231433 for Geolocation test failures. 45 'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptAlways', 46 'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestReject', 47 'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptOnce', 48 # b/9159785 49 'android.webkit.cts.WebViewTest#testFindNext', 50 # b/5006389 51 'android.webkit.cts.WebViewTest#testFlingScroll', 52 # b/9103603 53 'android.webkit.cts.WebViewTest#testRequestImageRef', 54 # b/9121594 55 'android.webkit.cts.WebHistoryItemTest#testWebHistoryItem', 56 # Don't forget to update the spreadsheet! :) 57 ]) 58 59 def main(): 60 proc = None 61 62 # Send INT signal to test runner and exit gracefully so not to lose all 63 # output information in a run. 64 def handler(signum, frame): 65 if proc: 66 proc.send_signal(signum) 67 signal.signal(signal.SIGINT, handler) 68 69 proc = subprocess.Popen( 70 ['cts-tradefed', 'run', 'singleCommand', 'cts', '-p', 'android.webkit', '--screenshot-on-failure'], 71 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 72 73 (stdout, stderr) = proc.communicate(); 74 75 passes = set(re.findall(r'.*: (.*) PASS', stdout)) 76 failures = set(re.findall(r'.*: (.*) FAIL', stdout)) 77 test_results = '%d passes; %d failures' % (len(passes), len(failures)) 78 79 unexpected_passes = EXPECTED_FAILURES.difference(failures) 80 if len(unexpected_passes) > 0: 81 test_results += '\n' + 'UNEXPECTED PASSES (update expectations!):' 82 for test in unexpected_passes: 83 test_results += '\n' + '\t%s' % (test) 84 85 unexpected_failures = failures.difference(EXPECTED_FAILURES) 86 if len(unexpected_failures) > 0: 87 test_results += '\n' + 'UNEXPECTED FAILURES (please fix!):' 88 for test in unexpected_failures: 89 test_results += '\n' + '\t%s' % (test) 90 91 unexpected_failures_count = len(unexpected_failures) 92 unexpected_passes_count = len(unexpected_passes) 93 94 # on the buildbot this is most useful at the start 95 print test_results 96 97 print '\nstdout dump follows...' 98 print stdout 99 print '\n' 100 101 # on the cmd line this is most useful at the end 102 print test_results 103 104 # Allow buildbot script to distinguish failures and possibly out of date 105 # test expectations. 106 if len(passes) + len(failures) < 100: 107 print 'Ran less than 100 cts tests? Something must be wrong' 108 return 2 109 elif unexpected_failures_count > 0: 110 return 1 111 elif unexpected_passes_count >= 5: 112 print ('More than 5 new passes? Either you''re running webview classic, or ' 113 'it really is time to fix failure expectations.') 114 return 2 115 elif unexpected_passes_count > 0: 116 return 3 # STEP_WARNINGS 117 else: 118 return 0 119 120 121 if __name__ == '__main__': 122 sys.exit(main()) 123
