1 #!/usr/bin/env python 2 # Copyright 2016 The PDFium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import cStringIO 7 import functools 8 import multiprocessing 9 import optparse 10 import os 11 import re 12 import shutil 13 import subprocess 14 import sys 15 16 import common 17 import gold 18 import pngdiffer 19 import suppressor 20 21 class KeyboardInterruptError(Exception): pass 22 23 # Nomenclature: 24 # x_root - "x" 25 # x_filename - "x.ext" 26 # x_path - "path/to/a/b/c/x.ext" 27 # c_dir - "path/to/a/b/c" 28 29 def TestOneFileParallel(this, test_case): 30 """Wrapper to call GenerateAndTest() and redirect output to stdout.""" 31 try: 32 input_filename, source_dir = test_case 33 result = this.GenerateAndTest(input_filename, source_dir); 34 return (result, input_filename, source_dir) 35 except KeyboardInterrupt: 36 raise KeyboardInterruptError() 37 38 39 class TestRunner: 40 def __init__(self, dirname): 41 self.test_dir = dirname 42 self.enforce_expected_images = False 43 self.oneshot_renderer = False 44 45 # GenerateAndTest returns a tuple <success, outputfiles> where 46 # success is a boolean indicating whether the tests passed comparison 47 # tests and outputfiles is a list tuples: 48 # (path_to_image, md5_hash_of_pixelbuffer) 49 def GenerateAndTest(self, input_filename, source_dir): 50 input_root, _ = os.path.splitext(input_filename) 51 expected_txt_path = os.path.join(source_dir, input_root + '_expected.txt') 52 53 pdf_path = os.path.join(self.working_dir, input_root + '.pdf') 54 55 # Remove any existing generated images from previous runs. 56 actual_images = self.image_differ.GetActualFiles(input_filename, source_dir, 57 self.working_dir) 58 for image in actual_images: 59 if os.path.exists(image): 60 os.remove(image) 61 62 sys.stdout.flush() 63 64 raised_exception = self.Generate(source_dir, input_filename, input_root, 65 pdf_path) 66 67 if raised_exception is not None: 68 print 'FAILURE: %s; %s' % (input_filename, raised_exception) 69 return False, [] 70 71 results = [] 72 if os.path.exists(expected_txt_path): 73 raised_exception = self.TestText(input_root, expected_txt_path, pdf_path) 74 else: 75 raised_exception, results = self.TestPixel(input_root, pdf_path) 76 77 if raised_exception is not None: 78 print 'FAILURE: %s; %s' % (input_filename, raised_exception) 79 return False, results 80 81 if actual_images: 82 if self.image_differ.HasDifferences(input_filename, source_dir, 83 self.working_dir): 84 if (self.options.regenerate_expected 85 and not self.test_suppressor.IsResultSuppressed(input_filename) 86 and not self.test_suppressor.IsImageDiffSuppressed(input_filename)): 87 platform_only = (self.options.regenerate_expected == 'platform') 88 self.image_differ.Regenerate(input_filename, source_dir, 89 self.working_dir, platform_only) 90 return False, results 91 else: 92 if (self.enforce_expected_images 93 and not self.test_suppressor.IsImageDiffSuppressed(input_filename)): 94 print 'FAILURE: %s; Missing expected images' % input_filename 95 return False, results 96 97 return True, results 98 99 def Generate(self, source_dir, input_filename, input_root, pdf_path): 100 original_path = os.path.join(source_dir, input_filename) 101 input_path = os.path.join(source_dir, input_root + '.in') 102 103 input_event_path = os.path.join(source_dir, input_root + '.evt') 104 if os.path.exists(input_event_path): 105 output_event_path = os.path.splitext(pdf_path)[0] + '.evt' 106 shutil.copyfile(input_event_path, output_event_path) 107 108 if not os.path.exists(input_path): 109 if os.path.exists(original_path): 110 shutil.copyfile(original_path, pdf_path) 111 return None 112 113 sys.stdout.flush() 114 115 return common.RunCommand( 116 [sys.executable, self.fixup_path, '--output-dir=' + self.working_dir, 117 input_path]) 118 119 def TestText(self, input_root, expected_txt_path, pdf_path): 120 txt_path = os.path.join(self.working_dir, input_root + '.txt') 121 122 with open(txt_path, 'w') as outfile: 123 cmd_to_run = [self.pdfium_test_path, '--send-events', pdf_path] 124 subprocess.check_call(cmd_to_run, stdout=outfile) 125 126 cmd = [sys.executable, self.text_diff_path, expected_txt_path, txt_path] 127 return common.RunCommand(cmd) 128 129 def TestPixel(self, input_root, pdf_path): 130 cmd_to_run = [self.pdfium_test_path, '--send-events', '--png', '--md5'] 131 if self.oneshot_renderer: 132 cmd_to_run.append('--render-oneshot') 133 cmd_to_run.append(pdf_path) 134 return common.RunCommandExtractHashedFiles(cmd_to_run) 135 136 def HandleResult(self, input_filename, input_path, result): 137 success, image_paths = result 138 139 if image_paths: 140 for img_path, md5_hash in image_paths: 141 # The output filename without image extension becomes the test name. 142 # For example, "/path/to/.../testing/corpus/example_005.pdf.0.png" 143 # becomes "example_005.pdf.0". 144 test_name = os.path.splitext(os.path.split(img_path)[1])[0] 145 146 if not self.test_suppressor.IsResultSuppressed(input_filename): 147 matched = self.gold_baseline.MatchLocalResult(test_name, md5_hash) 148 if matched == gold.GoldBaseline.MISMATCH: 149 print 'Skia Gold hash mismatch for test case: %s' % test_name 150 elif matched == gold.GoldBaseline.NO_BASELINE: 151 print 'No Skia Gold baseline found for test case: %s' % test_name 152 153 if self.gold_results: 154 self.gold_results.AddTestResult(test_name, md5_hash, img_path) 155 156 if self.test_suppressor.IsResultSuppressed(input_filename): 157 self.result_suppressed_cases.append(input_filename) 158 if success: 159 self.surprises.append(input_path) 160 else: 161 if not success: 162 self.failures.append(input_path) 163 164 def Run(self): 165 parser = optparse.OptionParser() 166 167 parser.add_option('--build-dir', default=os.path.join('out', 'Debug'), 168 help='relative path from the base source directory') 169 170 parser.add_option('-j', default=multiprocessing.cpu_count(), 171 dest='num_workers', type='int', 172 help='run NUM_WORKERS jobs in parallel') 173 174 parser.add_option('--gold_properties', default='', dest="gold_properties", 175 help='Key value pairs that are written to the top level ' 176 'of the JSON file that is ingested by Gold.') 177 178 parser.add_option('--gold_key', default='', dest="gold_key", 179 help='Key value pairs that are added to the "key" field ' 180 'of the JSON file that is ingested by Gold.') 181 182 parser.add_option('--gold_output_dir', default='', dest="gold_output_dir", 183 help='Path of where to write the JSON output to be ' 184 'uploaded to Gold.') 185 186 parser.add_option('--gold_ignore_hashes', default='', 187 dest="gold_ignore_hashes", 188 help='Path to a file with MD5 hashes we wish to ignore.') 189 190 parser.add_option('--regenerate_expected', default='', 191 dest="regenerate_expected", 192 help='Regenerates expected images. Valid values are ' 193 '"all" to regenerate all expected pngs, and ' 194 '"platform" to regenerate only platform-specific ' 195 'expected pngs.') 196 197 parser.add_option('--ignore_errors', action="store_true", 198 dest="ignore_errors", 199 help='Prevents the return value from being non-zero ' 200 'when image comparison fails.') 201 202 self.options, self.args = parser.parse_args() 203 204 if (self.options.regenerate_expected 205 and self.options.regenerate_expected not in ['all', 'platform']) : 206 print 'FAILURE: --regenerate_expected must be "all" or "platform"' 207 return 1 208 209 finder = common.DirectoryFinder(self.options.build_dir) 210 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') 211 self.text_diff_path = finder.ScriptPath('text_diff.py') 212 213 self.source_dir = finder.TestingDir() 214 if self.test_dir != 'corpus': 215 test_dir = finder.TestingDir(os.path.join('resources', self.test_dir)) 216 else: 217 test_dir = finder.TestingDir(self.test_dir) 218 219 self.pdfium_test_path = finder.ExecutablePath('pdfium_test') 220 if not os.path.exists(self.pdfium_test_path): 221 print "FAILURE: Can't find test executable '%s'" % self.pdfium_test_path 222 print 'Use --build-dir to specify its location.' 223 return 1 224 225 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) 226 if not os.path.exists(self.working_dir): 227 os.makedirs(self.working_dir) 228 229 self.feature_string = subprocess.check_output([self.pdfium_test_path, 230 '--show-config']) 231 self.test_suppressor = suppressor.Suppressor(finder, self.feature_string) 232 self.image_differ = pngdiffer.PNGDiffer(finder) 233 234 self.gold_baseline = gold.GoldBaseline(self.options.gold_properties) 235 236 walk_from_dir = finder.TestingDir(test_dir); 237 238 self.test_cases = [] 239 self.execution_suppressed_cases = [] 240 input_file_re = re.compile('^.+[.](in|pdf)$') 241 if self.args: 242 for file_name in self.args: 243 file_name.replace('.pdf', '.in') 244 input_path = os.path.join(walk_from_dir, file_name) 245 if not os.path.isfile(input_path): 246 print "Can't find test file '%s'" % file_name 247 return 1 248 249 self.test_cases.append((os.path.basename(input_path), 250 os.path.dirname(input_path))) 251 else: 252 for file_dir, _, filename_list in os.walk(walk_from_dir): 253 for input_filename in filename_list: 254 if input_file_re.match(input_filename): 255 input_path = os.path.join(file_dir, input_filename) 256 if self.test_suppressor.IsExecutionSuppressed(input_path): 257 self.execution_suppressed_cases.append(input_path) 258 else: 259 if os.path.isfile(input_path): 260 self.test_cases.append((input_filename, file_dir)) 261 262 self.failures = [] 263 self.surprises = [] 264 self.result_suppressed_cases = [] 265 266 # Collect Gold results if an output directory was named. 267 self.gold_results = None 268 if self.options.gold_output_dir: 269 self.gold_results = gold.GoldResults('pdfium', 270 self.options.gold_output_dir, 271 self.options.gold_properties, 272 self.options.gold_key, 273 self.options.gold_ignore_hashes) 274 275 if self.options.num_workers > 1 and len(self.test_cases) > 1: 276 try: 277 pool = multiprocessing.Pool(self.options.num_workers) 278 worker_func = functools.partial(TestOneFileParallel, self) 279 280 worker_results = pool.imap(worker_func, self.test_cases) 281 for worker_result in worker_results: 282 result, input_filename, source_dir = worker_result 283 input_path = os.path.join(source_dir, input_filename) 284 285 self.HandleResult(input_filename, input_path, result) 286 287 except KeyboardInterrupt: 288 pool.terminate() 289 finally: 290 pool.close() 291 pool.join() 292 else: 293 for test_case in self.test_cases: 294 input_filename, input_file_dir = test_case 295 result = self.GenerateAndTest(input_filename, input_file_dir) 296 self.HandleResult(input_filename, 297 os.path.join(input_file_dir, input_filename), result) 298 299 if self.gold_results: 300 self.gold_results.WriteResults() 301 302 if self.surprises: 303 self.surprises.sort() 304 print '\n\nUnexpected Successes:' 305 for surprise in self.surprises: 306 print surprise 307 308 if self.failures: 309 self.failures.sort() 310 print '\n\nSummary of Failures:' 311 for failure in self.failures: 312 print failure 313 314 self._PrintSummary() 315 316 if self.failures: 317 if not self.options.ignore_errors: 318 return 1 319 320 return 0 321 322 def _PrintSummary(self): 323 number_test_cases = len(self.test_cases) 324 number_failures = len(self.failures) 325 number_suppressed = len(self.result_suppressed_cases) 326 number_successes = number_test_cases - number_failures - number_suppressed 327 number_surprises = len(self.surprises) 328 print 329 print 'Test cases executed: %d' % number_test_cases 330 print ' Successes: %d' % number_successes 331 print ' Suppressed: %d' % number_suppressed 332 print ' Surprises: %d' % number_surprises 333 print ' Failures: %d' % number_failures 334 print 335 print 'Test cases not executed: %d' % len(self.execution_suppressed_cases) 336 337 def SetEnforceExpectedImages(self, new_value): 338 """Set whether to enforce that each test case provide an expected image.""" 339 self.enforce_expected_images = new_value 340 341 def SetOneShotRenderer(self, new_value): 342 """Set whether to use the oneshot renderer. """ 343 self.oneshot_renderer = new_value 344
