1 # Copyright (c) 2018 Google LLC 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 """A number of common spirv result checks coded in mixin classes. 15 16 A test case can use these checks by declaring their enclosing mixin classes 17 as superclass and providing the expected_* variables required by the check_*() 18 methods in the mixin classes. 19 """ 20 import difflib 21 import os 22 import re 23 import subprocess 24 from spirv_test_framework import SpirvTest 25 26 27 def convert_to_unix_line_endings(source): 28 """Converts all line endings in source to be unix line endings.""" 29 return source.replace('\r\n', '\n').replace('\r', '\n') 30 31 32 def substitute_file_extension(filename, extension): 33 """Substitutes file extension, respecting known shader extensions. 34 35 foo.vert -> foo.vert.[extension] [similarly for .frag, .comp, etc.] 36 foo.glsl -> foo.[extension] 37 foo.unknown -> foo.[extension] 38 foo -> foo.[extension] 39 """ 40 if filename[-5:] not in [ 41 '.vert', '.frag', '.tesc', '.tese', '.geom', '.comp', '.spvasm' 42 ]: 43 return filename.rsplit('.', 1)[0] + '.' + extension 44 else: 45 return filename + '.' + extension 46 47 48 def get_object_filename(source_filename): 49 """Gets the object filename for the given source file.""" 50 return substitute_file_extension(source_filename, 'spv') 51 52 53 def get_assembly_filename(source_filename): 54 """Gets the assembly filename for the given source file.""" 55 return substitute_file_extension(source_filename, 'spvasm') 56 57 58 def verify_file_non_empty(filename): 59 """Checks that a given file exists and is not empty.""" 60 if not os.path.isfile(filename): 61 return False, 'Cannot find file: ' + filename 62 if not os.path.getsize(filename): 63 return False, 'Empty file: ' + filename 64 return True, '' 65 66 67 class ReturnCodeIsZero(SpirvTest): 68 """Mixin class for checking that the return code is zero.""" 69 70 def check_return_code_is_zero(self, status): 71 if status.returncode: 72 return False, 'Non-zero return code: {ret}\n'.format( 73 ret=status.returncode) 74 return True, '' 75 76 77 class NoOutputOnStdout(SpirvTest): 78 """Mixin class for checking that there is no output on stdout.""" 79 80 def check_no_output_on_stdout(self, status): 81 if status.stdout: 82 return False, 'Non empty stdout: {out}\n'.format(out=status.stdout) 83 return True, '' 84 85 86 class NoOutputOnStderr(SpirvTest): 87 """Mixin class for checking that there is no output on stderr.""" 88 89 def check_no_output_on_stderr(self, status): 90 if status.stderr: 91 return False, 'Non empty stderr: {err}\n'.format(err=status.stderr) 92 return True, '' 93 94 95 class SuccessfulReturn(ReturnCodeIsZero, NoOutputOnStdout, NoOutputOnStderr): 96 """Mixin class for checking that return code is zero and no output on 97 stdout and stderr.""" 98 pass 99 100 101 class NoGeneratedFiles(SpirvTest): 102 """Mixin class for checking that there is no file generated.""" 103 104 def check_no_generated_files(self, status): 105 all_files = os.listdir(status.directory) 106 input_files = status.input_filenames 107 if all([f.startswith(status.directory) for f in input_files]): 108 all_files = [os.path.join(status.directory, f) for f in all_files] 109 generated_files = set(all_files) - set(input_files) 110 if len(generated_files) == 0: 111 return True, '' 112 else: 113 return False, 'Extra files generated: {}'.format(generated_files) 114 115 116 class CorrectBinaryLengthAndPreamble(SpirvTest): 117 """Provides methods for verifying preamble for a SPIR-V binary.""" 118 119 def verify_binary_length_and_header(self, binary, spv_version=0x10000): 120 """Checks that the given SPIR-V binary has valid length and header. 121 122 Returns: 123 False, error string if anything is invalid 124 True, '' otherwise 125 Args: 126 binary: a bytes object containing the SPIR-V binary 127 spv_version: target SPIR-V version number, with same encoding 128 as the version word in a SPIR-V header. 129 """ 130 131 def read_word(binary, index, little_endian): 132 """Reads the index-th word from the given binary file.""" 133 word = binary[index * 4:(index + 1) * 4] 134 if little_endian: 135 word = reversed(word) 136 return reduce(lambda w, b: (w << 8) | ord(b), word, 0) 137 138 def check_endianness(binary): 139 """Checks the endianness of the given SPIR-V binary. 140 141 Returns: 142 True if it's little endian, False if it's big endian. 143 None if magic number is wrong. 144 """ 145 first_word = read_word(binary, 0, True) 146 if first_word == 0x07230203: 147 return True 148 first_word = read_word(binary, 0, False) 149 if first_word == 0x07230203: 150 return False 151 return None 152 153 num_bytes = len(binary) 154 if num_bytes % 4 != 0: 155 return False, ('Incorrect SPV binary: size should be a multiple' 156 ' of words') 157 if num_bytes < 20: 158 return False, 'Incorrect SPV binary: size less than 5 words' 159 160 preamble = binary[0:19] 161 little_endian = check_endianness(preamble) 162 # SPIR-V module magic number 163 if little_endian is None: 164 return False, 'Incorrect SPV binary: wrong magic number' 165 166 # SPIR-V version number 167 version = read_word(preamble, 1, little_endian) 168 # TODO(dneto): Recent Glslang uses version word 0 for opengl_compat 169 # profile 170 171 if version != spv_version and version != 0: 172 return False, 'Incorrect SPV binary: wrong version number' 173 # Shaderc-over-Glslang (0x000d....) or 174 # SPIRV-Tools (0x0007....) generator number 175 if read_word(preamble, 2, little_endian) != 0x000d0007 and \ 176 read_word(preamble, 2, little_endian) != 0x00070000: 177 return False, ('Incorrect SPV binary: wrong generator magic ' 'number') 178 # reserved for instruction schema 179 if read_word(preamble, 4, little_endian) != 0: 180 return False, 'Incorrect SPV binary: the 5th byte should be 0' 181 182 return True, '' 183 184 185 class CorrectObjectFilePreamble(CorrectBinaryLengthAndPreamble): 186 """Provides methods for verifying preamble for a SPV object file.""" 187 188 def verify_object_file_preamble(self, filename, spv_version=0x10000): 189 """Checks that the given SPIR-V binary file has correct preamble.""" 190 191 success, message = verify_file_non_empty(filename) 192 if not success: 193 return False, message 194 195 with open(filename, 'rb') as object_file: 196 object_file.seek(0, os.SEEK_END) 197 num_bytes = object_file.tell() 198 199 object_file.seek(0) 200 201 binary = bytes(object_file.read()) 202 return self.verify_binary_length_and_header(binary, spv_version) 203 204 return True, '' 205 206 207 class CorrectAssemblyFilePreamble(SpirvTest): 208 """Provides methods for verifying preamble for a SPV assembly file.""" 209 210 def verify_assembly_file_preamble(self, filename): 211 success, message = verify_file_non_empty(filename) 212 if not success: 213 return False, message 214 215 with open(filename) as assembly_file: 216 line1 = assembly_file.readline() 217 line2 = assembly_file.readline() 218 line3 = assembly_file.readline() 219 220 if (line1 != '; SPIR-V\n' or line2 != '; Version: 1.0\n' or 221 (not line3.startswith('; Generator: Google Shaderc over Glslang;'))): 222 return False, 'Incorrect SPV assembly' 223 224 return True, '' 225 226 227 class ValidObjectFile(SuccessfulReturn, CorrectObjectFilePreamble): 228 """Mixin class for checking that every input file generates a valid SPIR-V 1.0 229 object file following the object file naming rule, and there is no output on 230 stdout/stderr.""" 231 232 def check_object_file_preamble(self, status): 233 for input_filename in status.input_filenames: 234 object_filename = get_object_filename(input_filename) 235 success, message = self.verify_object_file_preamble( 236 os.path.join(status.directory, object_filename)) 237 if not success: 238 return False, message 239 return True, '' 240 241 242 class ValidObjectFile1_3(ReturnCodeIsZero, CorrectObjectFilePreamble): 243 """Mixin class for checking that every input file generates a valid SPIR-V 1.3 244 object file following the object file naming rule, and there is no output on 245 stdout/stderr.""" 246 247 def check_object_file_preamble(self, status): 248 for input_filename in status.input_filenames: 249 object_filename = get_object_filename(input_filename) 250 success, message = self.verify_object_file_preamble( 251 os.path.join(status.directory, object_filename), 0x10300) 252 if not success: 253 return False, message 254 return True, '' 255 256 257 class ValidObjectFileWithAssemblySubstr(SuccessfulReturn, 258 CorrectObjectFilePreamble): 259 """Mixin class for checking that every input file generates a valid object 260 261 file following the object file naming rule, there is no output on 262 stdout/stderr, and the disassmbly contains a specified substring per 263 input. 264 """ 265 266 def check_object_file_disassembly(self, status): 267 for an_input in status.inputs: 268 object_filename = get_object_filename(an_input.filename) 269 obj_file = str(os.path.join(status.directory, object_filename)) 270 success, message = self.verify_object_file_preamble(obj_file) 271 if not success: 272 return False, message 273 cmd = [status.test_manager.disassembler_path, '--no-color', obj_file] 274 process = subprocess.Popen( 275 args=cmd, 276 stdin=subprocess.PIPE, 277 stdout=subprocess.PIPE, 278 stderr=subprocess.PIPE, 279 cwd=status.directory) 280 output = process.communicate(None) 281 disassembly = output[0] 282 if not isinstance(an_input.assembly_substr, str): 283 return False, 'Missing assembly_substr member' 284 if an_input.assembly_substr not in disassembly: 285 return False, ('Incorrect disassembly output:\n{asm}\n' 286 'Expected substring not found:\n{exp}'.format( 287 asm=disassembly, exp=an_input.assembly_substr)) 288 return True, '' 289 290 291 class ValidNamedObjectFile(SuccessfulReturn, CorrectObjectFilePreamble): 292 """Mixin class for checking that a list of object files with the given 293 names are correctly generated, and there is no output on stdout/stderr. 294 295 To mix in this class, subclasses need to provide expected_object_filenames 296 as the expected object filenames. 297 """ 298 299 def check_object_file_preamble(self, status): 300 for object_filename in self.expected_object_filenames: 301 success, message = self.verify_object_file_preamble( 302 os.path.join(status.directory, object_filename)) 303 if not success: 304 return False, message 305 return True, '' 306 307 308 class ValidFileContents(SpirvTest): 309 """Mixin class to test that a specific file contains specific text 310 To mix in this class, subclasses need to provide expected_file_contents as 311 the contents of the file and target_filename to determine the location.""" 312 313 def check_file(self, status): 314 target_filename = os.path.join(status.directory, self.target_filename) 315 if not os.path.isfile(target_filename): 316 return False, 'Cannot find file: ' + target_filename 317 with open(target_filename, 'r') as target_file: 318 file_contents = target_file.read() 319 if isinstance(self.expected_file_contents, str): 320 if file_contents == self.expected_file_contents: 321 return True, '' 322 return False, ('Incorrect file output: \n{act}\n' 323 'Expected:\n{exp}' 324 'With diff:\n{diff}'.format( 325 act=file_contents, 326 exp=self.expected_file_contents, 327 diff='\n'.join( 328 list( 329 difflib.unified_diff( 330 self.expected_file_contents.split('\n'), 331 file_contents.split('\n'), 332 fromfile='expected_output', 333 tofile='actual_output'))))) 334 elif isinstance(self.expected_file_contents, type(re.compile(''))): 335 if self.expected_file_contents.search(file_contents): 336 return True, '' 337 return False, ('Incorrect file output: \n{act}\n' 338 'Expected matching regex pattern:\n{exp}'.format( 339 act=file_contents, 340 exp=self.expected_file_contents.pattern)) 341 return False, ( 342 'Could not open target file ' + target_filename + ' for reading') 343 344 345 class ValidAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble): 346 """Mixin class for checking that every input file generates a valid assembly 347 file following the assembly file naming rule, and there is no output on 348 stdout/stderr.""" 349 350 def check_assembly_file_preamble(self, status): 351 for input_filename in status.input_filenames: 352 assembly_filename = get_assembly_filename(input_filename) 353 success, message = self.verify_assembly_file_preamble( 354 os.path.join(status.directory, assembly_filename)) 355 if not success: 356 return False, message 357 return True, '' 358 359 360 class ValidAssemblyFileWithSubstr(ValidAssemblyFile): 361 """Mixin class for checking that every input file generates a valid assembly 362 file following the assembly file naming rule, there is no output on 363 stdout/stderr, and all assembly files have the given substring specified 364 by expected_assembly_substr. 365 366 To mix in this class, subclasses need to provde expected_assembly_substr 367 as the expected substring. 368 """ 369 370 def check_assembly_with_substr(self, status): 371 for input_filename in status.input_filenames: 372 assembly_filename = get_assembly_filename(input_filename) 373 success, message = self.verify_assembly_file_preamble( 374 os.path.join(status.directory, assembly_filename)) 375 if not success: 376 return False, message 377 with open(assembly_filename, 'r') as f: 378 content = f.read() 379 if self.expected_assembly_substr not in convert_to_unix_line_endings( 380 content): 381 return False, ('Incorrect assembly output:\n{asm}\n' 382 'Expected substring not found:\n{exp}'.format( 383 asm=content, exp=self.expected_assembly_substr)) 384 return True, '' 385 386 387 class ValidAssemblyFileWithoutSubstr(ValidAssemblyFile): 388 """Mixin class for checking that every input file generates a valid assembly 389 file following the assembly file naming rule, there is no output on 390 stdout/stderr, and no assembly files have the given substring specified 391 by unexpected_assembly_substr. 392 393 To mix in this class, subclasses need to provde unexpected_assembly_substr 394 as the substring we expect not to see. 395 """ 396 397 def check_assembly_for_substr(self, status): 398 for input_filename in status.input_filenames: 399 assembly_filename = get_assembly_filename(input_filename) 400 success, message = self.verify_assembly_file_preamble( 401 os.path.join(status.directory, assembly_filename)) 402 if not success: 403 return False, message 404 with open(assembly_filename, 'r') as f: 405 content = f.read() 406 if self.unexpected_assembly_substr in convert_to_unix_line_endings( 407 content): 408 return False, ('Incorrect assembly output:\n{asm}\n' 409 'Unexpected substring found:\n{unexp}'.format( 410 asm=content, exp=self.unexpected_assembly_substr)) 411 return True, '' 412 413 414 class ValidNamedAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble): 415 """Mixin class for checking that a list of assembly files with the given 416 names are correctly generated, and there is no output on stdout/stderr. 417 418 To mix in this class, subclasses need to provide expected_assembly_filenames 419 as the expected assembly filenames. 420 """ 421 422 def check_object_file_preamble(self, status): 423 for assembly_filename in self.expected_assembly_filenames: 424 success, message = self.verify_assembly_file_preamble( 425 os.path.join(status.directory, assembly_filename)) 426 if not success: 427 return False, message 428 return True, '' 429 430 431 class ErrorMessage(SpirvTest): 432 """Mixin class for tests that fail with a specific error message. 433 434 To mix in this class, subclasses need to provide expected_error as the 435 expected error message. 436 437 The test should fail if the subprocess was terminated by a signal. 438 """ 439 440 def check_has_error_message(self, status): 441 if not status.returncode: 442 return False, ('Expected error message, but returned success from ' 443 'command execution') 444 if status.returncode < 0: 445 # On Unix, a negative value -N for Popen.returncode indicates 446 # termination by signal N. 447 # https://docs.python.org/2/library/subprocess.html 448 return False, ('Expected error message, but command was terminated by ' 449 'signal ' + str(status.returncode)) 450 if not status.stderr: 451 return False, 'Expected error message, but no output on stderr' 452 if self.expected_error != convert_to_unix_line_endings(status.stderr): 453 return False, ('Incorrect stderr output:\n{act}\n' 454 'Expected:\n{exp}'.format( 455 act=status.stderr, exp=self.expected_error)) 456 return True, '' 457 458 459 class ErrorMessageSubstr(SpirvTest): 460 """Mixin class for tests that fail with a specific substring in the error 461 message. 462 463 To mix in this class, subclasses need to provide expected_error_substr as 464 the expected error message substring. 465 466 The test should fail if the subprocess was terminated by a signal. 467 """ 468 469 def check_has_error_message_as_substring(self, status): 470 if not status.returncode: 471 return False, ('Expected error message, but returned success from ' 472 'command execution') 473 if status.returncode < 0: 474 # On Unix, a negative value -N for Popen.returncode indicates 475 # termination by signal N. 476 # https://docs.python.org/2/library/subprocess.html 477 return False, ('Expected error message, but command was terminated by ' 478 'signal ' + str(status.returncode)) 479 if not status.stderr: 480 return False, 'Expected error message, but no output on stderr' 481 if self.expected_error_substr not in convert_to_unix_line_endings( 482 status.stderr): 483 return False, ('Incorrect stderr output:\n{act}\n' 484 'Expected substring not found in stderr:\n{exp}'.format( 485 act=status.stderr, exp=self.expected_error_substr)) 486 return True, '' 487 488 489 class WarningMessage(SpirvTest): 490 """Mixin class for tests that succeed but have a specific warning message. 491 492 To mix in this class, subclasses need to provide expected_warning as the 493 expected warning message. 494 """ 495 496 def check_has_warning_message(self, status): 497 if status.returncode: 498 return False, ('Expected warning message, but returned failure from' 499 ' command execution') 500 if not status.stderr: 501 return False, 'Expected warning message, but no output on stderr' 502 if self.expected_warning != convert_to_unix_line_endings(status.stderr): 503 return False, ('Incorrect stderr output:\n{act}\n' 504 'Expected:\n{exp}'.format( 505 act=status.stderr, exp=self.expected_warning)) 506 return True, '' 507 508 509 class ValidObjectFileWithWarning(NoOutputOnStdout, CorrectObjectFilePreamble, 510 WarningMessage): 511 """Mixin class for checking that every input file generates a valid object 512 file following the object file naming rule, with a specific warning message. 513 """ 514 515 def check_object_file_preamble(self, status): 516 for input_filename in status.input_filenames: 517 object_filename = get_object_filename(input_filename) 518 success, message = self.verify_object_file_preamble( 519 os.path.join(status.directory, object_filename)) 520 if not success: 521 return False, message 522 return True, '' 523 524 525 class ValidAssemblyFileWithWarning(NoOutputOnStdout, 526 CorrectAssemblyFilePreamble, WarningMessage): 527 """Mixin class for checking that every input file generates a valid assembly 528 file following the assembly file naming rule, with a specific warning 529 message.""" 530 531 def check_assembly_file_preamble(self, status): 532 for input_filename in status.input_filenames: 533 assembly_filename = get_assembly_filename(input_filename) 534 success, message = self.verify_assembly_file_preamble( 535 os.path.join(status.directory, assembly_filename)) 536 if not success: 537 return False, message 538 return True, '' 539 540 541 class StdoutMatch(SpirvTest): 542 """Mixin class for tests that can expect output on stdout. 543 544 To mix in this class, subclasses need to provide expected_stdout as the 545 expected stdout output. 546 547 For expected_stdout, if it's True, then they expect something on stdout but 548 will not check what it is. If it's a string, expect an exact match. If it's 549 anything else, it is assumed to be a compiled regular expression which will 550 be matched against re.search(). It will expect 551 expected_stdout.search(status.stdout) to be true. 552 """ 553 554 def check_stdout_match(self, status): 555 # "True" in this case means we expect something on stdout, but we do not 556 # care what it is, we want to distinguish this from "blah" which means we 557 # expect exactly the string "blah". 558 if self.expected_stdout is True: 559 if not status.stdout: 560 return False, 'Expected something on stdout' 561 elif type(self.expected_stdout) == str: 562 if self.expected_stdout != convert_to_unix_line_endings(status.stdout): 563 return False, ('Incorrect stdout output:\n{ac}\n' 564 'Expected:\n{ex}'.format( 565 ac=status.stdout, ex=self.expected_stdout)) 566 else: 567 if not self.expected_stdout.search( 568 convert_to_unix_line_endings(status.stdout)): 569 return False, ('Incorrect stdout output:\n{ac}\n' 570 'Expected to match regex:\n{ex}'.format( 571 ac=status.stdout, ex=self.expected_stdout.pattern)) 572 return True, '' 573 574 575 class StderrMatch(SpirvTest): 576 """Mixin class for tests that can expect output on stderr. 577 578 To mix in this class, subclasses need to provide expected_stderr as the 579 expected stderr output. 580 581 For expected_stderr, if it's True, then they expect something on stderr, 582 but will not check what it is. If it's a string, expect an exact match. 583 If it's anything else, it is assumed to be a compiled regular expression 584 which will be matched against re.search(). It will expect 585 expected_stderr.search(status.stderr) to be true. 586 """ 587 588 def check_stderr_match(self, status): 589 # "True" in this case means we expect something on stderr, but we do not 590 # care what it is, we want to distinguish this from "blah" which means we 591 # expect exactly the string "blah". 592 if self.expected_stderr is True: 593 if not status.stderr: 594 return False, 'Expected something on stderr' 595 elif type(self.expected_stderr) == str: 596 if self.expected_stderr != convert_to_unix_line_endings(status.stderr): 597 return False, ('Incorrect stderr output:\n{ac}\n' 598 'Expected:\n{ex}'.format( 599 ac=status.stderr, ex=self.expected_stderr)) 600 else: 601 if not self.expected_stderr.search( 602 convert_to_unix_line_endings(status.stderr)): 603 return False, ('Incorrect stderr output:\n{ac}\n' 604 'Expected to match regex:\n{ex}'.format( 605 ac=status.stderr, ex=self.expected_stderr.pattern)) 606 return True, '' 607 608 609 class StdoutNoWiderThan80Columns(SpirvTest): 610 """Mixin class for tests that require stdout to 80 characters or narrower. 611 612 To mix in this class, subclasses need to provide expected_stdout as the 613 expected stdout output. 614 """ 615 616 def check_stdout_not_too_wide(self, status): 617 if not status.stdout: 618 return True, '' 619 else: 620 for line in status.stdout.splitlines(): 621 if len(line) > 80: 622 return False, ('Stdout line longer than 80 columns: %s' % line) 623 return True, '' 624 625 626 class NoObjectFile(SpirvTest): 627 """Mixin class for checking that no input file has a corresponding object 628 file.""" 629 630 def check_no_object_file(self, status): 631 for input_filename in status.input_filenames: 632 object_filename = get_object_filename(input_filename) 633 full_object_file = os.path.join(status.directory, object_filename) 634 print('checking %s' % full_object_file) 635 if os.path.isfile(full_object_file): 636 return False, ( 637 'Expected no object file, but found: %s' % full_object_file) 638 return True, '' 639 640 641 class NoNamedOutputFiles(SpirvTest): 642 """Mixin class for checking that no specified output files exist. 643 644 The expected_output_filenames member should be full pathnames.""" 645 646 def check_no_named_output_files(self, status): 647 for object_filename in self.expected_output_filenames: 648 if os.path.isfile(object_filename): 649 return False, ( 650 'Expected no output file, but found: %s' % object_filename) 651 return True, '' 652 653 654 class ExecutedListOfPasses(SpirvTest): 655 """Mixin class for checking that a list of passes where executed. 656 657 It works by analyzing the output of the --print-all flag to spirv-opt. 658 659 For this mixin to work, the class member expected_passes should be a sequence 660 of pass names as returned by Pass::name(). 661 """ 662 663 def check_list_of_executed_passes(self, status): 664 # Collect all the output lines containing a pass name. 665 pass_names = [] 666 pass_name_re = re.compile(r'.*IR before pass (?P<pass_name>[\S]+)') 667 for line in status.stderr.splitlines(): 668 match = pass_name_re.match(line) 669 if match: 670 pass_names.append(match.group('pass_name')) 671 672 for (expected, actual) in zip(self.expected_passes, pass_names): 673 if expected != actual: 674 return False, ( 675 'Expected pass "%s" but found pass "%s"\n' % (expected, actual)) 676 677 return True, '' 678