1 #!/usr/bin/python3 2 3 # Copyright 2017, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 18 """NN Model Test Compiler Test. 19 20 Runs subdirectories of tests for the test generator/compiler. 21 """ 22 23 import filecmp 24 import glob 25 import os 26 import re 27 import shutil 28 import subprocess 29 import sys 30 import unittest 31 32 33 __author__ = 'Android' 34 35 36 DOTTED_LINE = '................' 37 38 class OrigFile: 39 OrigDir = None 40 41 class TestGeneratorTests(unittest.TestCase): 42 """Class to contain all the unittest test cases. 43 44 Tests will be dynamically added to this class as methods. 45 No static tests, so this class is initially empty. 46 47 """ 48 pass 49 50 51 def GenerateTests(dir_name): 52 """Creates a test method that can be added as method to GenerateTests.""" 53 cwd = os.getcwd() 54 def Test(self): 55 os.chdir(cwd) 56 ExecTest(dir_name, self) 57 return Test 58 59 60 def AddUnitTests(test_dirs): 61 """Adds a test to Tests for each directory in test_dirs.""" 62 63 for t in test_dirs: 64 # Must start with 'test_' according to unittest 65 test_name = 'test_%s' % t 66 test = GenerateTests(t) 67 # Add test as method to TestGeneratorTests with test_name as method name 68 setattr(TestGeneratorTests, test_name, test) 69 70 71 class Options(object): 72 verbose = 0 73 cleanup = 1 74 update_cts = 0 75 zero_return = 0 76 77 78 def CompareFiles(actual, expect): 79 """Compares actual and expect for equality.""" 80 if not os.path.isfile(actual): 81 if Options.verbose: 82 print ('Could not find %s' % actual) 83 return False 84 if not os.path.isfile(expect): 85 if Options.verbose: 86 print ('Could not find %s' % expect) 87 return False 88 89 return filecmp.cmp(actual, expect, False) 90 91 92 def CopyIfDifferent(src, dst): 93 """Updates dst if it is different from src.""" 94 if not CompareFiles(src, dst): 95 if Options.verbose: 96 print ('Copying from %s to %s' % (src, dst)) 97 shutil.copyfile(src, dst) 98 99 100 def GetCommandLineArgs(filename): 101 """Extracts command line arguments from first comment line in a file.""" 102 f = open(filename, 'r') 103 line = f.readline() 104 f.close() 105 if line[0] == '/' and line[1] == '/': 106 return line[2:].strip() 107 else: 108 return '' 109 110 111 def ReadFileToStr(filename): 112 """Returns contents of file as a str.""" 113 with open(filename, 'r') as f: 114 return f.read() 115 116 117 def ReportIfDifferFromExpected(tests, name, file1, file2): 118 """Fails tests if file1 and file2 differ.""" 119 if not CompareFiles(file1, file2): 120 if Options.verbose: 121 err_message = ('%s is different:\n' 122 'expected:\n%s\n%s%s\n\n' 123 'actual:\n%s\n%s%s\n') % ( 124 name, 125 DOTTED_LINE, ReadFileToStr(file1), DOTTED_LINE, 126 DOTTED_LINE, ReadFileToStr(file2), DOTTED_LINE) 127 else: 128 err_message = '%s is different' % name 129 tests.fail(err_message) 130 131 132 def GetRSFiles(): 133 """Returns a list of files in cwd with extension '.rs' or '.fs'.""" 134 rs_files = glob.glob('*.mod.py') 135 rs_files.sort() 136 return rs_files 137 138 139 def GetOutDir(): 140 return os.path.abspath(os.path.join(OrigFile.OrigDir, "../")) 141 142 143 # Declare/define cache variable for GetOutDir to cache results 144 # This way we only need to call subprocesses once to get the directory 145 GetOutDir.cache = None 146 147 148 def CreateCmd(): 149 """Creates the test command to run for the current test.""" 150 cmd_string = ('%s/test_generator.py' 151 ) % GetOutDir() 152 base_args = cmd_string.split() 153 rs_files = GetRSFiles() 154 155 # Extra command line arguments can be placed as // comments at the start of 156 # any .rs file. We automatically bundle up all of these extra args and invoke 157 # llvm-rs-cc with them. 158 extra_args_str = '' 159 for rs_file in rs_files: 160 extra_args_str += GetCommandLineArgs(rs_file) 161 extra_args = extra_args_str.split() 162 163 args = base_args + extra_args + rs_files 164 return args 165 166 def Cleanup(): 167 """Cleans up the cwd of any tmp files created in current test.""" 168 try: 169 os.remove('stdout.txt') 170 os.remove('stderr.txt') 171 shutil.rmtree('tmp/') 172 except OSError: 173 pass 174 175 176 def CheckTestResult(dir_name, subprocess_ret, tests, args): 177 """Checks the result of the subprocess command to see if it passed/failed. 178 179 If dir_name starts with 'F_', then subprocess is expected to fail. 180 If it instead succeeded, then this test is failed. 181 Vice versa with a dir_name starting with 'P_'. 182 183 Args: 184 dir_name: name of current directory/test name 185 subprocess_ret: return code of subprocess 186 tests: unittest, call tests.fail(reason) when failure 187 args: the arguments for the command that was run 188 """ 189 if dir_name[0:2] == 'F_': 190 if subprocess_ret == 0: 191 if Options.verbose: 192 err_message = ('Command (%s) passed on invalid input\n' 193 'stdout:\n%s\n%s%s\n') % ( 194 ' '.join(args), 195 DOTTED_LINE, ReadFileToStr('stdout.txt'), DOTTED_LINE 196 ) 197 else: 198 err_message = 'Command passed on invalid input' 199 tests.fail(err_message) 200 elif dir_name[0:2] == 'P_': 201 if subprocess_ret != 0: 202 if Options.verbose: 203 err_message = ('Command (%s) failed on valid input\n' 204 'stderr:\n%s\n%s%s\n') % ( 205 ' '.join(args), 206 DOTTED_LINE, ReadFileToStr('stderr.txt'), DOTTED_LINE 207 ) 208 else: 209 err_message = 'Command failed on valid input' 210 tests.fail(err_message) 211 else: 212 tests.fail('Invalid test name: ' + dir_name + 213 ', should start with F_ or P_') 214 215 216 217 def ExecTest(dir_name, tests): 218 """Executes an llvm-rs-cc test from dir_name.""" 219 220 os.chdir(dir_name) 221 stdout_file = open('stdout.txt', 'w+') 222 stderr_file = open('stderr.txt', 'w+') 223 224 args = CreateCmd() 225 226 if Options.verbose > 1: 227 print ('Executing:', ' '.join(args)) 228 229 # Execute the command and check the resulting shell return value. 230 # All tests that are expected to FAIL have directory names that 231 # start with 'F_'. Other tests that are expected to PASS have 232 # directory names that start with 'P_'. 233 ret = 0 234 try: 235 ret = subprocess.call(args, stdout=stdout_file, stderr=stderr_file) 236 except OSError: 237 tests.fail('subprocess.call failed: ' + ' '.join(args)) 238 239 stdout_file.close() 240 stderr_file.close() 241 242 CheckTestResult(dir_name, ret, tests, args) 243 244 ReportIfDifferFromExpected(tests, 'stdout', 'stdout.txt.expect', 'stdout.txt') 245 ReportIfDifferFromExpected(tests, 'stderr', 'stderr.txt.expect', 'stderr.txt') 246 247 if Options.cleanup: 248 Cleanup() 249 250 251 def Usage(): 252 """Print out usage information.""" 253 print ('Usage: %s [OPTION]... [TESTNAME]...' 254 'Renderscript Compiler Test Harness\n' 255 'Runs TESTNAMEs (all tests by default)\n' 256 'Available Options:\n' 257 ' -h, --help Help message\n' 258 ' -n, --no-cleanup Don\'t clean up after running tests\n' 259 ' -v, --verbose Verbose output. Enter multiple -v to get more verbose.\n' 260 ' -z, --zero-return Return 0 as exit code no matter if tests fail. Required for TreeHugger.\n' 261 ) % (sys.argv[0]), 262 return 263 264 265 def main(): 266 """Runs the unittest suite. 267 268 Parses command line arguments, adds test directories as tests. 269 270 Returns: 271 0 if '-z' flag is set. 272 Else unittest.main() returns with its own error code. 273 """ 274 275 OrigFile.OrigDir = os.path.dirname(os.path.abspath(__file__)) 276 # Chdir to the directory this file is in since tests are in this directory 277 os.chdir(OrigFile.OrigDir) 278 files = [] 279 for arg in sys.argv[1:]: 280 if arg in ('-h', '--help'): 281 Usage() 282 return 0 283 elif arg in ('-n', '--no-cleanup'): 284 Options.cleanup = 0 285 elif arg in ('-u', '--update-cts'): 286 Options.update_cts = 1 287 elif arg in ('-v', '--verbose'): 288 Options.verbose += 1 289 elif arg in ('-z', '--zero-return'): 290 Options.zero_return = 1 291 else: 292 # Test list to run 293 if os.path.isdir(arg): 294 files.append(arg) 295 else: 296 print >> sys.stderr, 'Invalid test or option: %s' % arg 297 return 1 298 299 if not files: 300 file_names = os.listdir('.') 301 # Test names must start with 'F_' or 'P_' 302 # 'F_' tests are expected to fail 303 # 'P_' tests are expected to pass 304 for f in file_names: 305 if os.path.isdir(f) and (f[0:2] == 'F_' or f[0:2] == 'P_'): 306 files.append(f) 307 files.sort() 308 309 AddUnitTests(files) 310 311 # verbosity=2 is necessary for PythonUnitTestRunner to parse the results 312 # Otherwise verbosity does not matter 313 # If Options.zero_return is set, do not let unittest.main() exit 314 # This is necessary in TreeHugger to distinguish between failing tests and 315 # failing to execute the python script 316 # If Options.zero_return is not set, let unittest.main() exit 317 # In this case it will return a non-zero code if any tests fail 318 unittest_exit = Options.zero_return == 0 319 unittest.main(verbosity=2, 320 argv=[sys.argv[0]] + ['TestGeneratorTests'], 321 exit=unittest_exit) 322 323 return 0 324 325 326 if __name__ == '__main__': 327 sys.exit(main()) 328 329
