1 #!/usr/bin/env ruby 2 # coding: binary 3 # 4 # Copyright 2015 Google Inc. All rights reserved 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); 7 # you may not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, 14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 18 require 'fileutils' 19 20 # suppress GNU make jobserver magic when calling "make" 21 ENV.delete('MAKEFLAGS') 22 ENV.delete('MAKELEVEL') 23 24 while true 25 if ARGV[0] == '-s' 26 test_serialization = true 27 ARGV.shift 28 elsif ARGV[0] == '-c' 29 ckati = true 30 ARGV.shift 31 ENV['KATI_VARIANT'] = 'c' 32 elsif ARGV[0] == '-n' 33 via_ninja = true 34 ARGV.shift 35 ENV['NINJA_STATUS'] = 'NINJACMD: ' 36 elsif ARGV[0] == '-a' 37 gen_all_targets = true 38 ARGV.shift 39 elsif ARGV[0] == '-v' 40 show_failing = true 41 ARGV.shift 42 else 43 break 44 end 45 end 46 47 def get_output_filenames 48 files = Dir.glob('*') 49 files.delete('Makefile') 50 files.delete('build.ninja') 51 files.delete('env.sh') 52 files.delete('ninja.sh') 53 files.delete('gmon.out') 54 files.delete('submake') 55 files.reject!{|f|f =~ /\.json$/} 56 files.reject!{|f|f =~ /^kati\.*/} 57 files 58 end 59 60 def cleanup 61 (get_output_filenames + Dir.glob('.*')).each do |fname| 62 next if fname == '.' || fname == '..' 63 FileUtils.rm_rf fname 64 end 65 end 66 67 def move_circular_dep(l) 68 # We don't care when circular dependency detection happens. 69 circ = '' 70 while l.sub!(/Circular .* dropped\.\n/, '') do 71 circ += $& 72 end 73 circ + l 74 end 75 76 expected_failures = [] 77 unexpected_passes = [] 78 failures = [] 79 passes = [] 80 81 if !ARGV.empty? 82 test_files = ARGV.map do |test| 83 "testcase/#{File.basename(test)}" 84 end 85 else 86 test_files = Dir.glob('testcase/*.mk').sort 87 test_files += Dir.glob('testcase/*.sh').sort 88 end 89 90 def run_in_testdir(test_filename) 91 c = File.read(test_filename) 92 name = File.basename(test_filename) 93 dir = "out/#{name}" 94 95 FileUtils.mkdir_p(dir) 96 Dir.glob("#{dir}/*").each do |fname| 97 FileUtils.rm_rf(fname) 98 end 99 100 Dir.chdir(dir) do 101 yield name 102 end 103 end 104 105 def normalize_ninja_log(log, mk) 106 log.gsub!(/^NINJACMD: .*\n/, '') 107 log.gsub!(/^ninja: no work to do\.\n/, '') 108 log.gsub!(/^ninja: error: (.*, needed by .*),.*/, 109 '*** No rule to make target \\1.') 110 log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/, 111 'ninja: warning: multiple rules generate \\1.') 112 113 if mk =~ /err_error_in_recipe.mk/ 114 # This test expects ninja fails. Strip ninja specific error logs. 115 ninja_failed_subst = '' 116 elsif mk =~ /\/fail_/ 117 # Recipes in these tests fail. 118 ninja_failed_subst = "*** [test] Error 1\n" 119 end 120 if ninja_failed_subst 121 log.gsub!(/^FAILED: (.*\n\/bin\/bash)?.*\n/, ninja_failed_subst) 122 log.gsub!(/^ninja: .*\n/, '') 123 end 124 log 125 end 126 127 def normalize_quotes(log) 128 log.gsub!(/[`'"]/, '"') 129 # For recent GNU find, which uses Unicode characters. 130 log.gsub!(/(\xe2\x80\x98|\xe2\x80\x99)/, '"') 131 log 132 end 133 134 def normalize_make_log(expected, mk, via_ninja) 135 expected = normalize_quotes(expected) 136 expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') 137 expected.gsub!(/^make(?:\[\d+\])?: /, '') 138 expected = move_circular_dep(expected) 139 140 # Normalizations for old/new GNU make. 141 expected.gsub!(' recipe for target ', ' commands for target ') 142 expected.gsub!(' recipe commences ', ' commands commence ') 143 expected.gsub!('missing rule before recipe.', 'missing rule before commands.') 144 expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') 145 expected.gsub!('Extraneous text after', 'extraneous text after') 146 # Not sure if this is useful. 147 expected.gsub!(/\s+Stop\.$/, '') 148 # GNU make 4.0 has this output. 149 expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') 150 # We treat some warnings as errors. 151 expected.gsub!(/^\/bin\/(ba)?sh: line 0: /, '') 152 # We print out some ninja warnings in some tests to match what we expect 153 # ninja to produce. Remove them if we're not testing ninja. 154 if !via_ninja 155 expected.gsub!(/^ninja: warning: .*\n/, '') 156 end 157 # Normalization for "include foo" with C++ kati. 158 expected.gsub!(/(: )(\S+): (No such file or directory)\n\*\*\* No rule to make target "\2"./, '\1\2: \3') 159 160 expected 161 end 162 163 def normalize_kati_log(output) 164 output = normalize_quotes(output) 165 output = move_circular_dep(output) 166 167 # kati specific log messages. 168 output.gsub!(/^\*kati\*.*\n/, '') 169 output.gsub!(/^c?kati: /, '') 170 output.gsub!(/\/bin\/sh: ([^:]*): command not found/, 171 "\\1: Command not found") 172 output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '') 173 output.gsub!(/^([^ ]+: )?FindEmulator: /, '') 174 output.gsub!(/^\/bin\/sh: line 0: /, '') 175 output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk 176 output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk 177 # Normalization for "include foo" with Go kati. 178 output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/, 179 "\\1\\2: N\\3") 180 output 181 end 182 183 bash_var = ' SHELL=/bin/bash' 184 185 run_make_test = proc do |mk| 186 c = File.read(mk) 187 expected_failure = false 188 if c =~ /\A# TODO(?:\(([-a-z|]+)\))?/ 189 if $1 190 todos = $1.split('|') 191 if todos.include?('go') && !ckati 192 expected_failure = true 193 end 194 if todos.include?('c') && ckati 195 expected_failure = true 196 end 197 if todos.include?('go-ninja') && !ckati && via_ninja 198 expected_failure = true 199 end 200 if todos.include?('c-ninja') && ckati && via_ninja 201 expected_failure = true 202 end 203 if todos.include?('c-exec') && ckati && !via_ninja 204 expected_failure = true 205 end 206 if todos.include?('ninja') && via_ninja 207 expected_failure = true 208 end 209 else 210 expected_failure = true 211 end 212 end 213 214 run_in_testdir(mk) do |name| 215 File.open("Makefile", 'w') do |ofile| 216 ofile.print(c) 217 end 218 File.symlink('../../testcase/submake', 'submake') 219 220 expected = '' 221 output = '' 222 223 testcases = c.scan(/^test\d*/).sort.uniq 224 if testcases.empty? 225 testcases = [''] 226 end 227 228 is_silent_test = mk =~ /\/submake_/ 229 230 cleanup 231 testcases.each do |tc| 232 cmd = 'make' 233 if via_ninja || is_silent_test 234 cmd += ' -s' 235 end 236 cmd += bash_var 237 cmd += " #{tc} 2>&1" 238 res = IO.popen(cmd, 'r:binary', &:read) 239 res = normalize_make_log(res, mk, via_ninja) 240 expected += "=== #{tc} ===\n" + res 241 expected_files = get_output_filenames 242 expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" 243 end 244 245 cleanup 246 testcases.each do |tc| 247 json = "#{tc.empty? ? 'test' : tc}" 248 cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator" 249 if ckati 250 cmd = "../../ckati --use_find_emulator" 251 end 252 if via_ninja 253 cmd += ' --ninja' 254 end 255 if gen_all_targets 256 if !ckati || !via_ninja 257 raise "-a should be used with -c -n" 258 end 259 cmd += ' --gen_all_targets' 260 end 261 if is_silent_test 262 cmd += ' -s' 263 end 264 cmd += bash_var 265 if !gen_all_targets || mk =~ /makecmdgoals/ 266 cmd += " #{tc}" 267 end 268 cmd += " 2>&1" 269 res = IO.popen(cmd, 'r:binary', &:read) 270 if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh') 271 cmd = './ninja.sh -j1 -v' 272 if gen_all_targets 273 cmd += " #{tc}" 274 end 275 cmd += ' 2>&1' 276 log = IO.popen(cmd, 'r:binary', &:read) 277 res += normalize_ninja_log(log, mk) 278 end 279 res = normalize_kati_log(res) 280 output += "=== #{tc} ===\n" + res 281 output_files = get_output_filenames 282 output += "\n=== FILES ===\n#{output_files * "\n"}\n" 283 end 284 285 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 286 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 287 288 if expected =~ /FAIL/ 289 puts %Q(#{name} has a string "FAIL" in its expectation) 290 exit 1 291 end 292 293 if expected != output 294 if expected_failure 295 puts "#{name}: FAIL (expected)" 296 expected_failures << name 297 else 298 puts "#{name}: FAIL" 299 failures << name 300 end 301 if !expected_failure || show_failing 302 puts `diff -u out.make out.kati` 303 end 304 else 305 if expected_failure 306 puts "#{name}: PASS (unexpected)" 307 unexpected_passes << name 308 else 309 puts "#{name}: PASS" 310 passes << name 311 end 312 end 313 314 if name !~ /^err_/ && test_serialization && !expected_failure 315 testcases.each do |tc| 316 json = "#{tc.empty? ? 'test' : tc}" 317 cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1" 318 res = IO.popen(cmd, 'r:binary', &:read) 319 if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") 320 puts "#{name}##{json}: Serialize failure (not exist)" 321 puts res 322 else 323 json1 = File.read("#{json}.json") 324 json2 = File.read("#{json}_2.json") 325 if json1 != json2 326 puts "#{name}##{json}: Serialize failure" 327 puts res 328 end 329 end 330 end 331 end 332 end 333 end 334 335 run_shell_test = proc do |sh| 336 is_ninja_test = sh =~ /\/ninja_/ 337 if is_ninja_test && (!ckati || !via_ninja) 338 next 339 end 340 341 run_in_testdir(sh) do |name| 342 cleanup 343 cmd = "sh ../../#{sh} make" 344 if is_ninja_test 345 cmd += ' -s' 346 end 347 cmd += bash_var 348 expected = IO.popen(cmd, 'r:binary', &:read) 349 cleanup 350 351 if is_ninja_test 352 if ckati 353 cmd = "sh ../../#{sh} ../../ckati --ninja --regen" 354 else 355 next 356 end 357 else 358 if ckati 359 cmd = "sh ../../#{sh} ../../ckati" 360 else 361 cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=." 362 end 363 end 364 cmd += bash_var 365 366 output = IO.popen(cmd, 'r:binary', &:read) 367 368 expected = normalize_make_log(expected, sh, is_ninja_test) 369 output = normalize_kati_log(output) 370 if is_ninja_test 371 output = normalize_ninja_log(output, sh) 372 end 373 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 374 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 375 376 if expected != output 377 puts "#{name}: FAIL" 378 puts `diff -u out.make out.kati` 379 failures << name 380 else 381 puts "#{name}: PASS" 382 passes << name 383 end 384 end 385 end 386 387 test_files.each do |test| 388 if /\.mk$/ =~ test 389 run_make_test.call(test) 390 elsif /\.sh$/ =~ test 391 run_shell_test.call(test) 392 else 393 raise "Unknown test type: #{test}" 394 end 395 end 396 397 puts 398 399 if !expected_failures.empty? 400 puts "=== Expected failures ===" 401 expected_failures.each do |n| 402 puts n 403 end 404 end 405 406 if !unexpected_passes.empty? 407 puts "=== Unexpected passes ===" 408 unexpected_passes.each do |n| 409 puts n 410 end 411 end 412 413 if !failures.empty? 414 puts "=== Failures ===" 415 failures.each do |n| 416 puts n 417 end 418 end 419 420 puts 421 422 if !unexpected_passes.empty? || !failures.empty? 423 puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)" 424 exit 1 425 else 426 puts 'PASS!' 427 end 428
