1 #!/usr/bin/env ruby 2 # 3 # Copyright 2015 Google Inc. All rights reserved 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 require 'fileutils' 18 19 while true 20 if ARGV[0] == '-s' 21 test_serialization = true 22 ARGV.shift 23 elsif ARGV[0] == '-c' 24 ckati = true 25 ARGV.shift 26 ENV['KATI_VARIANT'] = 'c' 27 elsif ARGV[0] == '-n' 28 via_ninja = true 29 ARGV.shift 30 ENV['NINJA_STATUS'] = 'NINJACMD: ' 31 elsif ARGV[0] == '-a' 32 gen_all_targets = true 33 ARGV.shift 34 elsif ARGV[0] == '-v' 35 show_failing = true 36 ARGV.shift 37 else 38 break 39 end 40 end 41 42 def get_output_filenames 43 files = Dir.glob('*') 44 files.delete('Makefile') 45 files.delete('build.ninja') 46 files.delete('env.sh') 47 files.delete('ninja.sh') 48 files.delete('gmon.out') 49 files.delete('submake') 50 files.reject!{|f|f =~ /\.json$/} 51 files.reject!{|f|f =~ /^kati\.*/} 52 files 53 end 54 55 def cleanup 56 (get_output_filenames + Dir.glob('.*')).each do |fname| 57 next if fname == '.' || fname == '..' 58 FileUtils.rm_rf fname 59 end 60 end 61 62 def move_circular_dep(l) 63 # We don't care when circular dependency detection happens. 64 circ = '' 65 while l.sub!(/Circular .* dropped\.\n/, '') do 66 circ += $& 67 end 68 circ + l 69 end 70 71 expected_failures = [] 72 unexpected_passes = [] 73 failures = [] 74 passes = [] 75 76 if !ARGV.empty? 77 test_files = ARGV.map do |test| 78 "testcase/#{File.basename(test)}" 79 end 80 else 81 test_files = Dir.glob('testcase/*.mk').sort 82 test_files += Dir.glob('testcase/*.sh').sort 83 end 84 85 def run_in_testdir(test_filename) 86 c = File.read(test_filename) 87 name = File.basename(test_filename) 88 dir = "out/#{name}" 89 90 FileUtils.mkdir_p(dir) 91 Dir.glob("#{dir}/*").each do |fname| 92 FileUtils.rm_rf(fname) 93 end 94 95 Dir.chdir(dir) do 96 yield name 97 end 98 end 99 100 def normalize_ninja_log(log, mk) 101 log.gsub!(/^NINJACMD: .*\n/, '') 102 log.gsub!(/^ninja: no work to do\.\n/, '') 103 log.gsub!(/^ninja: error: (.*, needed by .*),.*/, 104 '*** No rule to make target \\1.') 105 log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/, 106 'ninja: warning: multiple rules generate \\1.') 107 if mk =~ /err_error_in_recipe.mk/ 108 # This test expects ninja fails. Strip ninja specific error logs. 109 log.gsub!(/^FAILED: .*\n/, '') 110 log.gsub!(/^ninja: .*\n/, '') 111 elsif mk =~ /\/fail_/ 112 # Recipes in these tests fail. 113 log.gsub!(/^FAILED: .*/, '*** [test] Error 1') 114 log.gsub!(/^ninja: .*\n/, '') 115 end 116 log 117 end 118 119 def normalize_make_log(expected, mk, via_ninja) 120 expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') 121 expected.gsub!(/^make(?:\[\d+\])?: /, '') 122 expected = move_circular_dep(expected) 123 124 # Normalizations for old/new GNU make. 125 expected.gsub!(/[`'"]/, '"') 126 expected.gsub!(/ (?:commands|recipe) for target /, 127 ' commands for target ') 128 expected.gsub!(/ (?:commands|recipe) commences /, 129 ' commands commence ') 130 expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') 131 expected.gsub!('Extraneous text after', 'extraneous text after') 132 # Not sure if this is useful. 133 expected.gsub!(/\s+Stop\.$/, '') 134 # GNU make 4.0 has this output. 135 expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') 136 # We treat some warnings as errors. 137 expected.gsub!(/^\/bin\/sh: line 0: /, '') 138 # We print out some ninja warnings in some tests to match what we expect 139 # ninja to produce. Remove them if we're not testing ninja. 140 if !via_ninja 141 expected.gsub!(/^ninja: warning: .*\n/, '') 142 end 143 # Normalization for "include foo" with C++ kati. 144 expected.gsub!(/(: )(\S+): (No such file or directory)\n\*\*\* No rule to make target "\2"./, '\1\2: \3') 145 146 expected 147 end 148 149 def normalize_kati_log(output) 150 output = move_circular_dep(output) 151 # kati specific log messages. 152 output.gsub!(/^\*kati\*.*\n/, '') 153 output.gsub!(/^c?kati: /, '') 154 output.gsub!(/[`'"]/, '"') 155 output.gsub!(/\/bin\/sh: ([^:]*): command not found/, 156 "\\1: Command not found") 157 output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '') 158 output.gsub!(/^FindEmulator: /, '') 159 output.gsub!(/^\/bin\/sh: line 0: /, '') 160 output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk 161 output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk 162 # Normalization for "include foo" with Go kati. 163 output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/, 164 "\\1\\2: N\\3") 165 output 166 end 167 168 run_make_test = proc do |mk| 169 c = File.read(mk) 170 expected_failure = false 171 if c =~ /\A# TODO(?:\(([-a-z|]+)\))?/ 172 if $1 173 todos = $1.split('|') 174 if todos.include?('go') && !ckati 175 expected_failure = true 176 end 177 if todos.include?('c') && ckati 178 expected_failure = true 179 end 180 if todos.include?('go-ninja') && !ckati && via_ninja 181 expected_failure = true 182 end 183 if todos.include?('c-ninja') && ckati && via_ninja 184 expected_failure = true 185 end 186 if todos.include?('ninja') && via_ninja 187 expected_failure = true 188 end 189 else 190 expected_failure = true 191 end 192 end 193 194 run_in_testdir(mk) do |name| 195 File.open("Makefile", 'w') do |ofile| 196 ofile.print(c) 197 end 198 File.symlink('../../testcase/submake', 'submake') 199 200 expected = '' 201 output = '' 202 203 testcases = c.scan(/^test\d*/).sort.uniq 204 if testcases.empty? 205 testcases = [''] 206 end 207 208 is_silent_test = mk =~ /\/submake_/ 209 210 cleanup 211 testcases.each do |tc| 212 cmd = 'make' 213 if via_ninja || is_silent_test 214 cmd += ' -s' 215 end 216 cmd += " #{tc} 2>&1" 217 res = `#{cmd}` 218 res = normalize_make_log(res, mk, via_ninja) 219 expected += "=== #{tc} ===\n" + res 220 expected_files = get_output_filenames 221 expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" 222 end 223 224 cleanup 225 testcases.each do |tc| 226 json = "#{tc.empty? ? 'test' : tc}" 227 cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator" 228 if ckati 229 cmd = "../../ckati --use_find_emulator" 230 end 231 if via_ninja 232 cmd += ' --ninja' 233 end 234 if gen_all_targets 235 if !ckati || !via_ninja 236 raise "-a should be used with -c -n" 237 end 238 cmd += ' --gen_all_targets' 239 end 240 if is_silent_test 241 cmd += ' -s' 242 end 243 if !gen_all_targets || mk =~ /makecmdgoals/ 244 cmd += " #{tc}" 245 end 246 cmd += " 2>&1" 247 res = IO.popen(cmd, 'r:binary', &:read) 248 if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh') 249 cmd = './ninja.sh -j1 -v' 250 if gen_all_targets 251 cmd += " #{tc}" 252 end 253 cmd += ' 2>&1' 254 log = IO.popen(cmd, 'r:binary', &:read) 255 res += normalize_ninja_log(log, mk) 256 end 257 res = normalize_kati_log(res) 258 output += "=== #{tc} ===\n" + res 259 output_files = get_output_filenames 260 output += "\n=== FILES ===\n#{output_files * "\n"}\n" 261 end 262 263 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 264 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 265 266 if expected =~ /FAIL/ 267 puts %Q(#{name} has a string "FAIL" in its expectation) 268 exit 1 269 end 270 271 if expected != output 272 if expected_failure 273 puts "#{name}: FAIL (expected)" 274 expected_failures << name 275 else 276 puts "#{name}: FAIL" 277 failures << name 278 end 279 if !expected_failure || show_failing 280 puts `diff -u out.make out.kati` 281 end 282 else 283 if expected_failure 284 puts "#{name}: PASS (unexpected)" 285 unexpected_passes << name 286 else 287 puts "#{name}: PASS" 288 passes << name 289 end 290 end 291 292 if name !~ /^err_/ && test_serialization && !expected_failure 293 testcases.each do |tc| 294 json = "#{tc.empty? ? 'test' : tc}" 295 cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1" 296 res = IO.popen(cmd, 'r:binary', &:read) 297 if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") 298 puts "#{name}##{json}: Serialize failure (not exist)" 299 puts res 300 else 301 json1 = File.read("#{json}.json") 302 json2 = File.read("#{json}_2.json") 303 if json1 != json2 304 puts "#{name}##{json}: Serialize failure" 305 puts res 306 end 307 end 308 end 309 end 310 end 311 end 312 313 run_shell_test = proc do |sh| 314 is_ninja_test = sh =~ /\/ninja_/ 315 if is_ninja_test && (!ckati || !via_ninja) 316 next 317 end 318 319 run_in_testdir(sh) do |name| 320 cleanup 321 cmd = "sh ../../#{sh} make" 322 if is_ninja_test 323 cmd += ' -s' 324 end 325 expected = IO.popen(cmd, 'r:binary', &:read) 326 cleanup 327 328 if is_ninja_test 329 if ckati 330 cmd = "sh ../../#{sh} ../../ckati --ninja --regen" 331 else 332 next 333 end 334 else 335 if ckati 336 cmd = "sh ../../#{sh} ../../ckati" 337 else 338 cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=." 339 end 340 end 341 342 output = IO.popen(cmd, 'r:binary', &:read) 343 344 expected = normalize_make_log(expected, sh, is_ninja_test) 345 output = normalize_kati_log(output) 346 if is_ninja_test 347 output = normalize_ninja_log(output, sh) 348 end 349 File.open('out.make', 'w'){|ofile|ofile.print(expected)} 350 File.open('out.kati', 'w'){|ofile|ofile.print(output)} 351 352 if expected != output 353 puts "#{name}: FAIL" 354 puts `diff -u out.make out.kati` 355 failures << name 356 else 357 puts "#{name}: PASS" 358 passes << name 359 end 360 end 361 end 362 363 test_files.each do |test| 364 if /\.mk$/ =~ test 365 run_make_test.call(test) 366 elsif /\.sh$/ =~ test 367 run_shell_test.call(test) 368 else 369 raise "Unknown test type: #{test}" 370 end 371 end 372 373 puts 374 375 if !expected_failures.empty? 376 puts "=== Expected failures ===" 377 expected_failures.each do |n| 378 puts n 379 end 380 end 381 382 if !unexpected_passes.empty? 383 puts "=== Unexpected passes ===" 384 unexpected_passes.each do |n| 385 puts n 386 end 387 end 388 389 if !failures.empty? 390 puts "=== Failures ===" 391 failures.each do |n| 392 puts n 393 end 394 end 395 396 puts 397 398 if !unexpected_passes.empty? || !failures.empty? 399 puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)" 400 exit 1 401 else 402 puts 'PASS!' 403 end 404