1 -- Copyright 2011 the V8 project authors. All rights reserved. 2 -- Redistribution and use in source and binary forms, with or without 3 -- modification, are permitted provided that the following conditions are 4 -- met: 5 -- 6 -- * Redistributions of source code must retain the above copyright 7 -- notice, this list of conditions and the following disclaimer. 8 -- * Redistributions in binary form must reproduce the above 9 -- copyright notice, this list of conditions and the following 10 -- disclaimer in the documentation and/or other materials provided 11 -- with the distribution. 12 -- * Neither the name of Google Inc. nor the names of its 13 -- contributors may be used to endorse or promote products derived 14 -- from this software without specific prior written permission. 15 -- 16 -- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 -- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 -- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 -- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 -- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 -- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 -- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 -- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 -- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 -- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 -- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 -- This is main driver for gcmole tool. See README for more details. 29 -- Usage: CLANG_BIN=clang-bin-dir lua tools/gcmole/gcmole.lua [arm|ia32|x64] 30 31 local DIR = arg[0]:match("^(.+)/[^/]+$") 32 33 local FLAGS = { 34 -- Do not build gcsuspects file and reuse previously generated one. 35 reuse_gcsuspects = false; 36 37 -- Don't use parallel python runner. 38 sequential = false; 39 40 -- Print commands to console before executing them. 41 verbose = false; 42 43 -- Perform dead variable analysis (generates many false positives). 44 -- TODO add some sort of whiteliste to filter out false positives. 45 dead_vars = false; 46 47 -- When building gcsuspects whitelist certain functions as if they 48 -- can be causing GC. Currently used to reduce number of false 49 -- positives in dead variables analysis. See TODO for WHITELIST 50 -- below. 51 whitelist = true; 52 } 53 local ARGS = {} 54 55 for i = 1, #arg do 56 local flag = arg[i]:match "^%-%-([%w_-]+)$" 57 if flag then 58 local no, real_flag = flag:match "^(no)([%w_-]+)$" 59 if real_flag then flag = real_flag end 60 61 flag = flag:gsub("%-", "_") 62 if FLAGS[flag] ~= nil then 63 FLAGS[flag] = (no ~= "no") 64 else 65 error("Unknown flag: " .. flag) 66 end 67 else 68 table.insert(ARGS, arg[i]) 69 end 70 end 71 72 local ARCHS = ARGS[1] and { ARGS[1] } or { 'ia32', 'arm', 'x64', 'arm64' } 73 74 local io = require "io" 75 local os = require "os" 76 77 function log(...) 78 io.stderr:write(string.format(...)) 79 io.stderr:write "\n" 80 end 81 82 ------------------------------------------------------------------------------- 83 -- Clang invocation 84 85 local CLANG_BIN = os.getenv "CLANG_BIN" 86 local CLANG_PLUGINS = os.getenv "CLANG_PLUGINS" 87 88 if not CLANG_BIN or CLANG_BIN == "" then 89 error "CLANG_BIN not set" 90 end 91 92 if not CLANG_PLUGINS or CLANG_PLUGINS == "" then 93 CLANG_PLUGINS = DIR 94 end 95 96 local function MakeClangCommandLine( 97 plugin, plugin_args, triple, arch_define, arch_options) 98 if plugin_args then 99 for i = 1, #plugin_args do 100 plugin_args[i] = "-Xclang -plugin-arg-" .. plugin 101 .. " -Xclang " .. plugin_args[i] 102 end 103 plugin_args = " " .. table.concat(plugin_args, " ") 104 end 105 return CLANG_BIN .. "/clang++ -std=c++11 -c " 106 .. " -Xclang -load -Xclang " .. CLANG_PLUGINS .. "/libgcmole.so" 107 .. " -Xclang -plugin -Xclang " .. plugin 108 .. (plugin_args or "") 109 .. " -Xclang -triple -Xclang " .. triple 110 .. " -D" .. arch_define 111 .. " -DENABLE_DEBUGGER_SUPPORT" 112 .. " -DV8_I18N_SUPPORT" 113 .. " -I./" 114 .. " -Iinclude/" 115 .. " -Ithird_party/icu/source/common" 116 .. " -Ithird_party/icu/source/i18n" 117 .. " " .. arch_options 118 end 119 120 local function IterTable(t) 121 return coroutine.wrap(function () 122 for i, v in ipairs(t) do 123 coroutine.yield(v) 124 end 125 end) 126 end 127 128 local function SplitResults(lines, func) 129 -- Splits the output of parallel.py and calls func on each result. 130 -- Bails out in case of an error in one of the executions. 131 local current = {} 132 local filename = "" 133 for line in lines do 134 local new_file = line:match "^______________ (.*)$" 135 local code = line:match "^______________ finish (%d+) ______________$" 136 if code then 137 if tonumber(code) > 0 then 138 log(table.concat(current, "\n")) 139 log("Failed to examine " .. filename) 140 return false 141 end 142 log("-- %s", filename) 143 func(filename, IterTable(current)) 144 elseif new_file then 145 filename = new_file 146 current = {} 147 else 148 table.insert(current, line) 149 end 150 end 151 return true 152 end 153 154 function InvokeClangPluginForEachFile(filenames, cfg, func) 155 local cmd_line = MakeClangCommandLine(cfg.plugin, 156 cfg.plugin_args, 157 cfg.triple, 158 cfg.arch_define, 159 cfg.arch_options) 160 if FLAGS.sequential then 161 log("** Sequential execution.") 162 for _, filename in ipairs(filenames) do 163 log("-- %s", filename) 164 local action = cmd_line .. " " .. filename .. " 2>&1" 165 if FLAGS.verbose then print('popen ', action) end 166 local pipe = io.popen(action) 167 func(filename, pipe:lines()) 168 local success = pipe:close() 169 if not success then error("Failed to run: " .. action) end 170 end 171 else 172 log("** Parallel execution.") 173 local action = "python tools/gcmole/parallel.py \"" 174 .. cmd_line .. "\" " .. table.concat(filenames, " ") 175 if FLAGS.verbose then print('popen ', action) end 176 local pipe = io.popen(action) 177 local success = SplitResults(pipe:lines(), func) 178 local closed = pipe:close() 179 if not (success and closed) then error("Failed to run: " .. action) end 180 end 181 end 182 183 ------------------------------------------------------------------------------- 184 -- GYP file parsing 185 186 -- TODO(machenbach): Remove this when deprecating gyp. 187 local function ParseGYPFile() 188 local result = {} 189 local gyp_files = { 190 { "src/v8.gyp", "'([^']-%.cc)'", "src/" }, 191 { "test/cctest/cctest.gyp", "'(test-[^']-%.cc)'", "test/cctest/" } 192 } 193 194 for i = 1, #gyp_files do 195 local filename = gyp_files[i][1] 196 local pattern = gyp_files[i][2] 197 local prefix = gyp_files[i][3] 198 local gyp_file = assert(io.open(filename), "failed to open GYP file") 199 local gyp = gyp_file:read('*a') 200 for condition, sources in 201 gyp:gmatch "%[.-### gcmole%((.-)%) ###(.-)%]" do 202 if result[condition] == nil then result[condition] = {} end 203 for file in sources:gmatch(pattern) do 204 table.insert(result[condition], prefix .. file) 205 end 206 end 207 gyp_file:close() 208 end 209 210 return result 211 end 212 213 local function ParseGNFile() 214 local result = {} 215 local gn_files = { 216 { "BUILD.gn", '"([^"]-%.cc)"', "" }, 217 { "test/cctest/BUILD.gn", '"(test-[^"]-%.cc)"', "test/cctest/" } 218 } 219 220 for i = 1, #gn_files do 221 local filename = gn_files[i][1] 222 local pattern = gn_files[i][2] 223 local prefix = gn_files[i][3] 224 local gn_file = assert(io.open(filename), "failed to open GN file") 225 local gn = gn_file:read('*a') 226 for condition, sources in 227 gn:gmatch "### gcmole%((.-)%) ###(.-)%]" do 228 if result[condition] == nil then result[condition] = {} end 229 for file in sources:gmatch(pattern) do 230 table.insert(result[condition], prefix .. file) 231 end 232 end 233 gn_file:close() 234 end 235 236 return result 237 end 238 239 local function EvaluateCondition(cond, props) 240 if cond == 'all' then return true end 241 242 local p, v = cond:match "(%w+):(%w+)" 243 244 assert(p and v, "failed to parse condition: " .. cond) 245 assert(props[p] ~= nil, "undefined configuration property: " .. p) 246 247 return props[p] == v 248 end 249 250 local function BuildFileList(sources, props) 251 local list = {} 252 for condition, files in pairs(sources) do 253 if EvaluateCondition(condition, props) then 254 for i = 1, #files do table.insert(list, files[i]) end 255 end 256 end 257 return list 258 end 259 260 261 local gyp_sources = ParseGYPFile() 262 local gn_sources = ParseGNFile() 263 264 -- TODO(machenbach): Remove this comparison logic when deprecating gyp. 265 local function CompareSources(sources1, sources2, what) 266 for condition, files1 in pairs(sources1) do 267 local files2 = sources2[condition] 268 assert( 269 files2 ~= nil, 270 "Missing gcmole condition in " .. what .. ": " .. condition) 271 272 -- Turn into set for speed. 273 files2_set = {} 274 for i, file in pairs(files2) do files2_set[file] = true end 275 276 for i, file in pairs(files1) do 277 assert( 278 files2_set[file] ~= nil, 279 "Missing file " .. file .. " in " .. what .. " for condition " .. 280 condition) 281 end 282 end 283 end 284 285 CompareSources(gyp_sources, gn_sources, "GN") 286 CompareSources(gn_sources, gyp_sources, "GYP") 287 288 289 local function FilesForArch(arch) 290 return BuildFileList(gn_sources, { os = 'linux', 291 arch = arch, 292 mode = 'debug', 293 simulator = ''}) 294 end 295 296 local mtConfig = {} 297 298 mtConfig.__index = mtConfig 299 300 local function config (t) return setmetatable(t, mtConfig) end 301 302 function mtConfig:extend(t) 303 local e = {} 304 for k, v in pairs(self) do e[k] = v end 305 for k, v in pairs(t) do e[k] = v end 306 return config(e) 307 end 308 309 local ARCHITECTURES = { 310 ia32 = config { triple = "i586-unknown-linux", 311 arch_define = "V8_TARGET_ARCH_IA32", 312 arch_options = "-m32" }, 313 arm = config { triple = "i586-unknown-linux", 314 arch_define = "V8_TARGET_ARCH_ARM", 315 arch_options = "-m32" }, 316 x64 = config { triple = "x86_64-unknown-linux", 317 arch_define = "V8_TARGET_ARCH_X64", 318 arch_options = "" }, 319 arm64 = config { triple = "x86_64-unknown-linux", 320 arch_define = "V8_TARGET_ARCH_ARM64", 321 arch_options = "" }, 322 } 323 324 ------------------------------------------------------------------------------- 325 -- GCSuspects Generation 326 327 local gc, gc_caused, funcs 328 329 local WHITELIST = { 330 -- The following functions call CEntryStub which is always present. 331 "MacroAssembler.*CallExternalReference", 332 "MacroAssembler.*CallRuntime", 333 "CompileCallLoadPropertyWithInterceptor", 334 "CallIC.*GenerateMiss", 335 336 -- DirectCEntryStub is a special stub used on ARM. 337 -- It is pinned and always present. 338 "DirectCEntryStub.*GenerateCall", 339 340 -- TODO GCMole currently is sensitive enough to understand that certain 341 -- functions only cause GC and return Failure simulataneously. 342 -- Callsites of such functions are safe as long as they are properly 343 -- check return value and propagate the Failure to the caller. 344 -- It should be possible to extend GCMole to understand this. 345 "Heap.*AllocateFunctionPrototype", 346 347 -- Ignore all StateTag methods. 348 "StateTag", 349 350 -- Ignore printing of elements transition. 351 "PrintElementsTransition" 352 }; 353 354 local function AddCause(name, cause) 355 local t = gc_caused[name] 356 if not t then 357 t = {} 358 gc_caused[name] = t 359 end 360 table.insert(t, cause) 361 end 362 363 local function resolve(name) 364 local f = funcs[name] 365 366 if not f then 367 f = {} 368 funcs[name] = f 369 370 if name:match "Collect.*Garbage" then 371 gc[name] = true 372 AddCause(name, "<GC>") 373 end 374 375 if FLAGS.whitelist then 376 for i = 1, #WHITELIST do 377 if name:match(WHITELIST[i]) then 378 gc[name] = false 379 end 380 end 381 end 382 end 383 384 return f 385 end 386 387 local function parse (filename, lines) 388 local scope 389 390 for funcname in lines do 391 if funcname:sub(1, 1) ~= '\t' then 392 resolve(funcname) 393 scope = funcname 394 else 395 local name = funcname:sub(2) 396 resolve(name)[scope] = true 397 end 398 end 399 end 400 401 local function propagate () 402 log "** Propagating GC information" 403 404 local function mark(from, callers) 405 for caller, _ in pairs(callers) do 406 if gc[caller] == nil then 407 gc[caller] = true 408 mark(caller, funcs[caller]) 409 end 410 AddCause(caller, from) 411 end 412 end 413 414 for funcname, callers in pairs(funcs) do 415 if gc[funcname] then mark(funcname, callers) end 416 end 417 end 418 419 local function GenerateGCSuspects(arch, files, cfg) 420 -- Reset the global state. 421 gc, gc_caused, funcs = {}, {}, {} 422 423 log ("** Building GC Suspects for %s", arch) 424 InvokeClangPluginForEachFile (files, 425 cfg:extend { plugin = "dump-callees" }, 426 parse) 427 428 propagate() 429 430 local out = assert(io.open("gcsuspects", "w")) 431 for name, value in pairs(gc) do if value then out:write (name, '\n') end end 432 out:close() 433 434 local out = assert(io.open("gccauses", "w")) 435 out:write "GC = {" 436 for name, causes in pairs(gc_caused) do 437 out:write("['", name, "'] = {") 438 for i = 1, #causes do out:write ("'", causes[i], "';") end 439 out:write("};\n") 440 end 441 out:write "}" 442 out:close() 443 444 log ("** GCSuspects generated for %s", arch) 445 end 446 447 -------------------------------------------------------------------------------- 448 -- Analysis 449 450 local function CheckCorrectnessForArch(arch) 451 local files = FilesForArch(arch) 452 local cfg = ARCHITECTURES[arch] 453 454 if not FLAGS.reuse_gcsuspects then 455 GenerateGCSuspects(arch, files, cfg) 456 end 457 458 local processed_files = 0 459 local errors_found = false 460 local function SearchForErrors(filename, lines) 461 processed_files = processed_files + 1 462 for l in lines do 463 errors_found = errors_found or 464 l:match "^[^:]+:%d+:%d+:" or 465 l:match "error" or 466 l:match "warning" 467 print(l) 468 end 469 end 470 471 log("** Searching for evaluation order problems%s for %s", 472 FLAGS.dead_vars and " and dead variables" or "", 473 arch) 474 local plugin_args 475 if FLAGS.dead_vars then plugin_args = { "--dead-vars" } end 476 InvokeClangPluginForEachFile(files, 477 cfg:extend { plugin = "find-problems", 478 plugin_args = plugin_args }, 479 SearchForErrors) 480 log("** Done processing %d files. %s", 481 processed_files, 482 errors_found and "Errors found" or "No errors found") 483 484 return errors_found 485 end 486 487 local function SafeCheckCorrectnessForArch(arch) 488 local status, errors = pcall(CheckCorrectnessForArch, arch) 489 if not status then 490 print(string.format("There was an error: %s", errors)) 491 errors = true 492 end 493 return errors 494 end 495 496 local errors = false 497 498 for _, arch in ipairs(ARCHS) do 499 if not ARCHITECTURES[arch] then 500 error ("Unknown arch: " .. arch) 501 end 502 503 errors = SafeCheckCorrectnessForArch(arch, report) or errors 504 end 505 506 os.exit(errors and 1 or 0) 507
