1 // Copyright 2017 syzkaller project authors. All rights reserved. 2 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 4 package report 5 6 import ( 7 "bufio" 8 "bytes" 9 "fmt" 10 "net/mail" 11 "path/filepath" 12 "regexp" 13 "strconv" 14 "strings" 15 "time" 16 17 "github.com/google/syzkaller/pkg/osutil" 18 "github.com/google/syzkaller/pkg/symbolizer" 19 ) 20 21 type linux struct { 22 kernelSrc string 23 kernelObj string 24 vmlinux string 25 symbols map[string][]symbolizer.Symbol 26 ignores []*regexp.Regexp 27 consoleOutputRe *regexp.Regexp 28 questionableRe *regexp.Regexp 29 guiltyFileBlacklist []*regexp.Regexp 30 reportStartIgnores [][]byte 31 infoMessagesWithStack [][]byte 32 eoi []byte 33 } 34 35 func ctorLinux(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { 36 vmlinux := "" 37 var symbols map[string][]symbolizer.Symbol 38 if kernelObj != "" { 39 vmlinux = filepath.Join(kernelObj, "vmlinux") 40 var err error 41 symbols, err = symbolizer.ReadSymbols(vmlinux) 42 if err != nil { 43 return nil, nil, err 44 } 45 } 46 ctx := &linux{ 47 kernelSrc: kernelSrc, 48 kernelObj: kernelObj, 49 vmlinux: vmlinux, 50 symbols: symbols, 51 ignores: ignores, 52 } 53 ctx.consoleOutputRe = regexp.MustCompile(`^(?:\*\* [0-9]+ printk messages dropped \*\* )?(?:.* login: )?(?:\<[0-9]+\>)?\[ *[0-9]+\.[0-9]+\] `) 54 ctx.questionableRe = regexp.MustCompile(`(?:\[\<[0-9a-f]+\>\])? \? +[a-zA-Z0-9_.]+\+0x[0-9a-f]+/[0-9a-f]+`) 55 ctx.eoi = []byte("<EOI>") 56 ctx.guiltyFileBlacklist = []*regexp.Regexp{ 57 regexp.MustCompile(`.*\.h`), 58 regexp.MustCompile(`^lib/.*`), 59 regexp.MustCompile(`^virt/lib/.*`), 60 regexp.MustCompile(`^mm/kasan/.*`), 61 regexp.MustCompile(`^mm/kmsan/.*`), 62 regexp.MustCompile(`^kernel/kcov.c`), 63 regexp.MustCompile(`^mm/sl.b.c`), 64 regexp.MustCompile(`^mm/percpu.*`), 65 regexp.MustCompile(`^mm/vmalloc.c`), 66 regexp.MustCompile(`^mm/page_alloc.c`), 67 regexp.MustCompile(`^mm/util.c`), 68 regexp.MustCompile(`^kernel/rcu/.*`), 69 regexp.MustCompile(`^arch/.*/kernel/traps.c`), 70 regexp.MustCompile(`^arch/.*/mm/fault.c`), 71 regexp.MustCompile(`^kernel/locking/.*`), 72 regexp.MustCompile(`^kernel/panic.c`), 73 regexp.MustCompile(`^kernel/softirq.c`), 74 regexp.MustCompile(`^kernel/kthread.c`), 75 regexp.MustCompile(`^kernel/sched/.*.c`), 76 regexp.MustCompile(`^kernel/time/timer.c`), 77 regexp.MustCompile(`^kernel/workqueue.c`), 78 regexp.MustCompile(`^net/core/dev.c`), 79 regexp.MustCompile(`^net/core/sock.c`), 80 regexp.MustCompile(`^net/core/skbuff.c`), 81 regexp.MustCompile(`^fs/proc/generic.c`), 82 } 83 // These pattern do _not_ start a new report, i.e. can be in a middle of another report. 84 ctx.reportStartIgnores = [][]byte{ 85 []byte("invalid opcode: 0000"), 86 []byte("Kernel panic - not syncing: panic_on_warn set"), 87 []byte("unregister_netdevice: waiting for"), 88 } 89 // These pattern math kernel reports which are not bugs in itself but contain stack traces. 90 // If we see them in the middle of another report, we know that the report is potentially corrupted. 91 ctx.infoMessagesWithStack = [][]byte{ 92 []byte("vmalloc: allocation failure:"), 93 []byte("FAULT_INJECTION: forcing a failure"), 94 []byte("FAULT_FLAG_ALLOW_RETRY missing"), 95 } 96 suppressions := []string{ 97 "fatal error: runtime: out of memory", 98 "fatal error: runtime: cannot allocate memory", 99 "panic: failed to start executor binary", 100 "panic: executor failed: pthread_create failed", 101 "panic: failed to create temp dir", 102 "fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS 103 "signal SIGBUS: bus error", // presubmably OOM turned into SIGBUS 104 "Out of memory: Kill process .* \\(syz-fuzzer\\)", 105 "Out of memory: Kill process .* \\(sshd\\)", 106 "Killed process .* \\(syz-fuzzer\\)", 107 "Killed process .* \\(sshd\\)", 108 "lowmemorykiller: Killing 'syz-fuzzer'", 109 "lowmemorykiller: Killing 'sshd'", 110 "INIT: PANIC: segmentation violation!", 111 } 112 return ctx, suppressions, nil 113 } 114 115 func (ctx *linux) ContainsCrash(output []byte) bool { 116 return containsCrash(output, linuxOopses, ctx.ignores) 117 } 118 119 func (ctx *linux) Parse(output []byte) *Report { 120 oops, startPos, endPos, logReport, consoleReport, consoleReportReliable, 121 logReportPrefix, consoleReportPrefix := ctx.parseOutput(output) 122 if oops == nil { 123 return nil 124 } 125 rep := &Report{ 126 Output: output, 127 StartPos: startPos, 128 EndPos: endPos, 129 } 130 var report []byte 131 var reportPrefix [][]byte 132 // Try extracting report from console output only. 133 title, corrupted, format := extractDescription(consoleReportReliable, oops, linuxStackParams) 134 if title != "" { 135 report = consoleReport 136 reportPrefix = consoleReportPrefix 137 } else { 138 // Failure. Try extracting report from the whole log. 139 report = logReport 140 reportPrefix = logReportPrefix 141 title, corrupted, format = extractDescription(report, oops, linuxStackParams) 142 if title == "" { 143 panic(fmt.Sprintf("non matching oops for %q in:\n%s\n\nconsole:\n%s\n"+ 144 "output [range:%v-%v]:\n%s\n", 145 oops.header, report, consoleReportReliable, 146 rep.StartPos, rep.StartPos+len(report), output)) 147 } 148 } 149 rep.Title = title 150 rep.Corrupted = corrupted != "" 151 rep.CorruptedReason = corrupted 152 // Prepend 5 lines preceding start of the report, 153 // they can contain additional info related to the report. 154 for _, prefix := range reportPrefix { 155 rep.Report = append(rep.Report, prefix...) 156 rep.Report = append(rep.Report, '\n') 157 } 158 rep.Report = append(rep.Report, report...) 159 if !rep.Corrupted { 160 rep.Corrupted, rep.CorruptedReason = ctx.isCorrupted(title, report, format) 161 } 162 return rep 163 } 164 165 // Yes, it is complex, but all state and logic are tightly coupled. It's unclear how to simplify it. 166 // nolint: gocyclo 167 func (ctx *linux) parseOutput(output []byte) ( 168 oops *oops, startPos, endPos int, 169 logReport, consoleReport, consoleReportReliable []byte, 170 logReportPrefix, consoleReportPrefix [][]byte) { 171 firstReportEnd := 0 172 secondReportPos := 0 173 textLines := 0 174 skipText := false 175 for pos := 0; pos < len(output); { 176 next := bytes.IndexByte(output[pos:], '\n') 177 if next != -1 { 178 next += pos 179 } else { 180 next = len(output) 181 } 182 line := output[pos:next] 183 for _, oops1 := range linuxOopses { 184 match := matchOops(line, oops1, ctx.ignores) 185 if match == -1 { 186 if oops != nil && secondReportPos == 0 { 187 for _, pattern := range ctx.infoMessagesWithStack { 188 if bytes.Contains(line, pattern) { 189 secondReportPos = pos 190 break 191 } 192 } 193 } 194 continue 195 } 196 endPos = next 197 if oops == nil { 198 oops = oops1 199 startPos = pos 200 break 201 } else if secondReportPos == 0 { 202 ignored := false 203 for _, ignore := range ctx.reportStartIgnores { 204 if bytes.Contains(line, ignore) { 205 ignored = true 206 break 207 } 208 } 209 if !ignored { 210 secondReportPos = pos 211 } 212 } 213 } 214 if oops == nil { 215 logReportPrefix = append(logReportPrefix, append([]byte{}, line...)) 216 if len(logReportPrefix) > 5 { 217 logReportPrefix = logReportPrefix[1:] 218 } 219 } 220 if ctx.consoleOutputRe.Match(line) && 221 (!ctx.questionableRe.Match(line) || bytes.Contains(line, ctx.eoi)) { 222 lineStart := bytes.Index(line, []byte("] ")) + pos + 2 223 lineEnd := next 224 if lineEnd != 0 && output[lineEnd-1] == '\r' { 225 lineEnd-- 226 } 227 if oops == nil { 228 consoleReportPrefix = append(consoleReportPrefix, 229 append([]byte{}, output[lineStart:lineEnd]...)) 230 if len(consoleReportPrefix) > 5 { 231 consoleReportPrefix = consoleReportPrefix[1:] 232 } 233 } else { 234 textLines++ 235 ln := output[lineStart:lineEnd] 236 skipLine := skipText 237 if bytes.Contains(ln, []byte("Disabling lock debugging due to kernel taint")) { 238 skipLine = true 239 } else if textLines > 25 && 240 bytes.Contains(ln, []byte("Kernel panic - not syncing")) { 241 // If panic_on_warn set, then we frequently have 2 stacks: 242 // one for the actual report (or maybe even more than one), 243 // and then one for panic caused by panic_on_warn. This makes 244 // reports unnecessary long and the panic (current) stack 245 // is always present in the actual report. So we strip the 246 // panic message. However, we check that we have enough lines 247 // before the panic, because sometimes we have, for example, 248 // a single WARNING line without a stack and then the panic 249 // with the stack. 250 skipText = true 251 skipLine = true 252 } 253 if !skipLine { 254 consoleReport = append(consoleReport, ln...) 255 consoleReport = append(consoleReport, '\n') 256 if secondReportPos == 0 { 257 firstReportEnd = len(consoleReport) 258 } 259 } 260 } 261 } 262 pos = next + 1 263 } 264 if oops == nil { 265 return 266 } 267 if secondReportPos == 0 { 268 secondReportPos = len(output) 269 } 270 logReport = output[startPos:secondReportPos] 271 consoleReportReliable = consoleReport[:firstReportEnd] 272 return 273 } 274 275 func (ctx *linux) Symbolize(rep *Report) error { 276 if ctx.vmlinux == "" { 277 return nil 278 } 279 symbolized, err := ctx.symbolize(rep.Report) 280 if err != nil { 281 return err 282 } 283 rep.Report = symbolized 284 guiltyFile := ctx.extractGuiltyFile(rep.Report) 285 if guiltyFile != "" { 286 rep.Maintainers, err = ctx.getMaintainers(guiltyFile) 287 if err != nil { 288 return err 289 } 290 } 291 return nil 292 } 293 294 func (ctx *linux) symbolize(text []byte) ([]byte, error) { 295 symb := symbolizer.NewSymbolizer() 296 defer symb.Close() 297 strip := ctx.stripPrefix(symb) 298 var symbolized []byte 299 s := bufio.NewScanner(bytes.NewReader(text)) 300 for s.Scan() { 301 line := append([]byte{}, s.Bytes()...) 302 line = append(line, '\n') 303 line = symbolizeLine(symb.Symbolize, ctx.symbols, ctx.vmlinux, strip, line) 304 symbolized = append(symbolized, line...) 305 } 306 return symbolized, nil 307 } 308 309 func (ctx *linux) stripPrefix(symb *symbolizer.Symbolizer) string { 310 // Vmlinux may have been moved, so check if we can find debug info 311 // for some known functions and infer correct strip prefix from it. 312 knownSymbols := []struct { 313 symbol string 314 file string 315 }{ 316 {"__sanitizer_cov_trace_pc", "kernel/kcov.c"}, 317 {"__asan_load1", "mm/kasan/kasan.c"}, 318 {"start_kernel", "init/main.c"}, 319 } 320 for _, s := range knownSymbols { 321 for _, covSymb := range ctx.symbols[s.symbol] { 322 frames, _ := symb.Symbolize(ctx.vmlinux, covSymb.Addr) 323 if len(frames) > 0 { 324 file := frames[len(frames)-1].File 325 if idx := strings.Index(file, s.file); idx != -1 { 326 return file[:idx] 327 } 328 } 329 } 330 } 331 // Strip vmlinux location from all paths. 332 strip, _ := filepath.Abs(ctx.vmlinux) 333 return filepath.Dir(strip) + string(filepath.Separator) 334 } 335 336 func symbolizeLine(symbFunc func(bin string, pc uint64) ([]symbolizer.Frame, error), 337 symbols map[string][]symbolizer.Symbol, vmlinux, strip string, line []byte) []byte { 338 match := linuxSymbolizeRe.FindSubmatchIndex(line) 339 if match == nil { 340 return line 341 } 342 fn := line[match[2]:match[3]] 343 off, err := strconv.ParseUint(string(line[match[4]:match[5]]), 16, 64) 344 if err != nil { 345 return line 346 } 347 size, err := strconv.ParseUint(string(line[match[6]:match[7]]), 16, 64) 348 if err != nil { 349 return line 350 } 351 symb := symbols[string(fn)] 352 if len(symb) == 0 { 353 return line 354 } 355 var funcStart uint64 356 for _, s := range symb { 357 if funcStart == 0 || int(size) == s.Size { 358 funcStart = s.Addr 359 } 360 } 361 frames, err := symbFunc(vmlinux, funcStart+off-1) 362 if err != nil || len(frames) == 0 { 363 return line 364 } 365 var symbolized []byte 366 for _, frame := range frames { 367 file := frame.File 368 file = strings.TrimPrefix(file, strip) 369 file = strings.TrimPrefix(file, "./") 370 info := fmt.Sprintf(" %v:%v", file, frame.Line) 371 modified := append([]byte{}, line...) 372 modified = replace(modified, match[7], match[7], []byte(info)) 373 if frame.Inline { 374 end := match[7] + len(info) 375 modified = replace(modified, end, end, []byte(" [inline]")) 376 modified = replace(modified, match[2], match[7], []byte(frame.Func)) 377 } 378 symbolized = append(symbolized, modified...) 379 } 380 return symbolized 381 } 382 383 func (ctx *linux) extractGuiltyFile(report []byte) string { 384 if linuxRcuStall.Match(report) { 385 // Special case for rcu stalls. 386 // There are too many frames that we want to skip before actual guilty frames, 387 // we would need to blacklist too many files and that would be fragile. 388 // So instead we try to extract guilty file starting from the known 389 // interrupt entry point first. 390 if pos := bytes.Index(report, []byte(" apic_timer_interrupt+0x")); pos != -1 { 391 if file := ctx.extractGuiltyFileImpl(report[pos:]); file != "" { 392 return file 393 } 394 } 395 } 396 return ctx.extractGuiltyFileImpl(report) 397 } 398 399 func (ctx *linux) extractGuiltyFileImpl(report []byte) string { 400 files := ctx.extractFiles(report) 401 nextFile: 402 for _, file := range files { 403 for _, re := range ctx.guiltyFileBlacklist { 404 if re.MatchString(file) { 405 continue nextFile 406 } 407 } 408 return file 409 } 410 return "" 411 } 412 413 func (ctx *linux) getMaintainers(file string) ([]string, error) { 414 mtrs, err := ctx.getMaintainersImpl(file, false) 415 if err != nil { 416 return nil, err 417 } 418 if len(mtrs) <= 1 { 419 mtrs, err = ctx.getMaintainersImpl(file, true) 420 if err != nil { 421 return nil, err 422 } 423 } 424 return mtrs, nil 425 } 426 427 func (ctx *linux) getMaintainersImpl(file string, blame bool) ([]string, error) { 428 args := []string{"--no-n", "--no-rolestats"} 429 if blame { 430 args = append(args, "--git-blame") 431 } 432 args = append(args, file) 433 output, err := osutil.RunCmd(time.Minute, ctx.kernelSrc, filepath.FromSlash("scripts/get_maintainer.pl"), args...) 434 if err != nil { 435 return nil, err 436 } 437 lines := strings.Split(string(output), "\n") 438 var mtrs []string 439 for _, line := range lines { 440 addr, err := mail.ParseAddress(line) 441 if err != nil { 442 continue 443 } 444 mtrs = append(mtrs, addr.Address) 445 } 446 return mtrs, nil 447 } 448 449 func (ctx *linux) extractFiles(report []byte) []string { 450 matches := filenameRe.FindAll(report, -1) 451 var files []string 452 for _, match := range matches { 453 f := string(bytes.Split(match, []byte{':'})[0]) 454 files = append(files, filepath.Clean(f)) 455 } 456 return files 457 } 458 459 func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (bool, string) { 460 // Check if the report contains stack trace. 461 if !format.noStackTrace && !bytes.Contains(report, []byte("Call Trace")) && 462 !bytes.Contains(report, []byte("backtrace")) { 463 return true, "no stack trace in report" 464 } 465 // Check for common title corruptions. 466 for _, re := range linuxCorruptedTitles { 467 if re.MatchString(title) { 468 return true, "title matches corrupted regexp" 469 } 470 } 471 // When a report contains 'Call Trace', 'backtrace', 'Allocated' or 'Freed' keywords, 472 // it must also contain at least a single stack frame after each of them. 473 for _, key := range linuxStackKeywords { 474 match := key.FindSubmatchIndex(report) 475 if match == nil { 476 continue 477 } 478 frames := bytes.Split(report[match[0]:], []byte{'\n'}) 479 if len(frames) < 4 { 480 return true, "call trace is missed" 481 } 482 frames = frames[1:] 483 corrupted := true 484 // Check that at least one of the next few lines contains a frame. 485 outer: 486 for i := 0; i < 15 && i < len(frames); i++ { 487 for _, key1 := range linuxStackKeywords { 488 // Next stack trace starts. 489 if key1.Match(frames[i]) { 490 break outer 491 } 492 } 493 if bytes.Contains(frames[i], []byte("(stack is not available)")) || 494 stackFrameRe.Match(frames[i]) { 495 corrupted = false 496 break 497 } 498 } 499 if corrupted { 500 return true, "no frames in a stack trace" 501 } 502 } 503 return false, "" 504 } 505 506 var ( 507 linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) 508 stackFrameRe = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) 509 linuxRcuStall = compile("INFO: rcu_(?:preempt|sched|bh) (?:self-)?detected(?: expedited)? stall") 510 linuxRipFrame = compile(`IP: (?:(?:[0-9]+:)?(?:{{PC}} +){0,2}{{FUNC}}|[0-9]+:0x[0-9a-f]+|(?:[0-9]+:)?{{PC}} +\[< *\(null\)>\] +\(null\)|[0-9]+: +\(null\))`) 511 ) 512 513 var linuxCorruptedTitles = []*regexp.Regexp{ 514 // Sometimes timestamps get merged into the middle of report description. 515 regexp.MustCompile(`\[ *[0-9]+\.[0-9]+\]`), 516 } 517 518 var linuxStackKeywords = []*regexp.Regexp{ 519 regexp.MustCompile(`Call Trace`), 520 regexp.MustCompile(`Allocated`), 521 regexp.MustCompile(`Freed`), 522 // Match 'backtrace:', but exclude 'stack backtrace:' 523 regexp.MustCompile(`[^k] backtrace:`), 524 } 525 526 var linuxStackParams = &stackParams{ 527 stackStartRes: linuxStackKeywords, 528 frameRes: []*regexp.Regexp{ 529 compile("^ +(?:{{PC}} )?{{FUNC}}"), 530 }, 531 skipPatterns: []string{ 532 "__sanitizer", 533 "__asan", 534 "kasan", 535 "check_memory_region", 536 "print_address_description", 537 "panic", 538 "invalid_op", 539 "report_bug", 540 "fixup_bug", 541 "do_error", 542 "invalid_op", 543 "_trap", 544 "dump_stack", 545 "warn_slowpath", 546 "warn_alloc", 547 "__warn", 548 "debug_object", 549 "work_is_static_object", 550 "lockdep", 551 "perf_trace", 552 "lock_acquire", 553 "lock_release", 554 "register_lock_class", 555 "spin_lock", 556 "spin_unlock", 557 "raw_read_lock", 558 "raw_write_lock", 559 "down_read", 560 "down_write", 561 "down_read_trylock", 562 "down_write_trylock", 563 "up_read", 564 "up_write", 565 "mutex_lock", 566 "mutex_unlock", 567 "memcpy", 568 "memcmp", 569 "memset", 570 "strcmp", 571 "strcpy", 572 "strlen", 573 "copy_to_user", 574 "copy_from_user", 575 "put_user", 576 "get_user", 577 "might_fault", 578 "might_sleep", 579 "list_add", 580 "list_del", 581 "list_replace", 582 "list_move", 583 "list_splice", 584 }, 585 corruptedLines: []*regexp.Regexp{ 586 // Fault injection stacks are frequently intermixed with crash reports. 587 compile(`^ should_fail(\.[a-z]+\.[0-9]+)?\+0x`), 588 compile(`^ should_failslab(\.[a-z]+\.[0-9]+)?\+0x`), 589 }, 590 } 591 592 func warningStackFmt(skip ...string) *stackFmt { 593 return &stackFmt{ 594 // In newer kernels WARNING traps and actual stack starts after invalid_op frame, 595 // older kernels just print stack. 596 parts: []*regexp.Regexp{ 597 linuxRipFrame, 598 parseStackTrace, 599 }, 600 parts2: []*regexp.Regexp{ 601 compile("Call Trace:"), 602 parseStackTrace, 603 }, 604 skip: skip, 605 } 606 } 607 608 var linuxOopses = []*oops{ 609 { 610 []byte("BUG:"), 611 []oopsFormat{ 612 { 613 title: compile("BUG: KASAN:"), 614 report: compile("BUG: KASAN: ([a-z\\-]+) in {{FUNC}}(?:.*\\n)+?.*(Read|Write) of size (?:[0-9]+)"), 615 616 fmt: "KASAN: %[1]v %[3]v in %[4]v", 617 stack: &stackFmt{ 618 parts: []*regexp.Regexp{ 619 compile("BUG: KASAN: (?:[a-z\\-]+) in {{FUNC}}"), 620 compile("Call Trace:"), 621 parseStackTrace, 622 }, 623 }, 624 }, 625 { 626 title: compile("BUG: KASAN:"), 627 report: compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"), 628 fmt: "KASAN: invalid-free in %[2]v", 629 stack: &stackFmt{ 630 parts: []*regexp.Regexp{ 631 compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"), 632 compile("Call Trace:"), 633 parseStackTrace, 634 }, 635 skip: []string{"kmem_", "slab_", "kfree", "vunmap", "vfree"}, 636 }, 637 }, 638 { 639 title: compile("BUG: KASAN: ([a-z\\-]+) on address(?:.*\\n)+?.*(Read|Write) of size ([0-9]+)"), 640 fmt: "KASAN: %[1]v %[2]v", 641 }, 642 { 643 title: compile("BUG: KASAN: (.*)"), 644 fmt: "KASAN: %[1]v", 645 corrupted: true, 646 }, 647 { 648 title: compile("BUG: KMSAN: (.*)"), 649 fmt: "KMSAN: %[1]v", 650 }, 651 { 652 title: compile("BUG: unable to handle kernel paging request"), 653 fmt: "BUG: unable to handle kernel paging request in %[1]v", 654 stack: &stackFmt{ 655 parts: []*regexp.Regexp{ 656 linuxRipFrame, 657 compile("Call Trace:"), 658 parseStackTrace, 659 }, 660 }, 661 }, 662 { 663 title: compile("BUG: unable to handle kernel NULL pointer dereference"), 664 fmt: "BUG: unable to handle kernel NULL pointer dereference in %[1]v", 665 stack: &stackFmt{ 666 parts: []*regexp.Regexp{ 667 linuxRipFrame, 668 compile("Call Trace:"), 669 parseStackTrace, 670 }, 671 }, 672 }, 673 { 674 // Sometimes with such BUG failures, the second part of the header doesn't get printed 675 // or gets corrupted, because kernel prints it as two separate printk() calls. 676 title: compile("BUG: unable to handle kernel"), 677 fmt: "BUG: unable to handle kernel", 678 corrupted: true, 679 }, 680 { 681 title: compile("BUG: spinlock (lockup suspected|already unlocked|recursion|bad magic|wrong owner|wrong CPU)"), 682 fmt: "BUG: spinlock %[1]v in %[2]v", 683 stack: &stackFmt{ 684 parts: []*regexp.Regexp{ 685 compile("Call Trace:"), 686 parseStackTrace, 687 }, 688 skip: []string{"spin_"}, 689 }, 690 }, 691 { 692 title: compile("BUG: soft lockup"), 693 fmt: "BUG: soft lockup in %[1]v", 694 stack: &stackFmt{ 695 parts: []*regexp.Regexp{ 696 compile("Call Trace:"), 697 parseStackTrace, 698 }, 699 }, 700 }, 701 { 702 title: compile("BUG: .*still has locks held!"), 703 report: compile("BUG: .*still has locks held!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"), 704 fmt: "BUG: still has locks held in %[1]v", 705 }, 706 { 707 title: compile("BUG: lock held when returning to user space"), 708 report: compile("BUG: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 709 fmt: "BUG: lock held when returning to user space in %[1]v", 710 noStackTrace: true, 711 }, 712 { 713 title: compile("BUG: bad unlock balance detected!"), 714 report: compile("BUG: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"), 715 fmt: "BUG: bad unlock balance in %[1]v", 716 }, 717 { 718 title: compile("BUG: held lock freed!"), 719 report: compile("BUG: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"), 720 fmt: "BUG: held lock freed in %[1]v", 721 }, 722 { 723 title: compile("BUG: Bad rss-counter state"), 724 fmt: "BUG: Bad rss-counter state", 725 noStackTrace: true, 726 }, 727 { 728 title: compile("BUG: non-zero nr_ptes on freeing mm"), 729 fmt: "BUG: non-zero nr_ptes on freeing mm", 730 noStackTrace: true, 731 }, 732 { 733 title: compile("BUG: non-zero nr_pmds on freeing mm"), 734 fmt: "BUG: non-zero nr_pmds on freeing mm", 735 noStackTrace: true, 736 }, 737 { 738 title: compile("BUG: Dentry .* still in use \\([0-9]+\\) \\[unmount of ([^\\]]+)\\]"), 739 fmt: "BUG: Dentry still in use [unmount of %[1]v]", 740 }, 741 { 742 title: compile("BUG: Bad page state"), 743 fmt: "BUG: Bad page state", 744 }, 745 { 746 title: compile("BUG: Bad page map"), 747 fmt: "BUG: Bad page map", 748 }, 749 { 750 title: compile("BUG: workqueue lockup"), 751 fmt: "BUG: workqueue lockup", 752 noStackTrace: true, 753 }, 754 { 755 title: compile("BUG: sleeping function called from invalid context (.*)"), 756 fmt: "BUG: sleeping function called from invalid context %[1]v", 757 }, 758 { 759 title: compile("BUG: using __this_cpu_([a-z_]+)\\(\\) in preemptible"), 760 fmt: "BUG: using __this_cpu_%[1]v() in preemptible code in %[2]v", 761 stack: &stackFmt{ 762 parts: []*regexp.Regexp{ 763 compile("Call Trace:"), 764 parseStackTrace, 765 }, 766 skip: []string{"dump_stack", "preemption", "preempt"}, 767 }, 768 }, 769 { 770 title: compile("BUG: workqueue leaked lock or atomic"), 771 report: compile("BUG: workqueue leaked lock or atomic(?:.*\\n)+?" + 772 ".*last function: ([a-zA-Z0-9_]+)\\n"), 773 fmt: "BUG: workqueue leaked lock or atomic in %[1]v", 774 noStackTrace: true, 775 }, 776 { 777 title: compile("BUG: executor-detected bug"), 778 fmt: "BUG: executor-detected bug", 779 noStackTrace: true, 780 }, 781 { 782 title: compile("BUG: memory leak"), 783 fmt: "memory leak in %[1]v", 784 stack: &stackFmt{ 785 parts: []*regexp.Regexp{ 786 compile("backtrace:"), 787 parseStackTrace, 788 }, 789 skip: []string{"kmemleak", "kmalloc", "kcalloc", "kzalloc", 790 "vmalloc", "kmem", "slab", "alloc", "create_object"}, 791 }, 792 }, 793 }, 794 []*regexp.Regexp{ 795 // CONFIG_DEBUG_OBJECTS output. 796 compile("ODEBUG:"), 797 // Android prints this sometimes during boot. 798 compile("Boot_DEBUG:"), 799 // pkg/host output in debug mode. 800 compile("BUG: no syscalls can create resource"), 801 }, 802 }, 803 { 804 []byte("WARNING:"), 805 []oopsFormat{ 806 { 807 title: compile("WARNING: .*lib/debugobjects\\.c.* debug_print_object"), 808 fmt: "WARNING: ODEBUG bug in %[1]v", 809 // Skip all users of ODEBUG as well. 810 stack: warningStackFmt("debug_", "rcu", "hrtimer_", "timer_", 811 "work_", "percpu_", "kmem_", "slab_", "kfree", "vunmap", "vfree"), 812 }, 813 { 814 title: compile("WARNING: .*mm/usercopy\\.c.* usercopy_warn"), 815 fmt: "WARNING: bad usercopy in %[1]v", 816 stack: warningStackFmt("usercopy", "__check"), 817 }, 818 { 819 title: compile("WARNING: .*lib/kobject\\.c.* kobject_"), 820 fmt: "WARNING: kobject bug in %[1]v", 821 stack: warningStackFmt("kobject_"), 822 }, 823 { 824 title: compile("WARNING: .*fs/proc/generic\\.c.* proc_register"), 825 fmt: "WARNING: proc registration bug in %[1]v", 826 stack: warningStackFmt("proc_"), 827 }, 828 { 829 title: compile("WARNING: .*lib/refcount\\.c.* refcount_"), 830 fmt: "WARNING: refcount bug in %[1]v", 831 stack: warningStackFmt("refcount"), 832 }, 833 { 834 title: compile("WARNING: .*kernel/locking/lockdep\\.c.*lock_"), 835 fmt: "WARNING: locking bug in %[1]v", 836 stack: warningStackFmt(), 837 }, 838 { 839 title: compile("WARNING: lock held when returning to user space"), 840 report: compile("WARNING: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 841 fmt: "WARNING: lock held when returning to user space in %[1]v", 842 noStackTrace: true, 843 }, 844 { 845 title: compile("WARNING: .*mm/.*\\.c.* k?.?malloc"), 846 fmt: "WARNING: kmalloc bug in %[1]v", 847 stack: warningStackFmt("kmalloc", "kcalloc", "kzalloc", "krealloc", 848 "vmalloc", "slab", "kmem"), 849 }, 850 { 851 title: compile("WARNING: .* at {{SRC}} {{FUNC}}"), 852 fmt: "WARNING in %[2]v", 853 }, 854 { 855 title: compile("WARNING: possible circular locking dependency detected"), 856 report: compile("WARNING: possible circular locking dependency detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 857 fmt: "possible deadlock in %[1]v", 858 }, 859 { 860 title: compile("WARNING: possible irq lock inversion dependency detected"), 861 report: compile("WARNING: possible irq lock inversion dependency detected(?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 862 fmt: "possible deadlock in %[1]v", 863 }, 864 { 865 title: compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detecte"), 866 report: compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected(?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 867 fmt: "possible deadlock in %[1]v", 868 }, 869 { 870 title: compile("WARNING: possible recursive locking detected"), 871 report: compile("WARNING: possible recursive locking detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 872 fmt: "possible deadlock in %[1]v", 873 }, 874 { 875 title: compile("WARNING: inconsistent lock state"), 876 report: compile("WARNING: inconsistent lock state(?:.*\\n)+?.*takes(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"), 877 fmt: "inconsistent lock state in %[1]v", 878 }, 879 { 880 title: compile("WARNING: suspicious RCU usage"), 881 report: compile("WARNING: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"), 882 fmt: "WARNING: suspicious RCU usage in %[2]v", 883 stack: &stackFmt{ 884 parts: []*regexp.Regexp{ 885 compile("Call Trace:"), 886 parseStackTrace, 887 }, 888 skip: []string{"rcu", "kmem", "slab", "kmalloc", 889 "vmalloc", "kcalloc", "kzalloc"}, 890 }, 891 }, 892 { 893 title: compile("WARNING: kernel stack regs at [0-9a-f]+ in [^ ]* has bad '([^']+)' value"), 894 fmt: "WARNING: kernel stack regs has bad '%[1]v' value", 895 noStackTrace: true, 896 }, 897 { 898 title: compile("WARNING: kernel stack frame pointer at [0-9a-f]+ in [^ ]* has bad value"), 899 fmt: "WARNING: kernel stack frame pointer has bad value", 900 noStackTrace: true, 901 }, 902 { 903 title: compile("WARNING: bad unlock balance detected!"), 904 report: compile("WARNING: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"), 905 fmt: "WARNING: bad unlock balance in %[1]v", 906 }, 907 { 908 title: compile("WARNING: held lock freed!"), 909 report: compile("WARNING: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"), 910 fmt: "WARNING: held lock freed in %[1]v", 911 }, 912 { 913 title: compile("WARNING: kernel stack regs .* has bad 'bp' value"), 914 fmt: "WARNING: kernel stack regs has bad value", 915 noStackTrace: true, 916 }, 917 { 918 title: compile("WARNING: kernel stack frame pointer .* has bad value"), 919 fmt: "WARNING: kernel stack regs has bad value", 920 noStackTrace: true, 921 }, 922 }, 923 []*regexp.Regexp{ 924 compile("WARNING: /etc/ssh/moduli does not exist, using fixed modulus"), // printed by sshd 925 }, 926 }, 927 { 928 []byte("INFO:"), 929 []oopsFormat{ 930 { 931 title: compile("INFO: possible circular locking dependency detected"), 932 report: compile("INFO: possible circular locking dependency detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"), 933 fmt: "possible deadlock in %[1]v", 934 }, 935 { 936 title: compile("INFO: possible irq lock inversion dependency detected"), 937 report: compile("INFO: possible irq lock inversion dependency detected \\](?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"), 938 fmt: "possible deadlock in %[1]v", 939 }, 940 { 941 title: compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected"), 942 report: compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected \\](?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"), 943 fmt: "possible deadlock in %[1]v", 944 }, 945 { 946 title: compile("INFO: possible recursive locking detected"), 947 report: compile("INFO: possible recursive locking detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"), 948 fmt: "possible deadlock in %[1]v", 949 }, 950 { 951 title: compile("INFO: inconsistent lock state"), 952 report: compile("INFO: inconsistent lock state \\](?:.*\\n)+?.*takes(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"), 953 fmt: "inconsistent lock state in %[1]v", 954 }, 955 { 956 title: linuxRcuStall, 957 fmt: "INFO: rcu detected stall in %[1]v", 958 stack: &stackFmt{ 959 parts: []*regexp.Regexp{ 960 compile("apic_timer_interrupt"), 961 parseStackTrace, 962 }, 963 skip: []string{"apic_timer_interrupt", "rcu"}, 964 }, 965 }, 966 { 967 title: linuxRcuStall, 968 fmt: "INFO: rcu detected stall in %[1]v", 969 stack: &stackFmt{ 970 parts: []*regexp.Regexp{ 971 linuxRipFrame, 972 parseStackTrace, 973 }, 974 skip: []string{"apic_timer_interrupt", "rcu"}, 975 }, 976 }, 977 { 978 title: compile("INFO: trying to register non-static key"), 979 fmt: "INFO: trying to register non-static key in %[1]v", 980 stack: &stackFmt{ 981 parts: []*regexp.Regexp{ 982 compile("Call Trace:"), 983 parseStackTrace, 984 }, 985 skip: []string{"stack", "lock", "IRQ"}, 986 }, 987 }, 988 { 989 title: compile("INFO: suspicious RCU usage"), 990 report: compile("INFO: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"), 991 fmt: "INFO: suspicious RCU usage in %[2]v", 992 stack: &stackFmt{ 993 parts: []*regexp.Regexp{ 994 compile("Call Trace:"), 995 parseStackTrace, 996 }, 997 skip: []string{"rcu", "kmem", "slab", "kmalloc", 998 "vmalloc", "kcalloc", "kzalloc"}, 999 }, 1000 }, 1001 { 1002 title: compile("INFO: task .* blocked for more than [0-9]+ seconds"), 1003 fmt: "INFO: task hung in %[1]v", 1004 stack: &stackFmt{ 1005 parts: []*regexp.Regexp{ 1006 compile("Call Trace:"), 1007 parseStackTrace, 1008 }, 1009 skip: []string{"sched", "_lock", "down", "completion", "kthread", 1010 "wait", "synchronize"}, 1011 }, 1012 }, 1013 { 1014 // This gets captured for corrupted old-style KASAN reports. 1015 title: compile("INFO: (Freed|Allocated) in (.*)"), 1016 fmt: "INFO: %[1]v in %[2]v", 1017 corrupted: true, 1018 }, 1019 }, 1020 []*regexp.Regexp{ 1021 compile("INFO: lockdep is turned off"), 1022 compile("INFO: Stall ended before state dump start"), 1023 compile("INFO: NMI handler"), 1024 compile("(handler|interrupt).*took too long"), 1025 compile("_INFO::"), // Android can print this during boot. 1026 compile("INFO: sys_.* is not present in /proc/kallsyms"), // pkg/host output in debug mode 1027 compile("INFO: no syscalls can create resource"), // pkg/host output in debug mode 1028 }, 1029 }, 1030 { 1031 []byte("Unable to handle kernel paging request"), 1032 []oopsFormat{ 1033 { 1034 title: compile("Unable to handle kernel paging request"), 1035 report: compile("Unable to handle kernel paging request(?:.*\\n)+?.*PC is at {{FUNC}}"), 1036 fmt: "unable to handle kernel paging request in %[1]v", 1037 }, 1038 }, 1039 []*regexp.Regexp{}, 1040 }, 1041 { 1042 []byte("general protection fault:"), 1043 []oopsFormat{ 1044 { 1045 title: compile("general protection fault:"), 1046 fmt: "general protection fault in %[1]v", 1047 stack: &stackFmt{ 1048 parts: []*regexp.Regexp{ 1049 linuxRipFrame, 1050 compile("Call Trace:"), 1051 parseStackTrace, 1052 }, 1053 }, 1054 }, 1055 }, 1056 []*regexp.Regexp{}, 1057 }, 1058 { 1059 []byte("Kernel panic"), 1060 []oopsFormat{ 1061 { 1062 title: compile("Kernel panic - not syncing: Attempted to kill init!"), 1063 fmt: "kernel panic: Attempted to kill init!", 1064 }, 1065 { 1066 title: compile("Kernel panic - not syncing: Couldn't open N_TTY ldisc for [^ ]+ --- error -[0-9]+"), 1067 fmt: "kernel panic: Couldn't open N_TTY ldisc", 1068 }, 1069 { 1070 // 'kernel panic: Fatal exception' is usually printed after BUG, 1071 // so if we captured it as a report description, that means the 1072 // report got truncated and we missed the actual BUG header. 1073 title: compile("Kernel panic - not syncing: Fatal exception"), 1074 fmt: "kernel panic: Fatal exception", 1075 corrupted: true, 1076 }, 1077 { 1078 // Same, but for WARNINGs and KASAN reports. 1079 title: compile("Kernel panic - not syncing: panic_on_warn set"), 1080 fmt: "kernel panic: panic_on_warn set", 1081 corrupted: true, 1082 }, 1083 { 1084 // Same, but for task hung reports. 1085 title: compile("Kernel panic - not syncing: hung_task: blocked tasks"), 1086 fmt: "kernel panic: hung_task: blocked tasks", 1087 corrupted: true, 1088 }, 1089 { 1090 title: compile("Kernel panic - not syncing: (.*)"), 1091 fmt: "kernel panic: %[1]v", 1092 }, 1093 }, 1094 []*regexp.Regexp{}, 1095 }, 1096 { 1097 []byte("kernel BUG"), 1098 []oopsFormat{ 1099 { 1100 title: compile("kernel BUG at mm/usercopy.c"), 1101 fmt: "BUG: bad usercopy in %[1]v", 1102 stack: &stackFmt{ 1103 parts: []*regexp.Regexp{ 1104 compile("Call Trace:"), 1105 parseStackTrace, 1106 }, 1107 }, 1108 }, 1109 { 1110 title: compile("kernel BUG at lib/list_debug.c"), 1111 fmt: "BUG: corrupted list in %[1]v", 1112 stack: &stackFmt{ 1113 parts: []*regexp.Regexp{ 1114 compile("Call Trace:"), 1115 parseStackTrace, 1116 }, 1117 }, 1118 }, 1119 }, 1120 []*regexp.Regexp{}, 1121 }, 1122 { 1123 []byte("Kernel BUG"), 1124 []oopsFormat{ 1125 { 1126 title: compile("Kernel BUG (.*)"), 1127 fmt: "kernel BUG %[1]v", 1128 }, 1129 }, 1130 []*regexp.Regexp{}, 1131 }, 1132 { 1133 []byte("BUG kmalloc-"), 1134 []oopsFormat{ 1135 { 1136 title: compile("BUG kmalloc-.*: Object already free"), 1137 fmt: "BUG: Object already free", 1138 }, 1139 }, 1140 []*regexp.Regexp{}, 1141 }, 1142 { 1143 []byte("divide error:"), 1144 []oopsFormat{ 1145 { 1146 title: compile("divide error: "), 1147 report: compile("divide error: (?:.*\\n)+?.*RIP: [0-9]+:(?:{{PC}} +{{PC}} +)?{{FUNC}}"), 1148 fmt: "divide error in %[1]v", 1149 }, 1150 }, 1151 []*regexp.Regexp{}, 1152 }, 1153 { 1154 []byte("invalid opcode:"), 1155 []oopsFormat{ 1156 { 1157 title: compile("invalid opcode: "), 1158 report: compile("invalid opcode: (?:.*\\n)+?.*RIP: [0-9]+:{{PC}} +{{PC}} +{{FUNC}}"), 1159 fmt: "invalid opcode in %[1]v", 1160 }, 1161 }, 1162 []*regexp.Regexp{}, 1163 }, 1164 { 1165 []byte("UBSAN:"), 1166 []oopsFormat{ 1167 { 1168 title: compile("UBSAN: (.*)"), 1169 fmt: "UBSAN: %[1]v", 1170 }, 1171 }, 1172 []*regexp.Regexp{}, 1173 }, 1174 { 1175 []byte("Booting the kernel."), 1176 []oopsFormat{ 1177 { 1178 title: compile("Booting the kernel."), 1179 fmt: "unexpected kernel reboot", 1180 noStackTrace: true, 1181 }, 1182 }, 1183 []*regexp.Regexp{}, 1184 }, 1185 { 1186 []byte("unregister_netdevice: waiting for"), 1187 []oopsFormat{ 1188 { 1189 title: compile("unregister_netdevice: waiting for (?:.*) to become free"), 1190 fmt: "unregister_netdevice: waiting for DEV to become free", 1191 noStackTrace: true, 1192 }, 1193 }, 1194 []*regexp.Regexp{}, 1195 }, 1196 } 1197
