1 // Copyright 2014 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package driver 16 17 import ( 18 "bytes" 19 "crypto/tls" 20 "fmt" 21 "io" 22 "io/ioutil" 23 "net/http" 24 "net/url" 25 "os" 26 "os/exec" 27 "path/filepath" 28 "runtime" 29 "strconv" 30 "strings" 31 "sync" 32 "time" 33 34 "github.com/google/pprof/internal/measurement" 35 "github.com/google/pprof/internal/plugin" 36 "github.com/google/pprof/profile" 37 ) 38 39 // fetchProfiles fetches and symbolizes the profiles specified by s. 40 // It will merge all the profiles it is able to retrieve, even if 41 // there are some failures. It will return an error if it is unable to 42 // fetch any profiles. 43 func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) { 44 sources := make([]profileSource, 0, len(s.Sources)) 45 for _, src := range s.Sources { 46 sources = append(sources, profileSource{ 47 addr: src, 48 source: s, 49 }) 50 } 51 52 bases := make([]profileSource, 0, len(s.Base)) 53 for _, src := range s.Base { 54 bases = append(bases, profileSource{ 55 addr: src, 56 source: s, 57 }) 58 } 59 60 p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI) 61 if err != nil { 62 return nil, err 63 } 64 65 if pbase != nil { 66 if s.Normalize { 67 err := p.Normalize(pbase) 68 if err != nil { 69 return nil, err 70 } 71 } 72 pbase.Scale(-1) 73 p, m, err = combineProfiles([]*profile.Profile{p, pbase}, []plugin.MappingSources{m, mbase}) 74 if err != nil { 75 return nil, err 76 } 77 } 78 79 // Symbolize the merged profile. 80 if err := o.Sym.Symbolize(s.Symbolize, m, p); err != nil { 81 return nil, err 82 } 83 p.RemoveUninteresting() 84 unsourceMappings(p) 85 86 if s.Comment != "" { 87 p.Comments = append(p.Comments, s.Comment) 88 } 89 90 // Save a copy of the merged profile if there is at least one remote source. 91 if save { 92 dir, err := setTmpDir(o.UI) 93 if err != nil { 94 return nil, err 95 } 96 97 prefix := "pprof." 98 if len(p.Mapping) > 0 && p.Mapping[0].File != "" { 99 prefix += filepath.Base(p.Mapping[0].File) + "." 100 } 101 for _, s := range p.SampleType { 102 prefix += s.Type + "." 103 } 104 105 tempFile, err := newTempFile(dir, prefix, ".pb.gz") 106 if err == nil { 107 if err = p.Write(tempFile); err == nil { 108 o.UI.PrintErr("Saved profile in ", tempFile.Name()) 109 } 110 } 111 if err != nil { 112 o.UI.PrintErr("Could not save profile: ", err) 113 } 114 } 115 116 if err := p.CheckValid(); err != nil { 117 return nil, err 118 } 119 120 return p, nil 121 } 122 123 func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) { 124 wg := sync.WaitGroup{} 125 wg.Add(2) 126 var psrc, pbase *profile.Profile 127 var msrc, mbase plugin.MappingSources 128 var savesrc, savebase bool 129 var errsrc, errbase error 130 var countsrc, countbase int 131 go func() { 132 defer wg.Done() 133 psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui) 134 }() 135 go func() { 136 defer wg.Done() 137 pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui) 138 }() 139 wg.Wait() 140 save := savesrc || savebase 141 142 if errsrc != nil { 143 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching source profiles: %v", errsrc) 144 } 145 if errbase != nil { 146 return nil, nil, nil, nil, false, fmt.Errorf("problem fetching base profiles: %v,", errbase) 147 } 148 if countsrc == 0 { 149 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any source profiles") 150 } 151 if countbase == 0 && len(bases) > 0 { 152 return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any base profiles") 153 } 154 if want, got := len(sources), countsrc; want != got { 155 ui.PrintErr(fmt.Sprintf("Fetched %d source profiles out of %d", got, want)) 156 } 157 if want, got := len(bases), countbase; want != got { 158 ui.PrintErr(fmt.Sprintf("Fetched %d base profiles out of %d", got, want)) 159 } 160 161 return psrc, pbase, msrc, mbase, save, nil 162 } 163 164 // chunkedGrab fetches the profiles described in source and merges them into 165 // a single profile. It fetches a chunk of profiles concurrently, with a maximum 166 // chunk size to limit its memory usage. 167 func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) { 168 const chunkSize = 64 169 170 var p *profile.Profile 171 var msrc plugin.MappingSources 172 var save bool 173 var count int 174 175 for start := 0; start < len(sources); start += chunkSize { 176 end := start + chunkSize 177 if end > len(sources) { 178 end = len(sources) 179 } 180 chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui) 181 switch { 182 case chunkErr != nil: 183 return nil, nil, false, 0, chunkErr 184 case chunkP == nil: 185 continue 186 case p == nil: 187 p, msrc, save, count = chunkP, chunkMsrc, chunkSave, chunkCount 188 default: 189 p, msrc, chunkErr = combineProfiles([]*profile.Profile{p, chunkP}, []plugin.MappingSources{msrc, chunkMsrc}) 190 if chunkErr != nil { 191 return nil, nil, false, 0, chunkErr 192 } 193 if chunkSave { 194 save = true 195 } 196 count += chunkCount 197 } 198 } 199 200 return p, msrc, save, count, nil 201 } 202 203 // concurrentGrab fetches multiple profiles concurrently 204 func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (*profile.Profile, plugin.MappingSources, bool, int, error) { 205 wg := sync.WaitGroup{} 206 wg.Add(len(sources)) 207 for i := range sources { 208 go func(s *profileSource) { 209 defer wg.Done() 210 s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui) 211 }(&sources[i]) 212 } 213 wg.Wait() 214 215 var save bool 216 profiles := make([]*profile.Profile, 0, len(sources)) 217 msrcs := make([]plugin.MappingSources, 0, len(sources)) 218 for i := range sources { 219 s := &sources[i] 220 if err := s.err; err != nil { 221 ui.PrintErr(s.addr + ": " + err.Error()) 222 continue 223 } 224 save = save || s.remote 225 profiles = append(profiles, s.p) 226 msrcs = append(msrcs, s.msrc) 227 *s = profileSource{} 228 } 229 230 if len(profiles) == 0 { 231 return nil, nil, false, 0, nil 232 } 233 234 p, msrc, err := combineProfiles(profiles, msrcs) 235 if err != nil { 236 return nil, nil, false, 0, err 237 } 238 return p, msrc, save, len(profiles), nil 239 } 240 241 func combineProfiles(profiles []*profile.Profile, msrcs []plugin.MappingSources) (*profile.Profile, plugin.MappingSources, error) { 242 // Merge profiles. 243 if err := measurement.ScaleProfiles(profiles); err != nil { 244 return nil, nil, err 245 } 246 247 p, err := profile.Merge(profiles) 248 if err != nil { 249 return nil, nil, err 250 } 251 252 // Combine mapping sources. 253 msrc := make(plugin.MappingSources) 254 for _, ms := range msrcs { 255 for m, s := range ms { 256 msrc[m] = append(msrc[m], s...) 257 } 258 } 259 return p, msrc, nil 260 } 261 262 type profileSource struct { 263 addr string 264 source *source 265 266 p *profile.Profile 267 msrc plugin.MappingSources 268 remote bool 269 err error 270 } 271 272 func homeEnv() string { 273 switch runtime.GOOS { 274 case "windows": 275 return "USERPROFILE" 276 case "plan9": 277 return "home" 278 default: 279 return "HOME" 280 } 281 } 282 283 // setTmpDir prepares the directory to use to save profiles retrieved 284 // remotely. It is selected from PPROF_TMPDIR, defaults to $HOME/pprof, and, if 285 // $HOME is not set, falls back to os.TempDir(). 286 func setTmpDir(ui plugin.UI) (string, error) { 287 var dirs []string 288 if profileDir := os.Getenv("PPROF_TMPDIR"); profileDir != "" { 289 dirs = append(dirs, profileDir) 290 } 291 if homeDir := os.Getenv(homeEnv()); homeDir != "" { 292 dirs = append(dirs, filepath.Join(homeDir, "pprof")) 293 } 294 dirs = append(dirs, os.TempDir()) 295 for _, tmpDir := range dirs { 296 if err := os.MkdirAll(tmpDir, 0755); err != nil { 297 ui.PrintErr("Could not use temp dir ", tmpDir, ": ", err.Error()) 298 continue 299 } 300 return tmpDir, nil 301 } 302 return "", fmt.Errorf("failed to identify temp dir") 303 } 304 305 const testSourceAddress = "pproftest.local" 306 307 // grabProfile fetches a profile. Returns the profile, sources for the 308 // profile mappings, a bool indicating if the profile was fetched 309 // remotely, and an error. 310 func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) { 311 var src string 312 duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second 313 if fetcher != nil { 314 p, src, err = fetcher.Fetch(source, duration, timeout) 315 if err != nil { 316 return 317 } 318 } 319 if err != nil || p == nil { 320 // Fetch the profile over HTTP or from a file. 321 p, src, err = fetch(source, duration, timeout, ui) 322 if err != nil { 323 return 324 } 325 } 326 327 if err = p.CheckValid(); err != nil { 328 return 329 } 330 331 // Update the binary locations from command line and paths. 332 locateBinaries(p, s, obj, ui) 333 334 // Collect the source URL for all mappings. 335 if src != "" { 336 msrc = collectMappingSources(p, src) 337 remote = true 338 if strings.HasPrefix(src, "http://"+testSourceAddress) { 339 // Treat test inputs as local to avoid saving 340 // testcase profiles during driver testing. 341 remote = false 342 } 343 } 344 return 345 } 346 347 // collectMappingSources saves the mapping sources of a profile. 348 func collectMappingSources(p *profile.Profile, source string) plugin.MappingSources { 349 ms := plugin.MappingSources{} 350 for _, m := range p.Mapping { 351 src := struct { 352 Source string 353 Start uint64 354 }{ 355 source, m.Start, 356 } 357 key := m.BuildID 358 if key == "" { 359 key = m.File 360 } 361 if key == "" { 362 // If there is no build id or source file, use the source as the 363 // mapping file. This will enable remote symbolization for this 364 // mapping, in particular for Go profiles on the legacy format. 365 // The source is reset back to empty string by unsourceMapping 366 // which is called after symbolization is finished. 367 m.File = source 368 key = source 369 } 370 ms[key] = append(ms[key], src) 371 } 372 return ms 373 } 374 375 // unsourceMappings iterates over the mappings in a profile and replaces file 376 // set to the remote source URL by collectMappingSources back to empty string. 377 func unsourceMappings(p *profile.Profile) { 378 for _, m := range p.Mapping { 379 if m.BuildID == "" { 380 if u, err := url.Parse(m.File); err == nil && u.IsAbs() { 381 m.File = "" 382 } 383 } 384 } 385 } 386 387 // locateBinaries searches for binary files listed in the profile and, if found, 388 // updates the profile accordingly. 389 func locateBinaries(p *profile.Profile, s *source, obj plugin.ObjTool, ui plugin.UI) { 390 // Construct search path to examine 391 searchPath := os.Getenv("PPROF_BINARY_PATH") 392 if searchPath == "" { 393 // Use $HOME/pprof/binaries as default directory for local symbolization binaries 394 searchPath = filepath.Join(os.Getenv(homeEnv()), "pprof", "binaries") 395 } 396 mapping: 397 for _, m := range p.Mapping { 398 var baseName string 399 if m.File != "" { 400 baseName = filepath.Base(m.File) 401 } 402 403 for _, path := range filepath.SplitList(searchPath) { 404 var fileNames []string 405 if m.BuildID != "" { 406 fileNames = []string{filepath.Join(path, m.BuildID, baseName)} 407 if matches, err := filepath.Glob(filepath.Join(path, m.BuildID, "*")); err == nil { 408 fileNames = append(fileNames, matches...) 409 } 410 } 411 if m.File != "" { 412 // Try both the basename and the full path, to support the same directory 413 // structure as the perf symfs option. 414 if baseName != "" { 415 fileNames = append(fileNames, filepath.Join(path, baseName)) 416 } 417 fileNames = append(fileNames, filepath.Join(path, m.File)) 418 } 419 for _, name := range fileNames { 420 if f, err := obj.Open(name, m.Start, m.Limit, m.Offset); err == nil { 421 defer f.Close() 422 fileBuildID := f.BuildID() 423 if m.BuildID != "" && m.BuildID != fileBuildID { 424 ui.PrintErr("Ignoring local file " + name + ": build-id mismatch (" + m.BuildID + " != " + fileBuildID + ")") 425 } else { 426 m.File = name 427 continue mapping 428 } 429 } 430 } 431 } 432 } 433 if len(p.Mapping) == 0 { 434 // If there are no mappings, add a fake mapping to attempt symbolization. 435 // This is useful for some profiles generated by the golang runtime, which 436 // do not include any mappings. Symbolization with a fake mapping will only 437 // be successful against a non-PIE binary. 438 m := &profile.Mapping{ID: 1} 439 p.Mapping = []*profile.Mapping{m} 440 for _, l := range p.Location { 441 l.Mapping = m 442 } 443 } 444 // Replace executable filename/buildID with the overrides from source. 445 // Assumes the executable is the first Mapping entry. 446 if execName, buildID := s.ExecName, s.BuildID; execName != "" || buildID != "" { 447 m := p.Mapping[0] 448 if execName != "" { 449 m.File = execName 450 } 451 if buildID != "" { 452 m.BuildID = buildID 453 } 454 } 455 } 456 457 // fetch fetches a profile from source, within the timeout specified, 458 // producing messages through the ui. It returns the profile and the 459 // url of the actual source of the profile for remote profiles. 460 func fetch(source string, duration, timeout time.Duration, ui plugin.UI) (p *profile.Profile, src string, err error) { 461 var f io.ReadCloser 462 463 if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" { 464 ui.Print("Fetching profile over HTTP from " + sourceURL) 465 if duration > 0 { 466 ui.Print(fmt.Sprintf("Please wait... (%v)", duration)) 467 } 468 f, err = fetchURL(sourceURL, timeout) 469 src = sourceURL 470 } else if isPerfFile(source) { 471 f, err = convertPerfData(source, ui) 472 } else { 473 f, err = os.Open(source) 474 } 475 if err == nil { 476 defer f.Close() 477 p, err = profile.Parse(f) 478 } 479 return 480 } 481 482 // fetchURL fetches a profile from a URL using HTTP. 483 func fetchURL(source string, timeout time.Duration) (io.ReadCloser, error) { 484 resp, err := httpGet(source, timeout) 485 if err != nil { 486 return nil, fmt.Errorf("http fetch: %v", err) 487 } 488 if resp.StatusCode != http.StatusOK { 489 defer resp.Body.Close() 490 return nil, statusCodeError(resp) 491 } 492 493 return resp.Body, nil 494 } 495 496 func statusCodeError(resp *http.Response) error { 497 if resp.Header.Get("X-Go-Pprof") != "" && strings.Contains(resp.Header.Get("Content-Type"), "text/plain") { 498 // error is from pprof endpoint 499 if body, err := ioutil.ReadAll(resp.Body); err == nil { 500 return fmt.Errorf("server response: %s - %s", resp.Status, body) 501 } 502 } 503 return fmt.Errorf("server response: %s", resp.Status) 504 } 505 506 // isPerfFile checks if a file is in perf.data format. It also returns false 507 // if it encounters an error during the check. 508 func isPerfFile(path string) bool { 509 sourceFile, openErr := os.Open(path) 510 if openErr != nil { 511 return false 512 } 513 defer sourceFile.Close() 514 515 // If the file is the output of a perf record command, it should begin 516 // with the string PERFILE2. 517 perfHeader := []byte("PERFILE2") 518 actualHeader := make([]byte, len(perfHeader)) 519 if _, readErr := sourceFile.Read(actualHeader); readErr != nil { 520 return false 521 } 522 return bytes.Equal(actualHeader, perfHeader) 523 } 524 525 // convertPerfData converts the file at path which should be in perf.data format 526 // using the perf_to_profile tool and returns the file containing the 527 // profile.proto formatted data. 528 func convertPerfData(perfPath string, ui plugin.UI) (*os.File, error) { 529 ui.Print(fmt.Sprintf( 530 "Converting %s to a profile.proto... (May take a few minutes)", 531 perfPath)) 532 profile, err := newTempFile(os.TempDir(), "pprof_", ".pb.gz") 533 if err != nil { 534 return nil, err 535 } 536 deferDeleteTempFile(profile.Name()) 537 cmd := exec.Command("perf_to_profile", perfPath, profile.Name()) 538 if err := cmd.Run(); err != nil { 539 profile.Close() 540 return nil, fmt.Errorf("failed to convert perf.data file. Try github.com/google/perf_data_converter: %v", err) 541 } 542 return profile, nil 543 } 544 545 // adjustURL validates if a profile source is a URL and returns an 546 // cleaned up URL and the timeout to use for retrieval over HTTP. 547 // If the source cannot be recognized as a URL it returns an empty string. 548 func adjustURL(source string, duration, timeout time.Duration) (string, time.Duration) { 549 u, err := url.Parse(source) 550 if err != nil || (u.Host == "" && u.Scheme != "" && u.Scheme != "file") { 551 // Try adding http:// to catch sources of the form hostname:port/path. 552 // url.Parse treats "hostname" as the scheme. 553 u, err = url.Parse("http://" + source) 554 } 555 if err != nil || u.Host == "" { 556 return "", 0 557 } 558 559 // Apply duration/timeout overrides to URL. 560 values := u.Query() 561 if duration > 0 { 562 values.Set("seconds", fmt.Sprint(int(duration.Seconds()))) 563 } else { 564 if urlSeconds := values.Get("seconds"); urlSeconds != "" { 565 if us, err := strconv.ParseInt(urlSeconds, 10, 32); err == nil { 566 duration = time.Duration(us) * time.Second 567 } 568 } 569 } 570 if timeout <= 0 { 571 if duration > 0 { 572 timeout = duration + duration/2 573 } else { 574 timeout = 60 * time.Second 575 } 576 } 577 u.RawQuery = values.Encode() 578 return u.String(), timeout 579 } 580 581 // httpGet is a wrapper around http.Get; it is defined as a variable 582 // so it can be redefined during for testing. 583 var httpGet = func(source string, timeout time.Duration) (*http.Response, error) { 584 url, err := url.Parse(source) 585 if err != nil { 586 return nil, err 587 } 588 589 var tlsConfig *tls.Config 590 if url.Scheme == "https+insecure" { 591 tlsConfig = &tls.Config{ 592 InsecureSkipVerify: true, 593 } 594 url.Scheme = "https" 595 source = url.String() 596 } 597 598 client := &http.Client{ 599 Transport: &http.Transport{ 600 ResponseHeaderTimeout: timeout + 5*time.Second, 601 Proxy: http.ProxyFromEnvironment, 602 TLSClientConfig: tlsConfig, 603 }, 604 } 605 return client.Get(source) 606 } 607