1 /** 2 * @file daemon/opd_ibs.c 3 * AMD Family10h Instruction Based Sampling (IBS) handling. 4 * 5 * @remark Copyright 2007 OProfile authors 6 * @remark Read the file COPYING 7 * 8 * @author Jason Yeh <jason.yeh (at) amd.com> 9 * @author Paul Drongowski <paul.drongowski (at) amd.com> 10 * @author Suravee Suthikulpanit <suravee.suthikulpanit (at) amd.com> 11 * Copyright (c) 2008 Advanced Micro Devices, Inc. 12 */ 13 14 #include "op_hw_config.h" 15 #include "op_events.h" 16 #include "op_string.h" 17 #include "op_libiberty.h" 18 #include "opd_printf.h" 19 #include "opd_trans.h" 20 #include "opd_events.h" 21 #include "opd_kernel.h" 22 #include "opd_anon.h" 23 #include "opd_sfile.h" 24 #include "opd_interface.h" 25 #include "opd_mangling.h" 26 #include "opd_extended.h" 27 #include "opd_ibs.h" 28 #include "opd_ibs_trans.h" 29 #include "opd_ibs_macro.h" 30 31 #include <stdlib.h> 32 #include <stdio.h> 33 #include <errno.h> 34 #include <string.h> 35 36 extern op_cpu cpu_type; 37 extern int no_event_ok; 38 extern int sfile_equal(struct sfile const * sf, struct sfile const * sf2); 39 extern void sfile_dup(struct sfile * to, struct sfile * from); 40 41 /* IBS Select Arrays/Counters */ 42 static unsigned int ibs_selected_size; 43 static unsigned int ibs_fetch_selected_flag; 44 static unsigned int ibs_fetch_selected_size; 45 static unsigned int ibs_op_selected_flag; 46 static unsigned int ibs_op_selected_size; 47 static unsigned int ibs_op_ls_selected_flag; 48 static unsigned int ibs_op_ls_selected_size; 49 static unsigned int ibs_op_nb_selected_flag; 50 static unsigned int ibs_op_nb_selected_size; 51 52 /* IBS Statistics */ 53 static unsigned long ibs_fetch_sample_stats; 54 static unsigned long ibs_fetch_incomplete_stats; 55 static unsigned long ibs_op_sample_stats; 56 static unsigned long ibs_op_incomplete_stats; 57 static unsigned long ibs_derived_event_stats; 58 59 /* 60 * IBS Virtual Counter 61 */ 62 struct opd_event ibs_vc[OP_MAX_IBS_COUNTERS]; 63 64 /* IBS Virtual Counter Index(VCI) Map*/ 65 unsigned int ibs_vci_map[OP_MAX_IBS_COUNTERS]; 66 67 /** 68 * This function converts IBS fetch event flags and values into 69 * derived events. If the tagged (sampled) fetched caused a derived 70 * event, the derived event is tallied. 71 */ 72 static void opd_log_ibs_fetch(struct transient * trans) 73 { 74 struct ibs_fetch_sample * trans_fetch = ((struct ibs_sample*)(trans->ext))->fetch; 75 if (!trans_fetch) 76 return; 77 78 trans_ibs_fetch(trans, ibs_fetch_selected_flag, ibs_fetch_selected_size); 79 } 80 81 82 /** 83 * This function translates the IBS op event flags and values into 84 * IBS op derived events. If an op derived event occured, it's tallied. 85 */ 86 static void opd_log_ibs_op(struct transient * trans) 87 { 88 struct ibs_op_sample * trans_op = ((struct ibs_sample*)(trans->ext))->op; 89 if (!trans_op) 90 return; 91 92 trans_ibs_op(trans, ibs_op_selected_flag, ibs_op_selected_size); 93 trans_ibs_op_ls(trans, ibs_op_ls_selected_flag, ibs_op_ls_selected_size); 94 trans_ibs_op_nb(trans, ibs_op_nb_selected_flag, ibs_op_nb_selected_size); 95 } 96 97 98 static void opd_put_ibs_sample(struct transient * trans) 99 { 100 unsigned long long event = 0; 101 struct kernel_image * k_image = NULL; 102 struct ibs_fetch_sample * trans_fetch = ((struct ibs_sample*)(trans->ext))->fetch; 103 104 if (!enough_remaining(trans, 1)) { 105 trans->remaining = 0; 106 return; 107 } 108 109 /* IBS can generate samples with invalid dcookie and 110 * in kernel address range. Map such samples to vmlinux 111 * only if the user either specifies a range, or vmlinux. 112 */ 113 if (trans->cookie == INVALID_COOKIE 114 && (k_image = find_kernel_image(trans)) != NULL 115 && (k_image->start != 0 && k_image->end != 0) 116 && trans->in_kernel == 0) 117 trans->in_kernel = 1; 118 119 if (trans->tracing != TRACING_ON) 120 trans->event = event; 121 122 /* sfile can change at each sample for kernel */ 123 if (trans->in_kernel != 0) 124 clear_trans_current(trans); 125 126 if (!trans->in_kernel && trans->cookie == NO_COOKIE) 127 trans->anon = find_anon_mapping(trans); 128 129 /* get the current sfile if needed */ 130 if (!trans->current) 131 trans->current = sfile_find(trans); 132 133 /* 134 * can happen if kernel sample falls through the cracks, or if 135 * it's a sample from an anon region we couldn't find 136 */ 137 if (!trans->current) 138 goto out; 139 140 if (trans_fetch) 141 opd_log_ibs_fetch(trans); 142 else 143 opd_log_ibs_op(trans); 144 out: 145 /* switch to trace mode */ 146 if (trans->tracing == TRACING_START) 147 trans->tracing = TRACING_ON; 148 149 update_trans_last(trans); 150 } 151 152 153 void code_ibs_fetch_sample(struct transient * trans) 154 { 155 struct ibs_fetch_sample * trans_fetch = NULL; 156 157 if (!enough_remaining(trans, 7)) { 158 verbprintf(vext, "not enough remaining\n"); 159 trans->remaining = 0; 160 ibs_fetch_incomplete_stats++; 161 return; 162 } 163 164 ibs_fetch_sample_stats++; 165 166 trans->ext = xmalloc(sizeof(struct ibs_sample)); 167 ((struct ibs_sample*)(trans->ext))->fetch = xmalloc(sizeof(struct ibs_fetch_sample)); 168 trans_fetch = ((struct ibs_sample*)(trans->ext))->fetch; 169 170 trans_fetch->rip = pop_buffer_value(trans); 171 172 trans_fetch->ibs_fetch_lin_addr_low = pop_buffer_value(trans); 173 trans_fetch->ibs_fetch_lin_addr_high = pop_buffer_value(trans); 174 175 trans_fetch->ibs_fetch_ctl_low = pop_buffer_value(trans); 176 trans_fetch->ibs_fetch_ctl_high = pop_buffer_value(trans); 177 trans_fetch->ibs_fetch_phys_addr_low = pop_buffer_value(trans); 178 trans_fetch->ibs_fetch_phys_addr_high = pop_buffer_value(trans); 179 180 verbprintf(vsamples, 181 "FETCH_X CPU:%ld PID:%ld RIP:%lx CTL_H:%x LAT:%d P_HI:%x P_LO:%x L_HI:%x L_LO:%x\n", 182 trans->cpu, 183 (long)trans->tgid, 184 trans_fetch->rip, 185 (trans_fetch->ibs_fetch_ctl_high >> 16) & 0x3ff, 186 (trans_fetch->ibs_fetch_ctl_high) & 0xffff, 187 trans_fetch->ibs_fetch_phys_addr_high, 188 trans_fetch->ibs_fetch_phys_addr_low, 189 trans_fetch->ibs_fetch_lin_addr_high, 190 trans_fetch->ibs_fetch_lin_addr_low) ; 191 192 /* Overwrite the trans->pc with the more accurate trans_fetch->rip */ 193 trans->pc = trans_fetch->rip; 194 195 opd_put_ibs_sample(trans); 196 197 free(trans_fetch); 198 free(trans->ext); 199 trans->ext = NULL; 200 } 201 202 203 void code_ibs_op_sample(struct transient * trans) 204 { 205 struct ibs_op_sample * trans_op= NULL; 206 207 if (!enough_remaining(trans, 13)) { 208 verbprintf(vext, "not enough remaining\n"); 209 trans->remaining = 0; 210 ibs_op_incomplete_stats++; 211 return; 212 } 213 214 ibs_op_sample_stats++; 215 216 trans->ext = xmalloc(sizeof(struct ibs_sample)); 217 ((struct ibs_sample*)(trans->ext))->op = xmalloc(sizeof(struct ibs_op_sample)); 218 trans_op = ((struct ibs_sample*)(trans->ext))->op; 219 220 trans_op->rip = pop_buffer_value(trans); 221 222 trans_op->ibs_op_lin_addr_low = pop_buffer_value(trans); 223 trans_op->ibs_op_lin_addr_high = pop_buffer_value(trans); 224 225 trans_op->ibs_op_data1_low = pop_buffer_value(trans); 226 trans_op->ibs_op_data1_high = pop_buffer_value(trans); 227 trans_op->ibs_op_data2_low = pop_buffer_value(trans); 228 trans_op->ibs_op_data2_high = pop_buffer_value(trans); 229 trans_op->ibs_op_data3_low = pop_buffer_value(trans); 230 trans_op->ibs_op_data3_high = pop_buffer_value(trans); 231 trans_op->ibs_op_ldst_linaddr_low = pop_buffer_value(trans); 232 trans_op->ibs_op_ldst_linaddr_high = pop_buffer_value(trans); 233 trans_op->ibs_op_phys_addr_low = pop_buffer_value(trans); 234 trans_op->ibs_op_phys_addr_high = pop_buffer_value(trans); 235 236 verbprintf(vsamples, 237 "IBS_OP_X CPU:%ld PID:%d RIP:%lx D1HI:%x D1LO:%x D2LO:%x D3HI:%x D3LO:%x L_LO:%x P_LO:%x\n", 238 trans->cpu, 239 trans->tgid, 240 trans_op->rip, 241 trans_op->ibs_op_data1_high, 242 trans_op->ibs_op_data1_low, 243 trans_op->ibs_op_data2_low, 244 trans_op->ibs_op_data3_high, 245 trans_op->ibs_op_data3_low, 246 trans_op->ibs_op_ldst_linaddr_low, 247 trans_op->ibs_op_phys_addr_low); 248 249 /* Overwrite the trans->pc with the more accurate trans_op->rip */ 250 trans->pc = trans_op->rip; 251 252 opd_put_ibs_sample(trans); 253 254 free(trans_op); 255 free(trans->ext); 256 trans->ext = NULL; 257 } 258 259 260 /** Convert IBS event to value used for data structure indexing */ 261 static unsigned long ibs_event_to_counter(unsigned long x) 262 { 263 unsigned long ret = ~0UL; 264 265 if (IS_IBS_FETCH(x)) 266 ret = (x - IBS_FETCH_BASE); 267 else if (IS_IBS_OP(x)) 268 ret = (x - IBS_OP_BASE + IBS_FETCH_MAX); 269 else if (IS_IBS_OP_LS(x)) 270 ret = (x - IBS_OP_LS_BASE + IBS_OP_MAX + IBS_FETCH_MAX); 271 else if (IS_IBS_OP_NB(x)) 272 ret = (x - IBS_OP_NB_BASE + IBS_OP_LS_MAX + IBS_OP_MAX + IBS_FETCH_MAX); 273 274 return (ret != ~0UL) ? ret + OP_MAX_COUNTERS : ret; 275 } 276 277 278 void opd_log_ibs_event(unsigned int event, 279 struct transient * trans) 280 { 281 ibs_derived_event_stats++; 282 trans->event = event; 283 sfile_log_sample_count(trans, 1); 284 } 285 286 287 void opd_log_ibs_count(unsigned int event, 288 struct transient * trans, 289 unsigned int count) 290 { 291 ibs_derived_event_stats++; 292 trans->event = event; 293 sfile_log_sample_count(trans, count); 294 } 295 296 297 static unsigned long get_ibs_vci_key(unsigned int event) 298 { 299 unsigned long key = ibs_event_to_counter(event); 300 if (key == ~0UL || key < OP_MAX_COUNTERS) 301 return ~0UL; 302 303 key = key - OP_MAX_COUNTERS; 304 305 return key; 306 } 307 308 309 static int ibs_parse_and_set_events(char * str) 310 { 311 char * tmp, * ptr, * tok1, * tok2 = NULL; 312 int is_done = 0; 313 struct op_event * event = NULL; 314 op_cpu cpu_type = CPU_NO_GOOD; 315 unsigned long key; 316 317 if (!str) 318 return -1; 319 320 cpu_type = op_get_cpu_type(); 321 op_events(cpu_type); 322 323 tmp = op_xstrndup(str, strlen(str)); 324 ptr = tmp; 325 326 while (is_done != 1 327 && (tok1 = strtok_r(ptr, ",", &tok2)) != NULL) { 328 329 if ((ptr = strstr(tok1, ":")) != NULL) { 330 *ptr = '\0'; 331 is_done = 1; 332 } 333 334 // Resove event number 335 event = find_event_by_name(tok1, 0, 0); 336 if (!event) 337 return -1; 338 339 // Grouping 340 if (IS_IBS_FETCH(event->val)) { 341 ibs_fetch_selected_flag |= 1 << IBS_FETCH_OFFSET(event->val); 342 ibs_fetch_selected_size++; 343 } else if (IS_IBS_OP(event->val)) { 344 ibs_op_selected_flag |= 1 << IBS_OP_OFFSET(event->val); 345 ibs_op_selected_size++; 346 } else if (IS_IBS_OP_LS(event->val)) { 347 ibs_op_ls_selected_flag |= 1 << IBS_OP_LS_OFFSET(event->val); 348 ibs_op_ls_selected_size++; 349 } else if (IS_IBS_OP_NB(event->val)) { 350 ibs_op_nb_selected_flag |= 1 << IBS_OP_NB_OFFSET(event->val); 351 ibs_op_nb_selected_size++; 352 } else { 353 return -1; 354 } 355 356 key = get_ibs_vci_key(event->val); 357 if (key == ~0UL) 358 return -1; 359 360 ibs_vci_map[key] = ibs_selected_size; 361 362 /* Initialize part of ibs_vc */ 363 ibs_vc[ibs_selected_size].name = tok1; 364 ibs_vc[ibs_selected_size].value = event->val; 365 ibs_vc[ibs_selected_size].counter = ibs_selected_size + OP_MAX_COUNTERS; 366 ibs_vc[ibs_selected_size].kernel = 1; 367 ibs_vc[ibs_selected_size].user = 1; 368 369 ibs_selected_size++; 370 371 ptr = NULL; 372 } 373 374 return 0; 375 } 376 377 378 static int ibs_parse_counts(char * str, unsigned long int * count) 379 { 380 char * tmp, * tok1, * tok2 = NULL, *end = NULL; 381 if (!str) 382 return -1; 383 384 tmp = op_xstrndup(str, strlen(str)); 385 tok1 = strtok_r(tmp, ":", &tok2); 386 *count = strtoul(tok1, &end, 10); 387 if ((end && *end) || *count == 0 388 || errno == EINVAL || errno == ERANGE) { 389 fprintf(stderr,"Invalid count (%s)\n", str); 390 return -1; 391 } 392 393 return 0; 394 } 395 396 397 static int ibs_parse_and_set_um_fetch(char const * str) 398 { 399 if (!str) 400 return -1; 401 return 0; 402 } 403 404 405 406 static int ibs_parse_and_set_um_op(char const * str, unsigned long int * ibs_op_um) 407 { 408 char * end = NULL; 409 if (!str) 410 return -1; 411 412 *ibs_op_um = strtoul(str, &end, 16); 413 if ((end && *end) || errno == EINVAL || errno == ERANGE) { 414 fprintf(stderr,"Invalid unitmaks (%s)\n", str); 415 return -1; 416 } 417 return 0; 418 } 419 420 421 static int ibs_init(char const * argv) 422 { 423 char * tmp, * ptr, * tok1, * tok2 = NULL; 424 unsigned int i = 0; 425 unsigned long int ibs_fetch_count = 0; 426 unsigned long int ibs_op_count = 0; 427 unsigned long int ibs_op_um = 0; 428 429 if (!argv) 430 return -1; 431 432 if (empty_line(argv) != 0) 433 return -1; 434 435 tmp = op_xstrndup(argv, strlen(argv)); 436 ptr = (char *) skip_ws(tmp); 437 438 // "fetch:event1,event2,....:count:um|op:event1,event2,.....:count:um" 439 tok1 = strtok_r(ptr, "|", &tok2); 440 441 while (tok1 != NULL) { 442 443 if (!strncmp("fetch:", tok1, strlen("fetch:"))) { 444 // Get to event section 445 tok1 = tok1 + strlen("fetch:"); 446 if (ibs_parse_and_set_events(tok1) == -1) 447 return -1; 448 449 // Get to count section 450 while (tok1) { 451 if (*tok1 == '\0') 452 return -1; 453 if (*tok1 != ':') { 454 tok1++; 455 } else { 456 tok1++; 457 break; 458 } 459 } 460 461 if (ibs_parse_counts(tok1, &ibs_fetch_count) == -1) 462 return -1; 463 464 // Get to um section 465 while (tok1) { 466 if (*tok1 == '\0') 467 return -1; 468 if (*tok1 != ':') { 469 tok1++; 470 } else { 471 tok1++; 472 break; 473 } 474 } 475 476 if (ibs_parse_and_set_um_fetch(tok1) == -1) 477 return -1; 478 479 } else if (!strncmp("op:", tok1, strlen("op:"))) { 480 // Get to event section 481 tok1 = tok1 + strlen("op:"); 482 if (ibs_parse_and_set_events(tok1) == -1) 483 return -1; 484 485 // Get to count section 486 while (tok1) { 487 if (*tok1 == '\0') 488 return -1; 489 if (*tok1 != ':') { 490 tok1++; 491 } else { 492 tok1++; 493 break; 494 } 495 } 496 497 if (ibs_parse_counts(tok1, &ibs_op_count) == -1) 498 return -1; 499 500 // Get to um section 501 while (tok1) { 502 if (*tok1 == '\0') 503 return -1; 504 if (*tok1 != ':') { 505 tok1++; 506 } else { 507 tok1++; 508 break; 509 } 510 } 511 512 if (ibs_parse_and_set_um_op(tok1, &ibs_op_um)) 513 return -1; 514 515 } else 516 return -1; 517 518 tok1 = strtok_r(NULL, "|", &tok2); 519 } 520 521 /* Initialize ibs_vc */ 522 for (i = 0 ; i < ibs_selected_size ; i++) 523 { 524 if (IS_IBS_FETCH(ibs_vc[i].value)) { 525 ibs_vc[i].count = ibs_fetch_count; 526 ibs_vc[i].um = 0; 527 } else { 528 ibs_vc[i].count = ibs_op_count; 529 ibs_vc[i].um = ibs_op_um; 530 } 531 } 532 533 // Allow no event 534 no_event_ok = 1; 535 return 0; 536 } 537 538 539 static int ibs_print_stats() 540 { 541 printf("Nr. IBS Fetch samples : %lu (%lu entries)\n", ibs_fetch_sample_stats, (ibs_fetch_sample_stats * 7)); 542 printf("Nr. IBS Fetch incompletes : %lu\n", ibs_fetch_incomplete_stats); 543 printf("Nr. IBS Op samples : %lu (%lu entries)\n", ibs_op_sample_stats, (ibs_op_sample_stats * 13)); 544 printf("Nr. IBS Op incompletes : %lu\n", ibs_op_incomplete_stats); 545 printf("Nr. IBS derived events : %lu\n", ibs_derived_event_stats); 546 return 0; 547 } 548 549 550 static int ibs_sfile_create(struct sfile * sf) 551 { 552 unsigned int i; 553 sf->ext_files = xmalloc(ibs_selected_size * sizeof(odb_t)); 554 for (i = 0 ; i < ibs_selected_size ; ++i) 555 odb_init(&sf->ext_files[i]); 556 557 return 0; 558 } 559 560 561 static int ibs_sfile_dup (struct sfile * to, struct sfile * from) 562 { 563 unsigned int i; 564 if (from->ext_files != NULL) { 565 to->ext_files = xmalloc(ibs_selected_size * sizeof(odb_t)); 566 for (i = 0 ; i < ibs_selected_size ; ++i) 567 odb_init(&to->ext_files[i]); 568 } else { 569 to->ext_files = NULL; 570 } 571 return 0; 572 } 573 574 static int ibs_sfile_close(struct sfile * sf) 575 { 576 unsigned int i; 577 if (sf->ext_files != NULL) { 578 for (i = 0; i < ibs_selected_size ; ++i) 579 odb_close(&sf->ext_files[i]); 580 581 free(sf->ext_files); 582 sf->ext_files= NULL; 583 } 584 return 0; 585 } 586 587 static int ibs_sfile_sync(struct sfile * sf) 588 { 589 unsigned int i; 590 if (sf->ext_files != NULL) { 591 for (i = 0; i < ibs_selected_size ; ++i) 592 odb_sync(&sf->ext_files[i]); 593 } 594 return 0; 595 } 596 597 static odb_t * ibs_sfile_get(struct transient const * trans, int is_cg) 598 { 599 struct sfile * sf = trans->current; 600 struct sfile * last = trans->last; 601 struct cg_entry * cg; 602 struct list_head * pos; 603 unsigned long hash; 604 odb_t * file; 605 unsigned long counter, ibs_vci, key; 606 607 /* Note: "trans->event" for IBS is not the same as traditional 608 * events. Here, it has the actual event (0xfxxx), while the 609 * traditional event has the event index. 610 */ 611 key = get_ibs_vci_key(trans->event); 612 if (key == ~0UL) { 613 fprintf(stderr, "%s: Invalid IBS event %lu\n", __func__, trans->event); 614 abort(); 615 } 616 ibs_vci = ibs_vci_map[key]; 617 counter = ibs_vci + OP_MAX_COUNTERS; 618 619 /* Creating IBS sfile if it not already exists */ 620 if (sf->ext_files == NULL) 621 ibs_sfile_create(sf); 622 623 file = &(sf->ext_files[ibs_vci]); 624 if (!is_cg) 625 goto open; 626 627 hash = last->hashval & (CG_HASH_SIZE - 1); 628 629 /* Need to look for the right 'to'. Since we're looking for 630 * 'last', we use its hash. 631 */ 632 list_for_each(pos, &sf->cg_hash[hash]) { 633 cg = list_entry(pos, struct cg_entry, hash); 634 if (sfile_equal(last, &cg->to)) { 635 file = &(cg->to.ext_files[ibs_vci]); 636 goto open; 637 } 638 } 639 640 cg = xmalloc(sizeof(struct cg_entry)); 641 sfile_dup(&cg->to, last); 642 list_add(&cg->hash, &sf->cg_hash[hash]); 643 file = &(cg->to.ext_files[ibs_vci]); 644 645 open: 646 if (!odb_open_count(file)) 647 opd_open_sample_file(file, last, sf, counter, is_cg); 648 649 /* Error is logged by opd_open_sample_file */ 650 if (!odb_open_count(file)) 651 return NULL; 652 653 return file; 654 } 655 656 657 /** Filled opd_event structure with IBS derived event information 658 * from the given counter value. 659 */ 660 static struct opd_event * ibs_sfile_find_counter_event(unsigned long counter) 661 { 662 unsigned long ibs_vci; 663 664 if (counter >= OP_MAX_COUNTERS + OP_MAX_IBS_COUNTERS 665 || counter < OP_MAX_COUNTERS) { 666 fprintf(stderr,"Error: find_ibs_counter_event : " 667 "invalid counter value %lu.\n", counter); 668 abort(); 669 } 670 671 ibs_vci = counter - OP_MAX_COUNTERS; 672 return &ibs_vc[ibs_vci]; 673 } 674 675 676 struct opd_ext_sfile_handlers ibs_sfile_handlers = 677 { 678 .create = &ibs_sfile_create, 679 .dup = &ibs_sfile_dup, 680 .close = &ibs_sfile_close, 681 .sync = &ibs_sfile_sync, 682 .get = &ibs_sfile_get, 683 .find_counter_event = &ibs_sfile_find_counter_event 684 }; 685 686 687 struct opd_ext_handlers ibs_handlers = 688 { 689 .ext_init = &ibs_init, 690 .ext_print_stats = &ibs_print_stats, 691 .ext_sfile = &ibs_sfile_handlers 692 }; 693
