1 /* 2 * IO verification helpers 3 */ 4 #include <unistd.h> 5 #include <fcntl.h> 6 #include <string.h> 7 #include <assert.h> 8 #include <pthread.h> 9 #include <libgen.h> 10 11 #include "fio.h" 12 #include "verify.h" 13 #include "trim.h" 14 #include "lib/rand.h" 15 #include "lib/hweight.h" 16 17 #include "crc/md5.h" 18 #include "crc/crc64.h" 19 #include "crc/crc32.h" 20 #include "crc/crc32c.h" 21 #include "crc/crc16.h" 22 #include "crc/crc7.h" 23 #include "crc/sha256.h" 24 #include "crc/sha512.h" 25 #include "crc/sha1.h" 26 #include "crc/xxhash.h" 27 28 static void populate_hdr(struct thread_data *td, struct io_u *io_u, 29 struct verify_header *hdr, unsigned int header_num, 30 unsigned int header_len); 31 32 static void fill_pattern(struct thread_data *td, void *p, unsigned int len, 33 char *pattern, unsigned int pattern_bytes) 34 { 35 switch (pattern_bytes) { 36 case 0: 37 assert(0); 38 break; 39 case 1: 40 dprint(FD_VERIFY, "fill verify pattern b=0 len=%u\n", len); 41 memset(p, pattern[0], len); 42 break; 43 default: { 44 unsigned int i = 0, size = 0; 45 unsigned char *b = p; 46 47 dprint(FD_VERIFY, "fill verify pattern b=%d len=%u\n", 48 pattern_bytes, len); 49 50 while (i < len) { 51 size = pattern_bytes; 52 if (size > (len - i)) 53 size = len - i; 54 memcpy(b+i, pattern, size); 55 i += size; 56 } 57 break; 58 } 59 } 60 } 61 62 void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 63 { 64 fill_pattern(td, p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 65 } 66 67 void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 68 struct io_u *io_u, unsigned long seed, int use_seed) 69 { 70 if (!td->o.verify_pattern_bytes) { 71 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 72 73 if (use_seed) 74 __fill_random_buf(p, len, seed); 75 else 76 io_u->rand_seed = fill_random_buf(&td->__verify_state, p, len); 77 return; 78 } 79 80 if (io_u->buf_filled_len >= len) { 81 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 82 td->o.verify_pattern_bytes, len); 83 return; 84 } 85 86 fill_pattern(td, p, len, td->o.verify_pattern, td->o.verify_pattern_bytes); 87 88 io_u->buf_filled_len = len; 89 } 90 91 static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 92 { 93 unsigned int hdr_inc; 94 95 hdr_inc = io_u->buflen; 96 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 97 hdr_inc = td->o.verify_interval; 98 99 return hdr_inc; 100 } 101 102 static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 103 unsigned long seed, int use_seed) 104 { 105 unsigned int hdr_inc, header_num; 106 struct verify_header *hdr; 107 void *p = io_u->buf; 108 109 fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 110 111 hdr_inc = get_hdr_inc(td, io_u); 112 header_num = 0; 113 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 114 hdr = p; 115 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 116 header_num++; 117 } 118 } 119 120 static void memswp(void *buf1, void *buf2, unsigned int len) 121 { 122 char swap[200]; 123 124 assert(len <= sizeof(swap)); 125 126 memcpy(&swap, buf1, len); 127 memcpy(buf1, buf2, len); 128 memcpy(buf2, &swap, len); 129 } 130 131 static void hexdump(void *buffer, int len) 132 { 133 unsigned char *p = buffer; 134 int i; 135 136 for (i = 0; i < len; i++) 137 log_err("%02x", p[i]); 138 log_err("\n"); 139 } 140 141 /* 142 * Prepare for separation of verify_header and checksum header 143 */ 144 static inline unsigned int __hdr_size(int verify_type) 145 { 146 unsigned int len = 0; 147 148 switch (verify_type) { 149 case VERIFY_NONE: 150 case VERIFY_NULL: 151 len = 0; 152 break; 153 case VERIFY_MD5: 154 len = sizeof(struct vhdr_md5); 155 break; 156 case VERIFY_CRC64: 157 len = sizeof(struct vhdr_crc64); 158 break; 159 case VERIFY_CRC32C: 160 case VERIFY_CRC32: 161 case VERIFY_CRC32C_INTEL: 162 len = sizeof(struct vhdr_crc32); 163 break; 164 case VERIFY_CRC16: 165 len = sizeof(struct vhdr_crc16); 166 break; 167 case VERIFY_CRC7: 168 len = sizeof(struct vhdr_crc7); 169 break; 170 case VERIFY_SHA256: 171 len = sizeof(struct vhdr_sha256); 172 break; 173 case VERIFY_SHA512: 174 len = sizeof(struct vhdr_sha512); 175 break; 176 case VERIFY_XXHASH: 177 len = sizeof(struct vhdr_xxhash); 178 break; 179 case VERIFY_META: 180 len = sizeof(struct vhdr_meta); 181 break; 182 case VERIFY_SHA1: 183 len = sizeof(struct vhdr_sha1); 184 break; 185 case VERIFY_PATTERN: 186 len = 0; 187 break; 188 default: 189 log_err("fio: unknown verify header!\n"); 190 assert(0); 191 } 192 193 return len + sizeof(struct verify_header); 194 } 195 196 static inline unsigned int hdr_size(struct verify_header *hdr) 197 { 198 return __hdr_size(hdr->verify_type); 199 } 200 201 static void *hdr_priv(struct verify_header *hdr) 202 { 203 void *priv = hdr; 204 205 return priv + sizeof(struct verify_header); 206 } 207 208 /* 209 * Verify container, pass info to verify handlers and allow them to 210 * pass info back in case of error 211 */ 212 struct vcont { 213 /* 214 * Input 215 */ 216 struct io_u *io_u; 217 unsigned int hdr_num; 218 struct thread_data *td; 219 220 /* 221 * Output, only valid in case of error 222 */ 223 const char *name; 224 void *good_crc; 225 void *bad_crc; 226 unsigned int crc_len; 227 }; 228 229 #define DUMP_BUF_SZ 255 230 static int dump_buf_warned; 231 232 static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 233 const char *type, struct fio_file *f) 234 { 235 char *ptr, fname[DUMP_BUF_SZ]; 236 size_t buf_left = DUMP_BUF_SZ; 237 int ret, fd; 238 239 ptr = strdup(f->file_name); 240 241 fname[DUMP_BUF_SZ - 1] = '\0'; 242 strncpy(fname, basename(ptr), DUMP_BUF_SZ - 1); 243 244 buf_left -= strlen(fname); 245 if (buf_left <= 0) { 246 if (!dump_buf_warned) { 247 log_err("fio: verify failure dump buffer too small\n"); 248 dump_buf_warned = 1; 249 } 250 free(ptr); 251 return; 252 } 253 254 snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type); 255 256 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 257 if (fd < 0) { 258 perror("open verify buf file"); 259 return; 260 } 261 262 while (len) { 263 ret = write(fd, buf, len); 264 if (!ret) 265 break; 266 else if (ret < 0) { 267 perror("write verify buf file"); 268 break; 269 } 270 len -= ret; 271 buf += ret; 272 } 273 274 close(fd); 275 log_err(" %s data dumped as %s\n", type, fname); 276 free(ptr); 277 } 278 279 /* 280 * Dump the contents of the read block and re-generate the correct data 281 * and dump that too. 282 */ 283 static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 284 { 285 struct thread_data *td = vc->td; 286 struct io_u *io_u = vc->io_u; 287 unsigned long hdr_offset; 288 struct io_u dummy; 289 void *buf; 290 291 if (!td->o.verify_dump) 292 return; 293 294 /* 295 * Dump the contents we just read off disk 296 */ 297 hdr_offset = vc->hdr_num * hdr->len; 298 299 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 300 "received", vc->io_u->file); 301 302 /* 303 * Allocate a new buf and re-generate the original data 304 */ 305 buf = malloc(io_u->buflen); 306 dummy = *io_u; 307 dummy.buf = buf; 308 dummy.rand_seed = hdr->rand_seed; 309 dummy.buf_filled_len = 0; 310 dummy.buflen = io_u->buflen; 311 312 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 313 314 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 315 "expected", vc->io_u->file); 316 free(buf); 317 } 318 319 static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 320 { 321 unsigned long long offset; 322 323 offset = vc->io_u->offset; 324 offset += vc->hdr_num * hdr->len; 325 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 326 vc->name, vc->io_u->file->file_name, offset, hdr->len); 327 328 if (vc->good_crc && vc->bad_crc) { 329 log_err(" Expected CRC: "); 330 hexdump(vc->good_crc, vc->crc_len); 331 log_err(" Received CRC: "); 332 hexdump(vc->bad_crc, vc->crc_len); 333 } 334 335 dump_verify_buffers(hdr, vc); 336 } 337 338 /* 339 * Return data area 'header_num' 340 */ 341 static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 342 { 343 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 344 } 345 346 static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 347 { 348 struct thread_data *td = vc->td; 349 struct io_u *io_u = vc->io_u; 350 char *buf, *pattern; 351 unsigned int header_size = __hdr_size(td->o.verify); 352 unsigned int len, mod, i, size, pattern_size; 353 354 pattern = td->o.verify_pattern; 355 pattern_size = td->o.verify_pattern_bytes; 356 if (pattern_size <= 1) 357 pattern_size = MAX_PATTERN_SIZE; 358 buf = (void *) hdr + header_size; 359 len = get_hdr_inc(td, io_u) - header_size; 360 mod = header_size % pattern_size; 361 362 for (i = 0; i < len; i += size) { 363 size = pattern_size - mod; 364 if (size > (len - i)) 365 size = len - i; 366 if (memcmp(buf + i, pattern + mod, size)) 367 /* Let the slow compare find the first mismatch byte. */ 368 break; 369 mod = 0; 370 } 371 372 for (; i < len; i++) { 373 if (buf[i] != pattern[mod]) { 374 unsigned int bits; 375 376 bits = hweight8(buf[i] ^ pattern[mod]); 377 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 378 buf[i], pattern[mod], bits); 379 log_err("fio: bad pattern block offset %u\n", i); 380 dump_verify_buffers(hdr, vc); 381 return EILSEQ; 382 } 383 mod++; 384 if (mod == td->o.verify_pattern_bytes) 385 mod = 0; 386 } 387 388 return 0; 389 } 390 391 static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 392 { 393 struct thread_data *td = vc->td; 394 struct vhdr_meta *vh = hdr_priv(hdr); 395 struct io_u *io_u = vc->io_u; 396 int ret = EILSEQ; 397 398 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 399 400 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 401 ret = 0; 402 403 if (td->o.verify_pattern_bytes) 404 ret |= verify_io_u_pattern(hdr, vc); 405 406 /* 407 * For read-only workloads, the program cannot be certain of the 408 * last numberio written to a block. Checking of numberio will be done 409 * only for workloads that write data. 410 * For verify_only, numberio will be checked in the last iteration when 411 * the correct state of numberio, that would have been written to each 412 * block in a previous run of fio, has been reached. 413 */ 414 if (td_write(td) || td_rw(td)) 415 if (!td->o.verify_only || td->o.loops == 0) 416 if (vh->numberio != io_u->numberio) 417 ret = EILSEQ; 418 419 if (!ret) 420 return 0; 421 422 vc->name = "meta"; 423 log_verify_failure(hdr, vc); 424 return ret; 425 } 426 427 static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc) 428 { 429 void *p = io_u_verify_off(hdr, vc); 430 struct vhdr_xxhash *vh = hdr_priv(hdr); 431 uint32_t hash; 432 void *state; 433 434 dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len); 435 436 state = XXH32_init(1); 437 XXH32_update(state, p, hdr->len - hdr_size(hdr)); 438 hash = XXH32_digest(state); 439 440 if (vh->hash == hash) 441 return 0; 442 443 vc->name = "xxhash"; 444 vc->good_crc = &vh->hash; 445 vc->bad_crc = &hash; 446 vc->crc_len = sizeof(hash); 447 log_verify_failure(hdr, vc); 448 return EILSEQ; 449 } 450 451 static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 452 { 453 void *p = io_u_verify_off(hdr, vc); 454 struct vhdr_sha512 *vh = hdr_priv(hdr); 455 uint8_t sha512[128]; 456 struct fio_sha512_ctx sha512_ctx = { 457 .buf = sha512, 458 }; 459 460 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 461 462 fio_sha512_init(&sha512_ctx); 463 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 464 465 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 466 return 0; 467 468 vc->name = "sha512"; 469 vc->good_crc = vh->sha512; 470 vc->bad_crc = sha512_ctx.buf; 471 vc->crc_len = sizeof(vh->sha512); 472 log_verify_failure(hdr, vc); 473 return EILSEQ; 474 } 475 476 static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 477 { 478 void *p = io_u_verify_off(hdr, vc); 479 struct vhdr_sha256 *vh = hdr_priv(hdr); 480 uint8_t sha256[64]; 481 struct fio_sha256_ctx sha256_ctx = { 482 .buf = sha256, 483 }; 484 485 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 486 487 fio_sha256_init(&sha256_ctx); 488 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 489 490 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 491 return 0; 492 493 vc->name = "sha256"; 494 vc->good_crc = vh->sha256; 495 vc->bad_crc = sha256_ctx.buf; 496 vc->crc_len = sizeof(vh->sha256); 497 log_verify_failure(hdr, vc); 498 return EILSEQ; 499 } 500 501 static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 502 { 503 void *p = io_u_verify_off(hdr, vc); 504 struct vhdr_sha1 *vh = hdr_priv(hdr); 505 uint32_t sha1[5]; 506 struct fio_sha1_ctx sha1_ctx = { 507 .H = sha1, 508 }; 509 510 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 511 512 fio_sha1_init(&sha1_ctx); 513 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 514 515 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 516 return 0; 517 518 vc->name = "sha1"; 519 vc->good_crc = vh->sha1; 520 vc->bad_crc = sha1_ctx.H; 521 vc->crc_len = sizeof(vh->sha1); 522 log_verify_failure(hdr, vc); 523 return EILSEQ; 524 } 525 526 static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 527 { 528 void *p = io_u_verify_off(hdr, vc); 529 struct vhdr_crc7 *vh = hdr_priv(hdr); 530 unsigned char c; 531 532 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 533 534 c = fio_crc7(p, hdr->len - hdr_size(hdr)); 535 536 if (c == vh->crc7) 537 return 0; 538 539 vc->name = "crc7"; 540 vc->good_crc = &vh->crc7; 541 vc->bad_crc = &c; 542 vc->crc_len = 1; 543 log_verify_failure(hdr, vc); 544 return EILSEQ; 545 } 546 547 static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 548 { 549 void *p = io_u_verify_off(hdr, vc); 550 struct vhdr_crc16 *vh = hdr_priv(hdr); 551 unsigned short c; 552 553 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 554 555 c = fio_crc16(p, hdr->len - hdr_size(hdr)); 556 557 if (c == vh->crc16) 558 return 0; 559 560 vc->name = "crc16"; 561 vc->good_crc = &vh->crc16; 562 vc->bad_crc = &c; 563 vc->crc_len = 2; 564 log_verify_failure(hdr, vc); 565 return EILSEQ; 566 } 567 568 static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 569 { 570 void *p = io_u_verify_off(hdr, vc); 571 struct vhdr_crc64 *vh = hdr_priv(hdr); 572 unsigned long long c; 573 574 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 575 576 c = fio_crc64(p, hdr->len - hdr_size(hdr)); 577 578 if (c == vh->crc64) 579 return 0; 580 581 vc->name = "crc64"; 582 vc->good_crc = &vh->crc64; 583 vc->bad_crc = &c; 584 vc->crc_len = 8; 585 log_verify_failure(hdr, vc); 586 return EILSEQ; 587 } 588 589 static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 590 { 591 void *p = io_u_verify_off(hdr, vc); 592 struct vhdr_crc32 *vh = hdr_priv(hdr); 593 uint32_t c; 594 595 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 596 597 c = fio_crc32(p, hdr->len - hdr_size(hdr)); 598 599 if (c == vh->crc32) 600 return 0; 601 602 vc->name = "crc32"; 603 vc->good_crc = &vh->crc32; 604 vc->bad_crc = &c; 605 vc->crc_len = 4; 606 log_verify_failure(hdr, vc); 607 return EILSEQ; 608 } 609 610 static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 611 { 612 void *p = io_u_verify_off(hdr, vc); 613 struct vhdr_crc32 *vh = hdr_priv(hdr); 614 uint32_t c; 615 616 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 617 618 c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 619 620 if (c == vh->crc32) 621 return 0; 622 623 vc->name = "crc32c"; 624 vc->good_crc = &vh->crc32; 625 vc->bad_crc = &c; 626 vc->crc_len = 4; 627 log_verify_failure(hdr, vc); 628 return EILSEQ; 629 } 630 631 static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 632 { 633 void *p = io_u_verify_off(hdr, vc); 634 struct vhdr_md5 *vh = hdr_priv(hdr); 635 uint32_t hash[MD5_HASH_WORDS]; 636 struct fio_md5_ctx md5_ctx = { 637 .hash = hash, 638 }; 639 640 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 641 642 fio_md5_init(&md5_ctx); 643 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 644 645 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 646 return 0; 647 648 vc->name = "md5"; 649 vc->good_crc = vh->md5_digest; 650 vc->bad_crc = md5_ctx.hash; 651 vc->crc_len = sizeof(hash); 652 log_verify_failure(hdr, vc); 653 return EILSEQ; 654 } 655 656 /* 657 * Push IO verification to a separate thread 658 */ 659 int verify_io_u_async(struct thread_data *td, struct io_u *io_u) 660 { 661 if (io_u->file) 662 put_file_log(td, io_u->file); 663 664 pthread_mutex_lock(&td->io_u_lock); 665 666 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 667 td->cur_depth--; 668 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 669 } 670 flist_add_tail(&io_u->verify_list, &td->verify_list); 671 io_u->flags |= IO_U_F_FREE_DEF; 672 pthread_mutex_unlock(&td->io_u_lock); 673 674 pthread_cond_signal(&td->verify_cond); 675 return 0; 676 } 677 678 static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 679 { 680 static char zero_buf[1024]; 681 unsigned int this_len, len; 682 int ret = 0; 683 void *p; 684 685 if (!td->o.trim_zero) 686 return 0; 687 688 len = io_u->buflen; 689 p = io_u->buf; 690 do { 691 this_len = sizeof(zero_buf); 692 if (this_len > len) 693 this_len = len; 694 if (memcmp(p, zero_buf, this_len)) { 695 ret = EILSEQ; 696 break; 697 } 698 len -= this_len; 699 p += this_len; 700 } while (len); 701 702 if (!ret) 703 return 0; 704 705 log_err("trim: verify failed at file %s offset %llu, length %lu" 706 ", block offset %lu\n", 707 io_u->file->file_name, io_u->offset, io_u->buflen, 708 (unsigned long) (p - io_u->buf)); 709 return ret; 710 } 711 712 static int verify_header(struct io_u *io_u, struct verify_header *hdr) 713 { 714 void *p = hdr; 715 uint32_t crc; 716 717 if (hdr->magic != FIO_HDR_MAGIC) 718 return 1; 719 if (hdr->len > io_u->buflen) 720 return 2; 721 if (hdr->rand_seed != io_u->rand_seed) 722 return 3; 723 724 crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 725 if (crc == hdr->crc32) 726 return 0; 727 log_err("fio: verify header crc %x, calculated %x\n", hdr->crc32, crc); 728 return 4; 729 } 730 731 int verify_io_u(struct thread_data *td, struct io_u *io_u) 732 { 733 struct verify_header *hdr; 734 unsigned int header_size, hdr_inc, hdr_num = 0; 735 void *p; 736 int ret; 737 738 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 739 return 0; 740 if (io_u->flags & IO_U_F_TRIMMED) { 741 ret = verify_trimmed_io_u(td, io_u); 742 goto done; 743 } 744 745 hdr_inc = get_hdr_inc(td, io_u); 746 747 ret = 0; 748 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 749 p += hdr_inc, hdr_num++) { 750 struct vcont vc = { 751 .io_u = io_u, 752 .hdr_num = hdr_num, 753 .td = td, 754 }; 755 unsigned int verify_type; 756 757 if (ret && td->o.verify_fatal) 758 break; 759 760 header_size = __hdr_size(td->o.verify); 761 if (td->o.verify_offset) 762 memswp(p, p + td->o.verify_offset, header_size); 763 hdr = p; 764 765 /* 766 * Make rand_seed check pass when have verifysort or 767 * verify_backlog. 768 */ 769 if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 770 io_u->rand_seed = hdr->rand_seed; 771 772 ret = verify_header(io_u, hdr); 773 switch (ret) { 774 case 0: 775 break; 776 case 1: 777 log_err("verify: bad magic header %x, wanted %x at " 778 "file %s offset %llu, length %u\n", 779 hdr->magic, FIO_HDR_MAGIC, 780 io_u->file->file_name, 781 io_u->offset + hdr_num * hdr->len, hdr->len); 782 return EILSEQ; 783 break; 784 case 2: 785 log_err("fio: verify header exceeds buffer length (%u " 786 "> %lu)\n", hdr->len, io_u->buflen); 787 return EILSEQ; 788 break; 789 case 3: 790 log_err("verify: bad header rand_seed %"PRIu64 791 ", wanted %"PRIu64" at file %s offset %llu, " 792 "length %u\n", 793 hdr->rand_seed, io_u->rand_seed, 794 io_u->file->file_name, 795 io_u->offset + hdr_num * hdr->len, hdr->len); 796 return EILSEQ; 797 break; 798 case 4: 799 return EILSEQ; 800 break; 801 default: 802 log_err("verify: unknown header error at file %s " 803 "offset %llu, length %u\n", 804 io_u->file->file_name, 805 io_u->offset + hdr_num * hdr->len, hdr->len); 806 return EILSEQ; 807 } 808 809 if (td->o.verify != VERIFY_NONE) 810 verify_type = td->o.verify; 811 else 812 verify_type = hdr->verify_type; 813 814 switch (verify_type) { 815 case VERIFY_MD5: 816 ret = verify_io_u_md5(hdr, &vc); 817 break; 818 case VERIFY_CRC64: 819 ret = verify_io_u_crc64(hdr, &vc); 820 break; 821 case VERIFY_CRC32C: 822 case VERIFY_CRC32C_INTEL: 823 ret = verify_io_u_crc32c(hdr, &vc); 824 break; 825 case VERIFY_CRC32: 826 ret = verify_io_u_crc32(hdr, &vc); 827 break; 828 case VERIFY_CRC16: 829 ret = verify_io_u_crc16(hdr, &vc); 830 break; 831 case VERIFY_CRC7: 832 ret = verify_io_u_crc7(hdr, &vc); 833 break; 834 case VERIFY_SHA256: 835 ret = verify_io_u_sha256(hdr, &vc); 836 break; 837 case VERIFY_SHA512: 838 ret = verify_io_u_sha512(hdr, &vc); 839 break; 840 case VERIFY_XXHASH: 841 ret = verify_io_u_xxhash(hdr, &vc); 842 break; 843 case VERIFY_META: 844 ret = verify_io_u_meta(hdr, &vc); 845 break; 846 case VERIFY_SHA1: 847 ret = verify_io_u_sha1(hdr, &vc); 848 break; 849 case VERIFY_PATTERN: 850 ret = verify_io_u_pattern(hdr, &vc); 851 break; 852 default: 853 log_err("Bad verify type %u\n", hdr->verify_type); 854 ret = EINVAL; 855 } 856 857 if (ret && verify_type != hdr->verify_type) 858 log_err("fio: verify type mismatch (%u media, %u given)\n", 859 hdr->verify_type, verify_type); 860 } 861 862 done: 863 if (ret && td->o.verify_fatal) 864 td->terminate = 1; 865 866 return ret; 867 } 868 869 static void fill_meta(struct verify_header *hdr, struct thread_data *td, 870 struct io_u *io_u, unsigned int header_num) 871 { 872 struct vhdr_meta *vh = hdr_priv(hdr); 873 874 vh->thread = td->thread_number; 875 876 vh->time_sec = io_u->start_time.tv_sec; 877 vh->time_usec = io_u->start_time.tv_usec; 878 879 vh->numberio = io_u->numberio; 880 881 vh->offset = io_u->offset + header_num * td->o.verify_interval; 882 } 883 884 static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len) 885 { 886 struct vhdr_xxhash *vh = hdr_priv(hdr); 887 void *state; 888 889 state = XXH32_init(1); 890 XXH32_update(state, p, len); 891 vh->hash = XXH32_digest(state); 892 } 893 894 static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 895 { 896 struct vhdr_sha512 *vh = hdr_priv(hdr); 897 struct fio_sha512_ctx sha512_ctx = { 898 .buf = vh->sha512, 899 }; 900 901 fio_sha512_init(&sha512_ctx); 902 fio_sha512_update(&sha512_ctx, p, len); 903 } 904 905 static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 906 { 907 struct vhdr_sha256 *vh = hdr_priv(hdr); 908 struct fio_sha256_ctx sha256_ctx = { 909 .buf = vh->sha256, 910 }; 911 912 fio_sha256_init(&sha256_ctx); 913 fio_sha256_update(&sha256_ctx, p, len); 914 } 915 916 static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 917 { 918 struct vhdr_sha1 *vh = hdr_priv(hdr); 919 struct fio_sha1_ctx sha1_ctx = { 920 .H = vh->sha1, 921 }; 922 923 fio_sha1_init(&sha1_ctx); 924 fio_sha1_update(&sha1_ctx, p, len); 925 } 926 927 static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 928 { 929 struct vhdr_crc7 *vh = hdr_priv(hdr); 930 931 vh->crc7 = fio_crc7(p, len); 932 } 933 934 static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 935 { 936 struct vhdr_crc16 *vh = hdr_priv(hdr); 937 938 vh->crc16 = fio_crc16(p, len); 939 } 940 941 static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 942 { 943 struct vhdr_crc32 *vh = hdr_priv(hdr); 944 945 vh->crc32 = fio_crc32(p, len); 946 } 947 948 static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 949 { 950 struct vhdr_crc32 *vh = hdr_priv(hdr); 951 952 vh->crc32 = fio_crc32c(p, len); 953 } 954 955 static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 956 { 957 struct vhdr_crc64 *vh = hdr_priv(hdr); 958 959 vh->crc64 = fio_crc64(p, len); 960 } 961 962 static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 963 { 964 struct vhdr_md5 *vh = hdr_priv(hdr); 965 struct fio_md5_ctx md5_ctx = { 966 .hash = (uint32_t *) vh->md5_digest, 967 }; 968 969 fio_md5_init(&md5_ctx); 970 fio_md5_update(&md5_ctx, p, len); 971 } 972 973 static void populate_hdr(struct thread_data *td, struct io_u *io_u, 974 struct verify_header *hdr, unsigned int header_num, 975 unsigned int header_len) 976 { 977 unsigned int data_len; 978 void *data, *p; 979 980 p = (void *) hdr; 981 982 hdr->magic = FIO_HDR_MAGIC; 983 hdr->verify_type = td->o.verify; 984 hdr->len = header_len; 985 hdr->rand_seed = io_u->rand_seed; 986 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 987 988 data_len = header_len - hdr_size(hdr); 989 990 data = p + hdr_size(hdr); 991 switch (td->o.verify) { 992 case VERIFY_MD5: 993 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 994 io_u, hdr->len); 995 fill_md5(hdr, data, data_len); 996 break; 997 case VERIFY_CRC64: 998 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 999 io_u, hdr->len); 1000 fill_crc64(hdr, data, data_len); 1001 break; 1002 case VERIFY_CRC32C: 1003 case VERIFY_CRC32C_INTEL: 1004 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 1005 io_u, hdr->len); 1006 fill_crc32c(hdr, data, data_len); 1007 break; 1008 case VERIFY_CRC32: 1009 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 1010 io_u, hdr->len); 1011 fill_crc32(hdr, data, data_len); 1012 break; 1013 case VERIFY_CRC16: 1014 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 1015 io_u, hdr->len); 1016 fill_crc16(hdr, data, data_len); 1017 break; 1018 case VERIFY_CRC7: 1019 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 1020 io_u, hdr->len); 1021 fill_crc7(hdr, data, data_len); 1022 break; 1023 case VERIFY_SHA256: 1024 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 1025 io_u, hdr->len); 1026 fill_sha256(hdr, data, data_len); 1027 break; 1028 case VERIFY_SHA512: 1029 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 1030 io_u, hdr->len); 1031 fill_sha512(hdr, data, data_len); 1032 break; 1033 case VERIFY_XXHASH: 1034 dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", 1035 io_u, hdr->len); 1036 fill_xxhash(hdr, data, data_len); 1037 break; 1038 case VERIFY_META: 1039 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 1040 io_u, hdr->len); 1041 fill_meta(hdr, td, io_u, header_num); 1042 break; 1043 case VERIFY_SHA1: 1044 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 1045 io_u, hdr->len); 1046 fill_sha1(hdr, data, data_len); 1047 break; 1048 case VERIFY_PATTERN: 1049 /* nothing to do here */ 1050 break; 1051 default: 1052 log_err("fio: bad verify type: %d\n", td->o.verify); 1053 assert(0); 1054 } 1055 if (td->o.verify_offset) 1056 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 1057 } 1058 1059 /* 1060 * fill body of io_u->buf with random data and add a header with the 1061 * checksum of choice 1062 */ 1063 void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1064 { 1065 if (td->o.verify == VERIFY_NULL) 1066 return; 1067 1068 io_u->numberio = td->io_issues[io_u->ddir]; 1069 1070 fill_pattern_headers(td, io_u, 0, 0); 1071 } 1072 1073 int get_next_verify(struct thread_data *td, struct io_u *io_u) 1074 { 1075 struct io_piece *ipo = NULL; 1076 1077 /* 1078 * this io_u is from a requeue, we already filled the offsets 1079 */ 1080 if (io_u->file) 1081 return 0; 1082 1083 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 1084 struct rb_node *n = rb_first(&td->io_hist_tree); 1085 1086 ipo = rb_entry(n, struct io_piece, rb_node); 1087 1088 /* 1089 * Ensure that the associated IO has completed 1090 */ 1091 read_barrier(); 1092 if (ipo->flags & IP_F_IN_FLIGHT) 1093 goto nothing; 1094 1095 rb_erase(n, &td->io_hist_tree); 1096 assert(ipo->flags & IP_F_ONRB); 1097 ipo->flags &= ~IP_F_ONRB; 1098 } else if (!flist_empty(&td->io_hist_list)) { 1099 ipo = flist_entry(td->io_hist_list.next, struct io_piece, list); 1100 1101 /* 1102 * Ensure that the associated IO has completed 1103 */ 1104 read_barrier(); 1105 if (ipo->flags & IP_F_IN_FLIGHT) 1106 goto nothing; 1107 1108 flist_del(&ipo->list); 1109 assert(ipo->flags & IP_F_ONLIST); 1110 ipo->flags &= ~IP_F_ONLIST; 1111 } 1112 1113 if (ipo) { 1114 td->io_hist_len--; 1115 1116 io_u->offset = ipo->offset; 1117 io_u->buflen = ipo->len; 1118 io_u->numberio = ipo->numberio; 1119 io_u->file = ipo->file; 1120 io_u->flags |= IO_U_F_VER_LIST; 1121 1122 if (ipo->flags & IP_F_TRIMMED) 1123 io_u->flags |= IO_U_F_TRIMMED; 1124 1125 if (!fio_file_open(io_u->file)) { 1126 int r = td_io_open_file(td, io_u->file); 1127 1128 if (r) { 1129 dprint(FD_VERIFY, "failed file %s open\n", 1130 io_u->file->file_name); 1131 return 1; 1132 } 1133 } 1134 1135 get_file(ipo->file); 1136 assert(fio_file_open(io_u->file)); 1137 io_u->ddir = DDIR_READ; 1138 io_u->xfer_buf = io_u->buf; 1139 io_u->xfer_buflen = io_u->buflen; 1140 1141 remove_trim_entry(td, ipo); 1142 free(ipo); 1143 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1144 1145 if (!td->o.verify_pattern_bytes) { 1146 io_u->rand_seed = __rand(&td->__verify_state); 1147 if (sizeof(int) != sizeof(long *)) 1148 io_u->rand_seed *= __rand(&td->__verify_state); 1149 } 1150 return 0; 1151 } 1152 1153 nothing: 1154 dprint(FD_VERIFY, "get_next_verify: empty\n"); 1155 return 1; 1156 } 1157 1158 void fio_verify_init(struct thread_data *td) 1159 { 1160 if (td->o.verify == VERIFY_CRC32C_INTEL || 1161 td->o.verify == VERIFY_CRC32C) { 1162 crc32c_intel_probe(); 1163 } 1164 } 1165 1166 static void *verify_async_thread(void *data) 1167 { 1168 struct thread_data *td = data; 1169 struct io_u *io_u; 1170 int ret = 0; 1171 1172 if (td->o.verify_cpumask_set && 1173 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1174 log_err("fio: failed setting verify thread affinity\n"); 1175 goto done; 1176 } 1177 1178 do { 1179 FLIST_HEAD(list); 1180 1181 read_barrier(); 1182 if (td->verify_thread_exit) 1183 break; 1184 1185 pthread_mutex_lock(&td->io_u_lock); 1186 1187 while (flist_empty(&td->verify_list) && 1188 !td->verify_thread_exit) { 1189 ret = pthread_cond_wait(&td->verify_cond, 1190 &td->io_u_lock); 1191 if (ret) { 1192 pthread_mutex_unlock(&td->io_u_lock); 1193 break; 1194 } 1195 } 1196 1197 flist_splice_init(&td->verify_list, &list); 1198 pthread_mutex_unlock(&td->io_u_lock); 1199 1200 if (flist_empty(&list)) 1201 continue; 1202 1203 while (!flist_empty(&list)) { 1204 io_u = flist_entry(list.next, struct io_u, verify_list); 1205 flist_del(&io_u->verify_list); 1206 1207 ret = verify_io_u(td, io_u); 1208 put_io_u(td, io_u); 1209 if (!ret) 1210 continue; 1211 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1212 update_error_count(td, ret); 1213 td_clear_error(td); 1214 ret = 0; 1215 } 1216 } 1217 } while (!ret); 1218 1219 if (ret) { 1220 td_verror(td, ret, "async_verify"); 1221 if (td->o.verify_fatal) 1222 td->terminate = 1; 1223 } 1224 1225 done: 1226 pthread_mutex_lock(&td->io_u_lock); 1227 td->nr_verify_threads--; 1228 pthread_mutex_unlock(&td->io_u_lock); 1229 1230 pthread_cond_signal(&td->free_cond); 1231 return NULL; 1232 } 1233 1234 int verify_async_init(struct thread_data *td) 1235 { 1236 int i, ret; 1237 pthread_attr_t attr; 1238 1239 pthread_attr_init(&attr); 1240 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1241 1242 td->verify_thread_exit = 0; 1243 1244 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1245 for (i = 0; i < td->o.verify_async; i++) { 1246 ret = pthread_create(&td->verify_threads[i], &attr, 1247 verify_async_thread, td); 1248 if (ret) { 1249 log_err("fio: async verify creation failed: %s\n", 1250 strerror(ret)); 1251 break; 1252 } 1253 ret = pthread_detach(td->verify_threads[i]); 1254 if (ret) { 1255 log_err("fio: async verify thread detach failed: %s\n", 1256 strerror(ret)); 1257 break; 1258 } 1259 td->nr_verify_threads++; 1260 } 1261 1262 pthread_attr_destroy(&attr); 1263 1264 if (i != td->o.verify_async) { 1265 log_err("fio: only %d verify threads started, exiting\n", i); 1266 td->verify_thread_exit = 1; 1267 write_barrier(); 1268 pthread_cond_broadcast(&td->verify_cond); 1269 return 1; 1270 } 1271 1272 return 0; 1273 } 1274 1275 void verify_async_exit(struct thread_data *td) 1276 { 1277 td->verify_thread_exit = 1; 1278 write_barrier(); 1279 pthread_cond_broadcast(&td->verify_cond); 1280 1281 pthread_mutex_lock(&td->io_u_lock); 1282 1283 while (td->nr_verify_threads) 1284 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1285 1286 pthread_mutex_unlock(&td->io_u_lock); 1287 free(td->verify_threads); 1288 td->verify_threads = NULL; 1289 } 1290
