1 /* 2 * IO verification helpers 3 */ 4 #include <unistd.h> 5 #include <fcntl.h> 6 #include <string.h> 7 #include <assert.h> 8 #include <pthread.h> 9 #include <libgen.h> 10 11 #include "fio.h" 12 #include "verify.h" 13 #include "trim.h" 14 #include "lib/rand.h" 15 #include "lib/hweight.h" 16 17 #include "crc/md5.h" 18 #include "crc/crc64.h" 19 #include "crc/crc32.h" 20 #include "crc/crc32c.h" 21 #include "crc/crc16.h" 22 #include "crc/crc7.h" 23 #include "crc/sha256.h" 24 #include "crc/sha512.h" 25 #include "crc/sha1.h" 26 #include "crc/xxhash.h" 27 28 static void populate_hdr(struct thread_data *td, struct io_u *io_u, 29 struct verify_header *hdr, unsigned int header_num, 30 unsigned int header_len); 31 32 void fill_buffer_pattern(struct thread_data *td, void *p, unsigned int len) 33 { 34 fill_pattern(p, len, td->o.buffer_pattern, td->o.buffer_pattern_bytes); 35 } 36 37 void __fill_buffer(struct thread_options *o, unsigned long seed, void *p, 38 unsigned int len) 39 { 40 __fill_random_buf_percentage(seed, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); 41 } 42 43 unsigned long fill_buffer(struct thread_data *td, void *p, unsigned int len) 44 { 45 struct frand_state *fs = &td->verify_state; 46 struct thread_options *o = &td->o; 47 48 return fill_random_buf_percentage(fs, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); 49 } 50 51 void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len, 52 struct io_u *io_u, unsigned long seed, int use_seed) 53 { 54 struct thread_options *o = &td->o; 55 56 if (!o->verify_pattern_bytes) { 57 dprint(FD_VERIFY, "fill random bytes len=%u\n", len); 58 59 if (use_seed) 60 __fill_buffer(o, seed, p, len); 61 else 62 io_u->rand_seed = fill_buffer(td, p, len); 63 return; 64 } 65 66 if (io_u->buf_filled_len >= len) { 67 dprint(FD_VERIFY, "using already filled verify pattern b=%d len=%u\n", 68 o->verify_pattern_bytes, len); 69 return; 70 } 71 72 fill_pattern(p, len, o->verify_pattern, o->verify_pattern_bytes); 73 io_u->buf_filled_len = len; 74 } 75 76 static unsigned int get_hdr_inc(struct thread_data *td, struct io_u *io_u) 77 { 78 unsigned int hdr_inc; 79 80 hdr_inc = io_u->buflen; 81 if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) 82 hdr_inc = td->o.verify_interval; 83 84 return hdr_inc; 85 } 86 87 static void fill_pattern_headers(struct thread_data *td, struct io_u *io_u, 88 unsigned long seed, int use_seed) 89 { 90 unsigned int hdr_inc, header_num; 91 struct verify_header *hdr; 92 void *p = io_u->buf; 93 94 fill_verify_pattern(td, p, io_u->buflen, io_u, seed, use_seed); 95 96 hdr_inc = get_hdr_inc(td, io_u); 97 header_num = 0; 98 for (; p < io_u->buf + io_u->buflen; p += hdr_inc) { 99 hdr = p; 100 populate_hdr(td, io_u, hdr, header_num, hdr_inc); 101 header_num++; 102 } 103 } 104 105 static void memswp(void *buf1, void *buf2, unsigned int len) 106 { 107 char swap[200]; 108 109 assert(len <= sizeof(swap)); 110 111 memcpy(&swap, buf1, len); 112 memcpy(buf1, buf2, len); 113 memcpy(buf2, &swap, len); 114 } 115 116 static void hexdump(void *buffer, int len) 117 { 118 unsigned char *p = buffer; 119 int i; 120 121 for (i = 0; i < len; i++) 122 log_err("%02x", p[i]); 123 log_err("\n"); 124 } 125 126 /* 127 * Prepare for separation of verify_header and checksum header 128 */ 129 static inline unsigned int __hdr_size(int verify_type) 130 { 131 unsigned int len = 0; 132 133 switch (verify_type) { 134 case VERIFY_NONE: 135 case VERIFY_NULL: 136 len = 0; 137 break; 138 case VERIFY_MD5: 139 len = sizeof(struct vhdr_md5); 140 break; 141 case VERIFY_CRC64: 142 len = sizeof(struct vhdr_crc64); 143 break; 144 case VERIFY_CRC32C: 145 case VERIFY_CRC32: 146 case VERIFY_CRC32C_INTEL: 147 len = sizeof(struct vhdr_crc32); 148 break; 149 case VERIFY_CRC16: 150 len = sizeof(struct vhdr_crc16); 151 break; 152 case VERIFY_CRC7: 153 len = sizeof(struct vhdr_crc7); 154 break; 155 case VERIFY_SHA256: 156 len = sizeof(struct vhdr_sha256); 157 break; 158 case VERIFY_SHA512: 159 len = sizeof(struct vhdr_sha512); 160 break; 161 case VERIFY_XXHASH: 162 len = sizeof(struct vhdr_xxhash); 163 break; 164 case VERIFY_META: 165 len = sizeof(struct vhdr_meta); 166 break; 167 case VERIFY_SHA1: 168 len = sizeof(struct vhdr_sha1); 169 break; 170 case VERIFY_PATTERN: 171 len = 0; 172 break; 173 default: 174 log_err("fio: unknown verify header!\n"); 175 assert(0); 176 } 177 178 return len + sizeof(struct verify_header); 179 } 180 181 static inline unsigned int hdr_size(struct verify_header *hdr) 182 { 183 return __hdr_size(hdr->verify_type); 184 } 185 186 static void *hdr_priv(struct verify_header *hdr) 187 { 188 void *priv = hdr; 189 190 return priv + sizeof(struct verify_header); 191 } 192 193 /* 194 * Verify container, pass info to verify handlers and allow them to 195 * pass info back in case of error 196 */ 197 struct vcont { 198 /* 199 * Input 200 */ 201 struct io_u *io_u; 202 unsigned int hdr_num; 203 struct thread_data *td; 204 205 /* 206 * Output, only valid in case of error 207 */ 208 const char *name; 209 void *good_crc; 210 void *bad_crc; 211 unsigned int crc_len; 212 }; 213 214 #define DUMP_BUF_SZ 255 215 static int dump_buf_warned; 216 217 static void dump_buf(char *buf, unsigned int len, unsigned long long offset, 218 const char *type, struct fio_file *f) 219 { 220 char *ptr, fname[DUMP_BUF_SZ]; 221 size_t buf_left = DUMP_BUF_SZ; 222 int ret, fd; 223 224 ptr = strdup(f->file_name); 225 226 fname[DUMP_BUF_SZ - 1] = '\0'; 227 strncpy(fname, basename(ptr), DUMP_BUF_SZ - 1); 228 229 buf_left -= strlen(fname); 230 if (buf_left <= 0) { 231 if (!dump_buf_warned) { 232 log_err("fio: verify failure dump buffer too small\n"); 233 dump_buf_warned = 1; 234 } 235 free(ptr); 236 return; 237 } 238 239 snprintf(fname + strlen(fname), buf_left, ".%llu.%s", offset, type); 240 241 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644); 242 if (fd < 0) { 243 perror("open verify buf file"); 244 return; 245 } 246 247 while (len) { 248 ret = write(fd, buf, len); 249 if (!ret) 250 break; 251 else if (ret < 0) { 252 perror("write verify buf file"); 253 break; 254 } 255 len -= ret; 256 buf += ret; 257 } 258 259 close(fd); 260 log_err(" %s data dumped as %s\n", type, fname); 261 free(ptr); 262 } 263 264 /* 265 * Dump the contents of the read block and re-generate the correct data 266 * and dump that too. 267 */ 268 static void dump_verify_buffers(struct verify_header *hdr, struct vcont *vc) 269 { 270 struct thread_data *td = vc->td; 271 struct io_u *io_u = vc->io_u; 272 unsigned long hdr_offset; 273 struct io_u dummy; 274 void *buf; 275 276 if (!td->o.verify_dump) 277 return; 278 279 /* 280 * Dump the contents we just read off disk 281 */ 282 hdr_offset = vc->hdr_num * hdr->len; 283 284 dump_buf(io_u->buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 285 "received", vc->io_u->file); 286 287 /* 288 * Allocate a new buf and re-generate the original data 289 */ 290 buf = malloc(io_u->buflen); 291 dummy = *io_u; 292 dummy.buf = buf; 293 dummy.rand_seed = hdr->rand_seed; 294 dummy.buf_filled_len = 0; 295 dummy.buflen = io_u->buflen; 296 297 fill_pattern_headers(td, &dummy, hdr->rand_seed, 1); 298 299 dump_buf(buf + hdr_offset, hdr->len, io_u->offset + hdr_offset, 300 "expected", vc->io_u->file); 301 free(buf); 302 } 303 304 static void log_verify_failure(struct verify_header *hdr, struct vcont *vc) 305 { 306 unsigned long long offset; 307 308 offset = vc->io_u->offset; 309 offset += vc->hdr_num * hdr->len; 310 log_err("%.8s: verify failed at file %s offset %llu, length %u\n", 311 vc->name, vc->io_u->file->file_name, offset, hdr->len); 312 313 if (vc->good_crc && vc->bad_crc) { 314 log_err(" Expected CRC: "); 315 hexdump(vc->good_crc, vc->crc_len); 316 log_err(" Received CRC: "); 317 hexdump(vc->bad_crc, vc->crc_len); 318 } 319 320 dump_verify_buffers(hdr, vc); 321 } 322 323 /* 324 * Return data area 'header_num' 325 */ 326 static inline void *io_u_verify_off(struct verify_header *hdr, struct vcont *vc) 327 { 328 return vc->io_u->buf + vc->hdr_num * hdr->len + hdr_size(hdr); 329 } 330 331 static int verify_io_u_pattern(struct verify_header *hdr, struct vcont *vc) 332 { 333 struct thread_data *td = vc->td; 334 struct io_u *io_u = vc->io_u; 335 char *buf, *pattern; 336 unsigned int header_size = __hdr_size(td->o.verify); 337 unsigned int len, mod, i, size, pattern_size; 338 339 pattern = td->o.verify_pattern; 340 pattern_size = td->o.verify_pattern_bytes; 341 if (pattern_size <= 1) 342 pattern_size = MAX_PATTERN_SIZE; 343 buf = (void *) hdr + header_size; 344 len = get_hdr_inc(td, io_u) - header_size; 345 mod = header_size % pattern_size; 346 347 for (i = 0; i < len; i += size) { 348 size = pattern_size - mod; 349 if (size > (len - i)) 350 size = len - i; 351 if (memcmp(buf + i, pattern + mod, size)) 352 /* Let the slow compare find the first mismatch byte. */ 353 break; 354 mod = 0; 355 } 356 357 for (; i < len; i++) { 358 if (buf[i] != pattern[mod]) { 359 unsigned int bits; 360 361 bits = hweight8(buf[i] ^ pattern[mod]); 362 log_err("fio: got pattern %x, wanted %x. Bad bits %d\n", 363 buf[i], pattern[mod], bits); 364 log_err("fio: bad pattern block offset %u\n", i); 365 dump_verify_buffers(hdr, vc); 366 return EILSEQ; 367 } 368 mod++; 369 if (mod == td->o.verify_pattern_bytes) 370 mod = 0; 371 } 372 373 return 0; 374 } 375 376 static int verify_io_u_meta(struct verify_header *hdr, struct vcont *vc) 377 { 378 struct thread_data *td = vc->td; 379 struct vhdr_meta *vh = hdr_priv(hdr); 380 struct io_u *io_u = vc->io_u; 381 int ret = EILSEQ; 382 383 dprint(FD_VERIFY, "meta verify io_u %p, len %u\n", io_u, hdr->len); 384 385 if (vh->offset == io_u->offset + vc->hdr_num * td->o.verify_interval) 386 ret = 0; 387 388 if (td->o.verify_pattern_bytes) 389 ret |= verify_io_u_pattern(hdr, vc); 390 391 /* 392 * For read-only workloads, the program cannot be certain of the 393 * last numberio written to a block. Checking of numberio will be 394 * done only for workloads that write data. For verify_only, 395 * numberio will be checked in the last iteration when the correct 396 * state of numberio, that would have been written to each block 397 * in a previous run of fio, has been reached. 398 */ 399 if ((td_write(td) || td_rw(td)) && (td_min_bs(td) == td_max_bs(td)) && 400 !td->o.time_based) 401 if (!td->o.verify_only || td->o.loops == 0) 402 if (vh->numberio != io_u->numberio) 403 ret = EILSEQ; 404 405 if (!ret) 406 return 0; 407 408 vc->name = "meta"; 409 log_verify_failure(hdr, vc); 410 return ret; 411 } 412 413 static int verify_io_u_xxhash(struct verify_header *hdr, struct vcont *vc) 414 { 415 void *p = io_u_verify_off(hdr, vc); 416 struct vhdr_xxhash *vh = hdr_priv(hdr); 417 uint32_t hash; 418 void *state; 419 420 dprint(FD_VERIFY, "xxhash verify io_u %p, len %u\n", vc->io_u, hdr->len); 421 422 state = XXH32_init(1); 423 XXH32_update(state, p, hdr->len - hdr_size(hdr)); 424 hash = XXH32_digest(state); 425 426 if (vh->hash == hash) 427 return 0; 428 429 vc->name = "xxhash"; 430 vc->good_crc = &vh->hash; 431 vc->bad_crc = &hash; 432 vc->crc_len = sizeof(hash); 433 log_verify_failure(hdr, vc); 434 return EILSEQ; 435 } 436 437 static int verify_io_u_sha512(struct verify_header *hdr, struct vcont *vc) 438 { 439 void *p = io_u_verify_off(hdr, vc); 440 struct vhdr_sha512 *vh = hdr_priv(hdr); 441 uint8_t sha512[128]; 442 struct fio_sha512_ctx sha512_ctx = { 443 .buf = sha512, 444 }; 445 446 dprint(FD_VERIFY, "sha512 verify io_u %p, len %u\n", vc->io_u, hdr->len); 447 448 fio_sha512_init(&sha512_ctx); 449 fio_sha512_update(&sha512_ctx, p, hdr->len - hdr_size(hdr)); 450 451 if (!memcmp(vh->sha512, sha512_ctx.buf, sizeof(sha512))) 452 return 0; 453 454 vc->name = "sha512"; 455 vc->good_crc = vh->sha512; 456 vc->bad_crc = sha512_ctx.buf; 457 vc->crc_len = sizeof(vh->sha512); 458 log_verify_failure(hdr, vc); 459 return EILSEQ; 460 } 461 462 static int verify_io_u_sha256(struct verify_header *hdr, struct vcont *vc) 463 { 464 void *p = io_u_verify_off(hdr, vc); 465 struct vhdr_sha256 *vh = hdr_priv(hdr); 466 uint8_t sha256[64]; 467 struct fio_sha256_ctx sha256_ctx = { 468 .buf = sha256, 469 }; 470 471 dprint(FD_VERIFY, "sha256 verify io_u %p, len %u\n", vc->io_u, hdr->len); 472 473 fio_sha256_init(&sha256_ctx); 474 fio_sha256_update(&sha256_ctx, p, hdr->len - hdr_size(hdr)); 475 fio_sha256_final(&sha256_ctx); 476 477 if (!memcmp(vh->sha256, sha256_ctx.buf, sizeof(sha256))) 478 return 0; 479 480 vc->name = "sha256"; 481 vc->good_crc = vh->sha256; 482 vc->bad_crc = sha256_ctx.buf; 483 vc->crc_len = sizeof(vh->sha256); 484 log_verify_failure(hdr, vc); 485 return EILSEQ; 486 } 487 488 static int verify_io_u_sha1(struct verify_header *hdr, struct vcont *vc) 489 { 490 void *p = io_u_verify_off(hdr, vc); 491 struct vhdr_sha1 *vh = hdr_priv(hdr); 492 uint32_t sha1[5]; 493 struct fio_sha1_ctx sha1_ctx = { 494 .H = sha1, 495 }; 496 497 dprint(FD_VERIFY, "sha1 verify io_u %p, len %u\n", vc->io_u, hdr->len); 498 499 fio_sha1_init(&sha1_ctx); 500 fio_sha1_update(&sha1_ctx, p, hdr->len - hdr_size(hdr)); 501 fio_sha1_final(&sha1_ctx); 502 503 if (!memcmp(vh->sha1, sha1_ctx.H, sizeof(sha1))) 504 return 0; 505 506 vc->name = "sha1"; 507 vc->good_crc = vh->sha1; 508 vc->bad_crc = sha1_ctx.H; 509 vc->crc_len = sizeof(vh->sha1); 510 log_verify_failure(hdr, vc); 511 return EILSEQ; 512 } 513 514 static int verify_io_u_crc7(struct verify_header *hdr, struct vcont *vc) 515 { 516 void *p = io_u_verify_off(hdr, vc); 517 struct vhdr_crc7 *vh = hdr_priv(hdr); 518 unsigned char c; 519 520 dprint(FD_VERIFY, "crc7 verify io_u %p, len %u\n", vc->io_u, hdr->len); 521 522 c = fio_crc7(p, hdr->len - hdr_size(hdr)); 523 524 if (c == vh->crc7) 525 return 0; 526 527 vc->name = "crc7"; 528 vc->good_crc = &vh->crc7; 529 vc->bad_crc = &c; 530 vc->crc_len = 1; 531 log_verify_failure(hdr, vc); 532 return EILSEQ; 533 } 534 535 static int verify_io_u_crc16(struct verify_header *hdr, struct vcont *vc) 536 { 537 void *p = io_u_verify_off(hdr, vc); 538 struct vhdr_crc16 *vh = hdr_priv(hdr); 539 unsigned short c; 540 541 dprint(FD_VERIFY, "crc16 verify io_u %p, len %u\n", vc->io_u, hdr->len); 542 543 c = fio_crc16(p, hdr->len - hdr_size(hdr)); 544 545 if (c == vh->crc16) 546 return 0; 547 548 vc->name = "crc16"; 549 vc->good_crc = &vh->crc16; 550 vc->bad_crc = &c; 551 vc->crc_len = 2; 552 log_verify_failure(hdr, vc); 553 return EILSEQ; 554 } 555 556 static int verify_io_u_crc64(struct verify_header *hdr, struct vcont *vc) 557 { 558 void *p = io_u_verify_off(hdr, vc); 559 struct vhdr_crc64 *vh = hdr_priv(hdr); 560 unsigned long long c; 561 562 dprint(FD_VERIFY, "crc64 verify io_u %p, len %u\n", vc->io_u, hdr->len); 563 564 c = fio_crc64(p, hdr->len - hdr_size(hdr)); 565 566 if (c == vh->crc64) 567 return 0; 568 569 vc->name = "crc64"; 570 vc->good_crc = &vh->crc64; 571 vc->bad_crc = &c; 572 vc->crc_len = 8; 573 log_verify_failure(hdr, vc); 574 return EILSEQ; 575 } 576 577 static int verify_io_u_crc32(struct verify_header *hdr, struct vcont *vc) 578 { 579 void *p = io_u_verify_off(hdr, vc); 580 struct vhdr_crc32 *vh = hdr_priv(hdr); 581 uint32_t c; 582 583 dprint(FD_VERIFY, "crc32 verify io_u %p, len %u\n", vc->io_u, hdr->len); 584 585 c = fio_crc32(p, hdr->len - hdr_size(hdr)); 586 587 if (c == vh->crc32) 588 return 0; 589 590 vc->name = "crc32"; 591 vc->good_crc = &vh->crc32; 592 vc->bad_crc = &c; 593 vc->crc_len = 4; 594 log_verify_failure(hdr, vc); 595 return EILSEQ; 596 } 597 598 static int verify_io_u_crc32c(struct verify_header *hdr, struct vcont *vc) 599 { 600 void *p = io_u_verify_off(hdr, vc); 601 struct vhdr_crc32 *vh = hdr_priv(hdr); 602 uint32_t c; 603 604 dprint(FD_VERIFY, "crc32c verify io_u %p, len %u\n", vc->io_u, hdr->len); 605 606 c = fio_crc32c(p, hdr->len - hdr_size(hdr)); 607 608 if (c == vh->crc32) 609 return 0; 610 611 vc->name = "crc32c"; 612 vc->good_crc = &vh->crc32; 613 vc->bad_crc = &c; 614 vc->crc_len = 4; 615 log_verify_failure(hdr, vc); 616 return EILSEQ; 617 } 618 619 static int verify_io_u_md5(struct verify_header *hdr, struct vcont *vc) 620 { 621 void *p = io_u_verify_off(hdr, vc); 622 struct vhdr_md5 *vh = hdr_priv(hdr); 623 uint32_t hash[MD5_HASH_WORDS]; 624 struct fio_md5_ctx md5_ctx = { 625 .hash = hash, 626 }; 627 628 dprint(FD_VERIFY, "md5 verify io_u %p, len %u\n", vc->io_u, hdr->len); 629 630 fio_md5_init(&md5_ctx); 631 fio_md5_update(&md5_ctx, p, hdr->len - hdr_size(hdr)); 632 fio_md5_final(&md5_ctx); 633 634 if (!memcmp(vh->md5_digest, md5_ctx.hash, sizeof(hash))) 635 return 0; 636 637 vc->name = "md5"; 638 vc->good_crc = vh->md5_digest; 639 vc->bad_crc = md5_ctx.hash; 640 vc->crc_len = sizeof(hash); 641 log_verify_failure(hdr, vc); 642 return EILSEQ; 643 } 644 645 /* 646 * Push IO verification to a separate thread 647 */ 648 int verify_io_u_async(struct thread_data *td, struct io_u **io_u_ptr) 649 { 650 struct io_u *io_u = *io_u_ptr; 651 652 pthread_mutex_lock(&td->io_u_lock); 653 654 if (io_u->file) 655 put_file_log(td, io_u->file); 656 657 if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { 658 td->cur_depth--; 659 io_u->flags &= ~IO_U_F_IN_CUR_DEPTH; 660 } 661 flist_add_tail(&io_u->verify_list, &td->verify_list); 662 *io_u_ptr = NULL; 663 pthread_mutex_unlock(&td->io_u_lock); 664 665 pthread_cond_signal(&td->verify_cond); 666 return 0; 667 } 668 669 static int verify_trimmed_io_u(struct thread_data *td, struct io_u *io_u) 670 { 671 static char zero_buf[1024]; 672 unsigned int this_len, len; 673 int ret = 0; 674 void *p; 675 676 if (!td->o.trim_zero) 677 return 0; 678 679 len = io_u->buflen; 680 p = io_u->buf; 681 do { 682 this_len = sizeof(zero_buf); 683 if (this_len > len) 684 this_len = len; 685 if (memcmp(p, zero_buf, this_len)) { 686 ret = EILSEQ; 687 break; 688 } 689 len -= this_len; 690 p += this_len; 691 } while (len); 692 693 if (!ret) 694 return 0; 695 696 log_err("trim: verify failed at file %s offset %llu, length %lu" 697 ", block offset %lu\n", 698 io_u->file->file_name, io_u->offset, io_u->buflen, 699 (unsigned long) (p - io_u->buf)); 700 return ret; 701 } 702 703 static int verify_header(struct io_u *io_u, struct verify_header *hdr, 704 unsigned int hdr_num, unsigned int hdr_len) 705 { 706 void *p = hdr; 707 uint32_t crc; 708 709 if (hdr->magic != FIO_HDR_MAGIC) { 710 log_err("verify: bad magic header %x, wanted %x", 711 hdr->magic, FIO_HDR_MAGIC); 712 goto err; 713 } 714 if (hdr->len != hdr_len) { 715 log_err("verify: bad header length %u, wanted %u", 716 hdr->len, hdr_len); 717 goto err; 718 } 719 if (hdr->rand_seed != io_u->rand_seed) { 720 log_err("verify: bad header rand_seed %"PRIu64 721 ", wanted %"PRIu64, 722 hdr->rand_seed, io_u->rand_seed); 723 goto err; 724 } 725 726 crc = fio_crc32c(p, offsetof(struct verify_header, crc32)); 727 if (crc != hdr->crc32) { 728 log_err("verify: bad header crc %x, calculated %x", 729 hdr->crc32, crc); 730 goto err; 731 } 732 return 0; 733 734 err: 735 log_err(" at file %s offset %llu, length %u\n", 736 io_u->file->file_name, 737 io_u->offset + hdr_num * hdr_len, hdr_len); 738 return EILSEQ; 739 } 740 741 int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr) 742 { 743 struct verify_header *hdr; 744 struct io_u *io_u = *io_u_ptr; 745 unsigned int header_size, hdr_inc, hdr_num = 0; 746 void *p; 747 int ret; 748 749 if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ) 750 return 0; 751 /* 752 * If the IO engine is faking IO (like null), then just pretend 753 * we verified everything. 754 */ 755 if (td->io_ops->flags & FIO_FAKEIO) 756 return 0; 757 758 if (io_u->flags & IO_U_F_TRIMMED) { 759 ret = verify_trimmed_io_u(td, io_u); 760 goto done; 761 } 762 763 hdr_inc = get_hdr_inc(td, io_u); 764 765 ret = 0; 766 for (p = io_u->buf; p < io_u->buf + io_u->buflen; 767 p += hdr_inc, hdr_num++) { 768 struct vcont vc = { 769 .io_u = io_u, 770 .hdr_num = hdr_num, 771 .td = td, 772 }; 773 unsigned int verify_type; 774 775 if (ret && td->o.verify_fatal) 776 break; 777 778 header_size = __hdr_size(td->o.verify); 779 if (td->o.verify_offset) 780 memswp(p, p + td->o.verify_offset, header_size); 781 hdr = p; 782 783 /* 784 * Make rand_seed check pass when have verifysort or 785 * verify_backlog. 786 */ 787 if (td->o.verifysort || (td->flags & TD_F_VER_BACKLOG)) 788 io_u->rand_seed = hdr->rand_seed; 789 790 ret = verify_header(io_u, hdr, hdr_num, hdr_inc); 791 if (ret) 792 return ret; 793 794 if (td->o.verify != VERIFY_NONE) 795 verify_type = td->o.verify; 796 else 797 verify_type = hdr->verify_type; 798 799 switch (verify_type) { 800 case VERIFY_MD5: 801 ret = verify_io_u_md5(hdr, &vc); 802 break; 803 case VERIFY_CRC64: 804 ret = verify_io_u_crc64(hdr, &vc); 805 break; 806 case VERIFY_CRC32C: 807 case VERIFY_CRC32C_INTEL: 808 ret = verify_io_u_crc32c(hdr, &vc); 809 break; 810 case VERIFY_CRC32: 811 ret = verify_io_u_crc32(hdr, &vc); 812 break; 813 case VERIFY_CRC16: 814 ret = verify_io_u_crc16(hdr, &vc); 815 break; 816 case VERIFY_CRC7: 817 ret = verify_io_u_crc7(hdr, &vc); 818 break; 819 case VERIFY_SHA256: 820 ret = verify_io_u_sha256(hdr, &vc); 821 break; 822 case VERIFY_SHA512: 823 ret = verify_io_u_sha512(hdr, &vc); 824 break; 825 case VERIFY_XXHASH: 826 ret = verify_io_u_xxhash(hdr, &vc); 827 break; 828 case VERIFY_META: 829 ret = verify_io_u_meta(hdr, &vc); 830 break; 831 case VERIFY_SHA1: 832 ret = verify_io_u_sha1(hdr, &vc); 833 break; 834 case VERIFY_PATTERN: 835 ret = verify_io_u_pattern(hdr, &vc); 836 break; 837 default: 838 log_err("Bad verify type %u\n", hdr->verify_type); 839 ret = EINVAL; 840 } 841 842 if (ret && verify_type != hdr->verify_type) 843 log_err("fio: verify type mismatch (%u media, %u given)\n", 844 hdr->verify_type, verify_type); 845 } 846 847 done: 848 if (ret && td->o.verify_fatal) 849 fio_mark_td_terminate(td); 850 851 return ret; 852 } 853 854 static void fill_meta(struct verify_header *hdr, struct thread_data *td, 855 struct io_u *io_u, unsigned int header_num) 856 { 857 struct vhdr_meta *vh = hdr_priv(hdr); 858 859 vh->thread = td->thread_number; 860 861 vh->time_sec = io_u->start_time.tv_sec; 862 vh->time_usec = io_u->start_time.tv_usec; 863 864 vh->numberio = io_u->numberio; 865 866 vh->offset = io_u->offset + header_num * td->o.verify_interval; 867 } 868 869 static void fill_xxhash(struct verify_header *hdr, void *p, unsigned int len) 870 { 871 struct vhdr_xxhash *vh = hdr_priv(hdr); 872 void *state; 873 874 state = XXH32_init(1); 875 XXH32_update(state, p, len); 876 vh->hash = XXH32_digest(state); 877 } 878 879 static void fill_sha512(struct verify_header *hdr, void *p, unsigned int len) 880 { 881 struct vhdr_sha512 *vh = hdr_priv(hdr); 882 struct fio_sha512_ctx sha512_ctx = { 883 .buf = vh->sha512, 884 }; 885 886 fio_sha512_init(&sha512_ctx); 887 fio_sha512_update(&sha512_ctx, p, len); 888 } 889 890 static void fill_sha256(struct verify_header *hdr, void *p, unsigned int len) 891 { 892 struct vhdr_sha256 *vh = hdr_priv(hdr); 893 struct fio_sha256_ctx sha256_ctx = { 894 .buf = vh->sha256, 895 }; 896 897 fio_sha256_init(&sha256_ctx); 898 fio_sha256_update(&sha256_ctx, p, len); 899 fio_sha256_final(&sha256_ctx); 900 } 901 902 static void fill_sha1(struct verify_header *hdr, void *p, unsigned int len) 903 { 904 struct vhdr_sha1 *vh = hdr_priv(hdr); 905 struct fio_sha1_ctx sha1_ctx = { 906 .H = vh->sha1, 907 }; 908 909 fio_sha1_init(&sha1_ctx); 910 fio_sha1_update(&sha1_ctx, p, len); 911 fio_sha1_final(&sha1_ctx); 912 } 913 914 static void fill_crc7(struct verify_header *hdr, void *p, unsigned int len) 915 { 916 struct vhdr_crc7 *vh = hdr_priv(hdr); 917 918 vh->crc7 = fio_crc7(p, len); 919 } 920 921 static void fill_crc16(struct verify_header *hdr, void *p, unsigned int len) 922 { 923 struct vhdr_crc16 *vh = hdr_priv(hdr); 924 925 vh->crc16 = fio_crc16(p, len); 926 } 927 928 static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len) 929 { 930 struct vhdr_crc32 *vh = hdr_priv(hdr); 931 932 vh->crc32 = fio_crc32(p, len); 933 } 934 935 static void fill_crc32c(struct verify_header *hdr, void *p, unsigned int len) 936 { 937 struct vhdr_crc32 *vh = hdr_priv(hdr); 938 939 vh->crc32 = fio_crc32c(p, len); 940 } 941 942 static void fill_crc64(struct verify_header *hdr, void *p, unsigned int len) 943 { 944 struct vhdr_crc64 *vh = hdr_priv(hdr); 945 946 vh->crc64 = fio_crc64(p, len); 947 } 948 949 static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) 950 { 951 struct vhdr_md5 *vh = hdr_priv(hdr); 952 struct fio_md5_ctx md5_ctx = { 953 .hash = (uint32_t *) vh->md5_digest, 954 }; 955 956 fio_md5_init(&md5_ctx); 957 fio_md5_update(&md5_ctx, p, len); 958 fio_md5_final(&md5_ctx); 959 } 960 961 static void populate_hdr(struct thread_data *td, struct io_u *io_u, 962 struct verify_header *hdr, unsigned int header_num, 963 unsigned int header_len) 964 { 965 unsigned int data_len; 966 void *data, *p; 967 968 p = (void *) hdr; 969 970 hdr->magic = FIO_HDR_MAGIC; 971 hdr->verify_type = td->o.verify; 972 hdr->len = header_len; 973 hdr->rand_seed = io_u->rand_seed; 974 hdr->crc32 = fio_crc32c(p, offsetof(struct verify_header, crc32)); 975 976 data_len = header_len - hdr_size(hdr); 977 978 data = p + hdr_size(hdr); 979 switch (td->o.verify) { 980 case VERIFY_MD5: 981 dprint(FD_VERIFY, "fill md5 io_u %p, len %u\n", 982 io_u, hdr->len); 983 fill_md5(hdr, data, data_len); 984 break; 985 case VERIFY_CRC64: 986 dprint(FD_VERIFY, "fill crc64 io_u %p, len %u\n", 987 io_u, hdr->len); 988 fill_crc64(hdr, data, data_len); 989 break; 990 case VERIFY_CRC32C: 991 case VERIFY_CRC32C_INTEL: 992 dprint(FD_VERIFY, "fill crc32c io_u %p, len %u\n", 993 io_u, hdr->len); 994 fill_crc32c(hdr, data, data_len); 995 break; 996 case VERIFY_CRC32: 997 dprint(FD_VERIFY, "fill crc32 io_u %p, len %u\n", 998 io_u, hdr->len); 999 fill_crc32(hdr, data, data_len); 1000 break; 1001 case VERIFY_CRC16: 1002 dprint(FD_VERIFY, "fill crc16 io_u %p, len %u\n", 1003 io_u, hdr->len); 1004 fill_crc16(hdr, data, data_len); 1005 break; 1006 case VERIFY_CRC7: 1007 dprint(FD_VERIFY, "fill crc7 io_u %p, len %u\n", 1008 io_u, hdr->len); 1009 fill_crc7(hdr, data, data_len); 1010 break; 1011 case VERIFY_SHA256: 1012 dprint(FD_VERIFY, "fill sha256 io_u %p, len %u\n", 1013 io_u, hdr->len); 1014 fill_sha256(hdr, data, data_len); 1015 break; 1016 case VERIFY_SHA512: 1017 dprint(FD_VERIFY, "fill sha512 io_u %p, len %u\n", 1018 io_u, hdr->len); 1019 fill_sha512(hdr, data, data_len); 1020 break; 1021 case VERIFY_XXHASH: 1022 dprint(FD_VERIFY, "fill xxhash io_u %p, len %u\n", 1023 io_u, hdr->len); 1024 fill_xxhash(hdr, data, data_len); 1025 break; 1026 case VERIFY_META: 1027 dprint(FD_VERIFY, "fill meta io_u %p, len %u\n", 1028 io_u, hdr->len); 1029 fill_meta(hdr, td, io_u, header_num); 1030 break; 1031 case VERIFY_SHA1: 1032 dprint(FD_VERIFY, "fill sha1 io_u %p, len %u\n", 1033 io_u, hdr->len); 1034 fill_sha1(hdr, data, data_len); 1035 break; 1036 case VERIFY_PATTERN: 1037 /* nothing to do here */ 1038 break; 1039 default: 1040 log_err("fio: bad verify type: %d\n", td->o.verify); 1041 assert(0); 1042 } 1043 if (td->o.verify_offset) 1044 memswp(p, p + td->o.verify_offset, hdr_size(hdr)); 1045 } 1046 1047 /* 1048 * fill body of io_u->buf with random data and add a header with the 1049 * checksum of choice 1050 */ 1051 void populate_verify_io_u(struct thread_data *td, struct io_u *io_u) 1052 { 1053 if (td->o.verify == VERIFY_NULL) 1054 return; 1055 1056 io_u->numberio = td->io_issues[io_u->ddir]; 1057 1058 fill_pattern_headers(td, io_u, 0, 0); 1059 } 1060 1061 int get_next_verify(struct thread_data *td, struct io_u *io_u) 1062 { 1063 struct io_piece *ipo = NULL; 1064 1065 /* 1066 * this io_u is from a requeue, we already filled the offsets 1067 */ 1068 if (io_u->file) 1069 return 0; 1070 1071 if (!RB_EMPTY_ROOT(&td->io_hist_tree)) { 1072 struct rb_node *n = rb_first(&td->io_hist_tree); 1073 1074 ipo = rb_entry(n, struct io_piece, rb_node); 1075 1076 /* 1077 * Ensure that the associated IO has completed 1078 */ 1079 read_barrier(); 1080 if (ipo->flags & IP_F_IN_FLIGHT) 1081 goto nothing; 1082 1083 rb_erase(n, &td->io_hist_tree); 1084 assert(ipo->flags & IP_F_ONRB); 1085 ipo->flags &= ~IP_F_ONRB; 1086 } else if (!flist_empty(&td->io_hist_list)) { 1087 ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list); 1088 1089 /* 1090 * Ensure that the associated IO has completed 1091 */ 1092 read_barrier(); 1093 if (ipo->flags & IP_F_IN_FLIGHT) 1094 goto nothing; 1095 1096 flist_del(&ipo->list); 1097 assert(ipo->flags & IP_F_ONLIST); 1098 ipo->flags &= ~IP_F_ONLIST; 1099 } 1100 1101 if (ipo) { 1102 td->io_hist_len--; 1103 1104 io_u->offset = ipo->offset; 1105 io_u->buflen = ipo->len; 1106 io_u->numberio = ipo->numberio; 1107 io_u->file = ipo->file; 1108 io_u->flags |= IO_U_F_VER_LIST; 1109 1110 if (ipo->flags & IP_F_TRIMMED) 1111 io_u->flags |= IO_U_F_TRIMMED; 1112 1113 if (!fio_file_open(io_u->file)) { 1114 int r = td_io_open_file(td, io_u->file); 1115 1116 if (r) { 1117 dprint(FD_VERIFY, "failed file %s open\n", 1118 io_u->file->file_name); 1119 return 1; 1120 } 1121 } 1122 1123 get_file(ipo->file); 1124 assert(fio_file_open(io_u->file)); 1125 io_u->ddir = DDIR_READ; 1126 io_u->xfer_buf = io_u->buf; 1127 io_u->xfer_buflen = io_u->buflen; 1128 1129 remove_trim_entry(td, ipo); 1130 free(ipo); 1131 dprint(FD_VERIFY, "get_next_verify: ret io_u %p\n", io_u); 1132 1133 if (!td->o.verify_pattern_bytes) { 1134 io_u->rand_seed = __rand(&td->verify_state); 1135 if (sizeof(int) != sizeof(long *)) 1136 io_u->rand_seed *= __rand(&td->verify_state); 1137 } 1138 return 0; 1139 } 1140 1141 nothing: 1142 dprint(FD_VERIFY, "get_next_verify: empty\n"); 1143 return 1; 1144 } 1145 1146 void fio_verify_init(struct thread_data *td) 1147 { 1148 if (td->o.verify == VERIFY_CRC32C_INTEL || 1149 td->o.verify == VERIFY_CRC32C) { 1150 crc32c_intel_probe(); 1151 } 1152 } 1153 1154 static void *verify_async_thread(void *data) 1155 { 1156 struct thread_data *td = data; 1157 struct io_u *io_u; 1158 int ret = 0; 1159 1160 if (fio_option_is_set(&td->o, verify_cpumask) && 1161 fio_setaffinity(td->pid, td->o.verify_cpumask)) { 1162 log_err("fio: failed setting verify thread affinity\n"); 1163 goto done; 1164 } 1165 1166 do { 1167 FLIST_HEAD(list); 1168 1169 read_barrier(); 1170 if (td->verify_thread_exit) 1171 break; 1172 1173 pthread_mutex_lock(&td->io_u_lock); 1174 1175 while (flist_empty(&td->verify_list) && 1176 !td->verify_thread_exit) { 1177 ret = pthread_cond_wait(&td->verify_cond, 1178 &td->io_u_lock); 1179 if (ret) { 1180 pthread_mutex_unlock(&td->io_u_lock); 1181 break; 1182 } 1183 } 1184 1185 flist_splice_init(&td->verify_list, &list); 1186 pthread_mutex_unlock(&td->io_u_lock); 1187 1188 if (flist_empty(&list)) 1189 continue; 1190 1191 while (!flist_empty(&list)) { 1192 io_u = flist_first_entry(&list, struct io_u, verify_list); 1193 flist_del_init(&io_u->verify_list); 1194 1195 io_u->flags |= IO_U_F_NO_FILE_PUT; 1196 ret = verify_io_u(td, &io_u); 1197 1198 put_io_u(td, io_u); 1199 if (!ret) 1200 continue; 1201 if (td_non_fatal_error(td, ERROR_TYPE_VERIFY_BIT, ret)) { 1202 update_error_count(td, ret); 1203 td_clear_error(td); 1204 ret = 0; 1205 } 1206 } 1207 } while (!ret); 1208 1209 if (ret) { 1210 td_verror(td, ret, "async_verify"); 1211 if (td->o.verify_fatal) 1212 fio_mark_td_terminate(td); 1213 } 1214 1215 done: 1216 pthread_mutex_lock(&td->io_u_lock); 1217 td->nr_verify_threads--; 1218 pthread_mutex_unlock(&td->io_u_lock); 1219 1220 pthread_cond_signal(&td->free_cond); 1221 return NULL; 1222 } 1223 1224 int verify_async_init(struct thread_data *td) 1225 { 1226 int i, ret; 1227 pthread_attr_t attr; 1228 1229 pthread_attr_init(&attr); 1230 pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); 1231 1232 td->verify_thread_exit = 0; 1233 1234 td->verify_threads = malloc(sizeof(pthread_t) * td->o.verify_async); 1235 for (i = 0; i < td->o.verify_async; i++) { 1236 ret = pthread_create(&td->verify_threads[i], &attr, 1237 verify_async_thread, td); 1238 if (ret) { 1239 log_err("fio: async verify creation failed: %s\n", 1240 strerror(ret)); 1241 break; 1242 } 1243 ret = pthread_detach(td->verify_threads[i]); 1244 if (ret) { 1245 log_err("fio: async verify thread detach failed: %s\n", 1246 strerror(ret)); 1247 break; 1248 } 1249 td->nr_verify_threads++; 1250 } 1251 1252 pthread_attr_destroy(&attr); 1253 1254 if (i != td->o.verify_async) { 1255 log_err("fio: only %d verify threads started, exiting\n", i); 1256 td->verify_thread_exit = 1; 1257 write_barrier(); 1258 pthread_cond_broadcast(&td->verify_cond); 1259 return 1; 1260 } 1261 1262 return 0; 1263 } 1264 1265 void verify_async_exit(struct thread_data *td) 1266 { 1267 td->verify_thread_exit = 1; 1268 write_barrier(); 1269 pthread_cond_broadcast(&td->verify_cond); 1270 1271 pthread_mutex_lock(&td->io_u_lock); 1272 1273 while (td->nr_verify_threads) 1274 pthread_cond_wait(&td->free_cond, &td->io_u_lock); 1275 1276 pthread_mutex_unlock(&td->io_u_lock); 1277 free(td->verify_threads); 1278 td->verify_threads = NULL; 1279 } 1280 1281 struct all_io_list *get_all_io_list(int save_mask, size_t *sz) 1282 { 1283 struct all_io_list *rep; 1284 struct thread_data *td; 1285 size_t depth; 1286 void *next; 1287 int i, nr; 1288 1289 compiletime_assert(sizeof(struct all_io_list) == 8, "all_io_list"); 1290 1291 /* 1292 * Calculate reply space needed. We need one 'io_state' per thread, 1293 * and the size will vary depending on depth. 1294 */ 1295 depth = 0; 1296 nr = 0; 1297 for_each_td(td, i) { 1298 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1299 continue; 1300 td->stop_io = 1; 1301 td->flags |= TD_F_VSTATE_SAVED; 1302 depth += td->o.iodepth; 1303 nr++; 1304 } 1305 1306 if (!nr) 1307 return NULL; 1308 1309 *sz = sizeof(*rep); 1310 *sz += nr * sizeof(struct thread_io_list); 1311 *sz += depth * sizeof(uint64_t); 1312 rep = malloc(*sz); 1313 1314 rep->threads = cpu_to_le64((uint64_t) nr); 1315 1316 next = &rep->state[0]; 1317 for_each_td(td, i) { 1318 struct thread_io_list *s = next; 1319 unsigned int comps; 1320 1321 if (save_mask != IO_LIST_ALL && (i + 1) != save_mask) 1322 continue; 1323 1324 if (td->last_write_comp) { 1325 int j, k; 1326 1327 if (td->io_blocks[DDIR_WRITE] < td->o.iodepth) 1328 comps = td->io_blocks[DDIR_WRITE]; 1329 else 1330 comps = td->o.iodepth; 1331 1332 k = td->last_write_idx - 1; 1333 for (j = 0; j < comps; j++) { 1334 if (k == -1) 1335 k = td->o.iodepth - 1; 1336 s->offsets[j] = cpu_to_le64(td->last_write_comp[k]); 1337 k--; 1338 } 1339 } else 1340 comps = 0; 1341 1342 s->no_comps = cpu_to_le64((uint64_t) comps); 1343 s->depth = cpu_to_le64((uint64_t) td->o.iodepth); 1344 s->numberio = cpu_to_le64((uint64_t) td->io_issues[DDIR_WRITE]); 1345 s->index = cpu_to_le64((uint64_t) i); 1346 s->rand.s[0] = cpu_to_le32(td->random_state.s1); 1347 s->rand.s[1] = cpu_to_le32(td->random_state.s2); 1348 s->rand.s[2] = cpu_to_le32(td->random_state.s3); 1349 s->rand.s[3] = 0; 1350 s->name[sizeof(s->name) - 1] = '\0'; 1351 strncpy((char *) s->name, td->o.name, sizeof(s->name) - 1); 1352 next = io_list_next(s); 1353 } 1354 1355 return rep; 1356 } 1357 1358 static int open_state_file(const char *name, const char *prefix, int num, 1359 int for_write) 1360 { 1361 char out[64]; 1362 int flags; 1363 int fd; 1364 1365 if (for_write) 1366 flags = O_CREAT | O_TRUNC | O_WRONLY | O_SYNC; 1367 else 1368 flags = O_RDONLY; 1369 1370 verify_state_gen_name(out, sizeof(out), name, prefix, num); 1371 1372 fd = open(out, flags, 0644); 1373 if (fd == -1) { 1374 perror("fio: open state file"); 1375 return -1; 1376 } 1377 1378 return fd; 1379 } 1380 1381 static int write_thread_list_state(struct thread_io_list *s, 1382 const char *prefix) 1383 { 1384 struct verify_state_hdr hdr; 1385 uint64_t crc; 1386 ssize_t ret; 1387 int fd; 1388 1389 fd = open_state_file((const char *) s->name, prefix, s->index, 1); 1390 if (fd == -1) 1391 return 1; 1392 1393 crc = fio_crc32c((void *)s, thread_io_list_sz(s)); 1394 1395 hdr.version = cpu_to_le64((uint64_t) VSTATE_HDR_VERSION); 1396 hdr.size = cpu_to_le64((uint64_t) thread_io_list_sz(s)); 1397 hdr.crc = cpu_to_le64(crc); 1398 ret = write(fd, &hdr, sizeof(hdr)); 1399 if (ret != sizeof(hdr)) 1400 goto write_fail; 1401 1402 ret = write(fd, s, thread_io_list_sz(s)); 1403 if (ret != thread_io_list_sz(s)) { 1404 write_fail: 1405 if (ret < 0) 1406 perror("fio: write state file"); 1407 log_err("fio: failed to write state file\n"); 1408 ret = 1; 1409 } else 1410 ret = 0; 1411 1412 close(fd); 1413 return ret; 1414 } 1415 1416 void __verify_save_state(struct all_io_list *state, const char *prefix) 1417 { 1418 struct thread_io_list *s = &state->state[0]; 1419 unsigned int i; 1420 1421 for (i = 0; i < le64_to_cpu(state->threads); i++) { 1422 write_thread_list_state(s, prefix); 1423 s = io_list_next(s); 1424 } 1425 } 1426 1427 void verify_save_state(void) 1428 { 1429 struct all_io_list *state; 1430 size_t sz; 1431 1432 state = get_all_io_list(IO_LIST_ALL, &sz); 1433 if (state) { 1434 __verify_save_state(state, "local"); 1435 free(state); 1436 } 1437 } 1438 1439 void verify_free_state(struct thread_data *td) 1440 { 1441 if (td->vstate) 1442 free(td->vstate); 1443 } 1444 1445 void verify_convert_assign_state(struct thread_data *td, 1446 struct thread_io_list *s) 1447 { 1448 int i; 1449 1450 s->no_comps = le64_to_cpu(s->no_comps); 1451 s->depth = le64_to_cpu(s->depth); 1452 s->numberio = le64_to_cpu(s->numberio); 1453 for (i = 0; i < 4; i++) 1454 s->rand.s[i] = le32_to_cpu(s->rand.s[i]); 1455 for (i = 0; i < s->no_comps; i++) 1456 s->offsets[i] = le64_to_cpu(s->offsets[i]); 1457 1458 td->vstate = s; 1459 } 1460 1461 int verify_state_hdr(struct verify_state_hdr *hdr, struct thread_io_list *s) 1462 { 1463 uint64_t crc; 1464 1465 hdr->version = le64_to_cpu(hdr->version); 1466 hdr->size = le64_to_cpu(hdr->size); 1467 hdr->crc = le64_to_cpu(hdr->crc); 1468 1469 if (hdr->version != VSTATE_HDR_VERSION) 1470 return 1; 1471 1472 crc = fio_crc32c((void *)s, hdr->size); 1473 if (crc != hdr->crc) 1474 return 1; 1475 1476 return 0; 1477 } 1478 1479 int verify_load_state(struct thread_data *td, const char *prefix) 1480 { 1481 struct thread_io_list *s = NULL; 1482 struct verify_state_hdr hdr; 1483 uint64_t crc; 1484 ssize_t ret; 1485 int fd; 1486 1487 if (!td->o.verify_state) 1488 return 0; 1489 1490 fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0); 1491 if (fd == -1) 1492 return 1; 1493 1494 ret = read(fd, &hdr, sizeof(hdr)); 1495 if (ret != sizeof(hdr)) { 1496 if (ret < 0) 1497 td_verror(td, errno, "read verify state hdr"); 1498 log_err("fio: failed reading verify state header\n"); 1499 goto err; 1500 } 1501 1502 hdr.version = le64_to_cpu(hdr.version); 1503 hdr.size = le64_to_cpu(hdr.size); 1504 hdr.crc = le64_to_cpu(hdr.crc); 1505 1506 if (hdr.version != VSTATE_HDR_VERSION) { 1507 log_err("fio: bad version in verify state header\n"); 1508 goto err; 1509 } 1510 1511 s = malloc(hdr.size); 1512 ret = read(fd, s, hdr.size); 1513 if (ret != hdr.size) { 1514 if (ret < 0) 1515 td_verror(td, errno, "read verify state"); 1516 log_err("fio: failed reading verity state\n"); 1517 goto err; 1518 } 1519 1520 crc = fio_crc32c((void *)s, hdr.size); 1521 if (crc != hdr.crc) { 1522 log_err("fio: verify state is corrupt\n"); 1523 goto err; 1524 } 1525 1526 close(fd); 1527 1528 verify_convert_assign_state(td, s); 1529 return 0; 1530 err: 1531 if (s) 1532 free(s); 1533 close(fd); 1534 return 1; 1535 } 1536 1537 /* 1538 * Use the loaded verify state to know when to stop doing verification 1539 */ 1540 int verify_state_should_stop(struct thread_data *td, struct io_u *io_u) 1541 { 1542 struct thread_io_list *s = td->vstate; 1543 int i; 1544 1545 if (!s) 1546 return 0; 1547 1548 /* 1549 * If we're not into the window of issues - depth yet, continue. If 1550 * issue is shorter than depth, do check. 1551 */ 1552 if ((td->io_blocks[DDIR_READ] < s->depth || 1553 s->numberio - td->io_blocks[DDIR_READ] > s->depth) && 1554 s->numberio > s->depth) 1555 return 0; 1556 1557 /* 1558 * We're in the window of having to check if this io was 1559 * completed or not. If the IO was seen as completed, then 1560 * lets verify it. 1561 */ 1562 for (i = 0; i < s->no_comps; i++) 1563 if (io_u->offset == s->offsets[i]) 1564 return 0; 1565 1566 /* 1567 * Not found, we have to stop 1568 */ 1569 return 1; 1570 } 1571
