1 #ifndef FIO_H 2 #define FIO_H 3 4 #include <sched.h> 5 #include <limits.h> 6 #include <pthread.h> 7 #include <sys/time.h> 8 #include <sys/resource.h> 9 #include <errno.h> 10 #include <stdlib.h> 11 #include <stdio.h> 12 #include <unistd.h> 13 #include <string.h> 14 #include <inttypes.h> 15 #include <assert.h> 16 17 #include "compiler/compiler.h" 18 #include "thread_options.h" 19 #include "flist.h" 20 #include "fifo.h" 21 #include "arch/arch.h" 22 #include "os/os.h" 23 #include "mutex.h" 24 #include "log.h" 25 #include "debug.h" 26 #include "file.h" 27 #include "io_ddir.h" 28 #include "ioengine.h" 29 #include "iolog.h" 30 #include "helpers.h" 31 #include "options.h" 32 #include "profile.h" 33 #include "fio_time.h" 34 #include "gettime.h" 35 #include "lib/getopt.h" 36 #include "lib/rand.h" 37 #include "lib/rbtree.h" 38 #include "client.h" 39 #include "server.h" 40 #include "stat.h" 41 #include "flow.h" 42 #include "io_u_queue.h" 43 44 #ifdef CONFIG_SOLARISAIO 45 #include <sys/asynch.h> 46 #endif 47 48 #ifdef CONFIG_LIBNUMA 49 #include <linux/mempolicy.h> 50 #include <numa.h> 51 52 /* 53 * "local" is pseudo-policy 54 */ 55 #define MPOL_LOCAL MPOL_MAX 56 #endif 57 58 /* 59 * offset generator types 60 */ 61 enum { 62 RW_SEQ_SEQ = 0, 63 RW_SEQ_IDENT, 64 }; 65 66 enum { 67 TD_F_VER_BACKLOG = 1, 68 TD_F_TRIM_BACKLOG = 2, 69 TD_F_READ_IOLOG = 4, 70 TD_F_REFILL_BUFFERS = 8, 71 TD_F_SCRAMBLE_BUFFERS = 16, 72 TD_F_VER_NONE = 32, 73 TD_F_PROFILE_OPS = 64, 74 TD_F_COMPRESS = 128, 75 TD_F_NOIO = 256, 76 TD_F_COMPRESS_LOG = 512, 77 TD_F_VSTATE_SAVED = 1024, 78 }; 79 80 enum { 81 FIO_RAND_BS_OFF = 0, 82 FIO_RAND_VER_OFF, 83 FIO_RAND_MIX_OFF, 84 FIO_RAND_FILE_OFF, 85 FIO_RAND_BLOCK_OFF, 86 FIO_RAND_FILE_SIZE_OFF, 87 FIO_RAND_TRIM_OFF, 88 FIO_RAND_BUF_OFF, 89 FIO_RAND_SEQ_RAND_READ_OFF, 90 FIO_RAND_SEQ_RAND_WRITE_OFF, 91 FIO_RAND_SEQ_RAND_TRIM_OFF, 92 FIO_RAND_START_DELAY, 93 FIO_DEDUPE_OFF, 94 FIO_RAND_NR_OFFS, 95 }; 96 97 /* 98 * This describes a single thread/process executing a fio job. 99 */ 100 struct thread_data { 101 struct thread_options o; 102 unsigned long flags; 103 void *eo; 104 char verror[FIO_VERROR_SIZE]; 105 pthread_t thread; 106 unsigned int thread_number; 107 unsigned int subjob_number; 108 unsigned int groupid; 109 struct thread_stat ts; 110 111 int client_type; 112 113 struct io_log *slat_log; 114 struct io_log *clat_log; 115 struct io_log *lat_log; 116 struct io_log *bw_log; 117 struct io_log *iops_log; 118 119 struct tp_data *tp_data; 120 121 uint64_t stat_io_bytes[DDIR_RWDIR_CNT]; 122 struct timeval bw_sample_time; 123 124 uint64_t stat_io_blocks[DDIR_RWDIR_CNT]; 125 struct timeval iops_sample_time; 126 127 /* 128 * Tracks the last iodepth number of completed writes, if data 129 * verification is enabled 130 */ 131 uint64_t *last_write_comp; 132 unsigned int last_write_idx; 133 134 volatile int update_rusage; 135 struct fio_mutex *rusage_sem; 136 struct rusage ru_start; 137 struct rusage ru_end; 138 139 struct fio_file **files; 140 unsigned char *file_locks; 141 unsigned int files_size; 142 unsigned int files_index; 143 unsigned int nr_open_files; 144 unsigned int nr_done_files; 145 unsigned int nr_normal_files; 146 union { 147 unsigned int next_file; 148 struct frand_state next_file_state; 149 }; 150 int error; 151 int sig; 152 int done; 153 int stop_io; 154 pid_t pid; 155 char *orig_buffer; 156 size_t orig_buffer_size; 157 volatile int terminate; 158 volatile int runstate; 159 unsigned int last_was_sync; 160 enum fio_ddir last_ddir; 161 162 int mmapfd; 163 164 void *iolog_buf; 165 FILE *iolog_f; 166 167 char *sysfs_root; 168 169 unsigned long rand_seeds[FIO_RAND_NR_OFFS]; 170 171 struct frand_state bsrange_state; 172 struct frand_state verify_state; 173 struct frand_state trim_state; 174 struct frand_state delay_state; 175 176 struct frand_state buf_state; 177 struct frand_state buf_state_prev; 178 struct frand_state dedupe_state; 179 180 unsigned int verify_batch; 181 unsigned int trim_batch; 182 183 struct thread_io_list *vstate; 184 185 int shm_id; 186 187 /* 188 * IO engine hooks, contains everything needed to submit an io_u 189 * to any of the available IO engines. 190 */ 191 struct ioengine_ops *io_ops; 192 193 /* 194 * Queue depth of io_u's that fio MIGHT do 195 */ 196 unsigned int cur_depth; 197 198 /* 199 * io_u's about to be committed 200 */ 201 unsigned int io_u_queued; 202 203 /* 204 * io_u's submitted but not completed yet 205 */ 206 unsigned int io_u_in_flight; 207 208 /* 209 * List of free and busy io_u's 210 */ 211 struct io_u_ring io_u_requeues; 212 struct io_u_queue io_u_freelist; 213 struct io_u_queue io_u_all; 214 pthread_mutex_t io_u_lock; 215 pthread_cond_t free_cond; 216 217 /* 218 * async verify offload 219 */ 220 struct flist_head verify_list; 221 pthread_t *verify_threads; 222 unsigned int nr_verify_threads; 223 pthread_cond_t verify_cond; 224 int verify_thread_exit; 225 226 /* 227 * Rate state 228 */ 229 uint64_t rate_bps[DDIR_RWDIR_CNT]; 230 long rate_pending_usleep[DDIR_RWDIR_CNT]; 231 unsigned long rate_bytes[DDIR_RWDIR_CNT]; 232 unsigned long rate_blocks[DDIR_RWDIR_CNT]; 233 struct timeval lastrate[DDIR_RWDIR_CNT]; 234 235 uint64_t total_io_size; 236 uint64_t fill_device_size; 237 238 /* 239 * Issue side 240 */ 241 uint64_t io_issues[DDIR_RWDIR_CNT]; 242 uint64_t io_issue_bytes[DDIR_RWDIR_CNT]; 243 uint64_t loops; 244 245 /* 246 * Completions 247 */ 248 uint64_t io_blocks[DDIR_RWDIR_CNT]; 249 uint64_t this_io_blocks[DDIR_RWDIR_CNT]; 250 uint64_t io_bytes[DDIR_RWDIR_CNT]; 251 uint64_t io_skip_bytes; 252 uint64_t this_io_bytes[DDIR_RWDIR_CNT]; 253 uint64_t zone_bytes; 254 struct fio_mutex *mutex; 255 256 /* 257 * State for random io, a bitmap of blocks done vs not done 258 */ 259 struct frand_state random_state; 260 261 struct timeval start; /* start of this loop */ 262 struct timeval epoch; /* time job was started */ 263 struct timeval last_issue; 264 long time_offset; 265 struct timeval tv_cache; 266 struct timeval terminate_time; 267 unsigned int tv_cache_nr; 268 unsigned int tv_cache_mask; 269 unsigned int ramp_time_over; 270 271 /* 272 * Time since last latency_window was started 273 */ 274 struct timeval latency_ts; 275 unsigned int latency_qd; 276 unsigned int latency_qd_high; 277 unsigned int latency_qd_low; 278 unsigned int latency_failed; 279 uint64_t latency_ios; 280 int latency_end_run; 281 282 /* 283 * read/write mixed workload state 284 */ 285 struct frand_state rwmix_state; 286 unsigned long rwmix_issues; 287 enum fio_ddir rwmix_ddir; 288 unsigned int ddir_seq_nr; 289 290 /* 291 * rand/seq mixed workload state 292 */ 293 struct frand_state seq_rand_state[DDIR_RWDIR_CNT]; 294 295 /* 296 * IO history logs for verification. We use a tree for sorting, 297 * if we are overwriting. Otherwise just use a fifo. 298 */ 299 struct rb_root io_hist_tree; 300 struct flist_head io_hist_list; 301 unsigned long io_hist_len; 302 303 /* 304 * For IO replaying 305 */ 306 struct flist_head io_log_list; 307 308 /* 309 * For tracking/handling discards 310 */ 311 struct flist_head trim_list; 312 unsigned long trim_entries; 313 314 struct flist_head next_rand_list; 315 316 /* 317 * for fileservice, how often to switch to a new file 318 */ 319 unsigned int file_service_nr; 320 unsigned int file_service_left; 321 struct fio_file *file_service_file; 322 323 unsigned int sync_file_range_nr; 324 325 /* 326 * For generating file sizes 327 */ 328 struct frand_state file_size_state; 329 330 /* 331 * Error counts 332 */ 333 unsigned int total_err_count; 334 int first_error; 335 336 struct fio_flow *flow; 337 338 /* 339 * Can be overloaded by profiles 340 */ 341 struct prof_io_ops prof_io_ops; 342 void *prof_data; 343 344 void *pinned_mem; 345 }; 346 347 /* 348 * when should interactive ETA output be generated 349 */ 350 enum { 351 FIO_ETA_AUTO, 352 FIO_ETA_ALWAYS, 353 FIO_ETA_NEVER, 354 }; 355 356 #define __td_verror(td, err, msg, func) \ 357 do { \ 358 unsigned int ____e = (err); \ 359 if ((td)->error) \ 360 break; \ 361 (td)->error = ____e; \ 362 if (!(td)->first_error) \ 363 snprintf(td->verror, sizeof(td->verror), "file:%s:%d, func=%s, error=%s", __FILE__, __LINE__, (func), (msg)); \ 364 } while (0) 365 366 367 #define td_clear_error(td) \ 368 (td)->error = 0; 369 #define td_verror(td, err, func) \ 370 __td_verror((td), (err), strerror((err)), (func)) 371 #define td_vmsg(td, err, msg, func) \ 372 __td_verror((td), (err), (msg), (func)) 373 374 #define __fio_stringify_1(x) #x 375 #define __fio_stringify(x) __fio_stringify_1(x) 376 377 extern int exitall_on_terminate; 378 extern unsigned int thread_number; 379 extern unsigned int stat_number; 380 extern int shm_id; 381 extern int groupid; 382 extern int output_format; 383 extern int append_terse_output; 384 extern int temp_stall_ts; 385 extern uintptr_t page_mask, page_size; 386 extern int read_only; 387 extern int eta_print; 388 extern int eta_new_line; 389 extern unsigned long done_secs; 390 extern char *job_section; 391 extern int fio_gtod_offload; 392 extern int fio_gtod_cpu; 393 extern enum fio_cs fio_clock_source; 394 extern int fio_clock_source_set; 395 extern int warnings_fatal; 396 extern int terse_version; 397 extern int is_backend; 398 extern int nr_clients; 399 extern int log_syslog; 400 extern int status_interval; 401 extern const char fio_version_string[]; 402 extern int helper_do_stat; 403 extern pthread_cond_t helper_cond; 404 extern char *trigger_file; 405 extern char *trigger_cmd; 406 extern char *trigger_remote_cmd; 407 extern long long trigger_timeout; 408 409 extern struct thread_data *threads; 410 411 static inline void fio_ro_check(const struct thread_data *td, struct io_u *io_u) 412 { 413 assert(!(io_u->ddir == DDIR_WRITE && !td_write(td))); 414 } 415 416 #define REAL_MAX_JOBS 2048 417 418 static inline int should_fsync(struct thread_data *td) 419 { 420 if (td->last_was_sync) 421 return 0; 422 if (td_write(td) || td_rw(td) || td->o.override_sync) 423 return 1; 424 425 return 0; 426 } 427 428 /* 429 * Init/option functions 430 */ 431 extern int __must_check fio_init_options(void); 432 extern int __must_check parse_options(int, char **); 433 extern int parse_jobs_ini(char *, int, int, int); 434 extern int parse_cmd_line(int, char **, int); 435 extern int fio_backend(void); 436 extern void reset_fio_state(void); 437 extern void clear_io_state(struct thread_data *); 438 extern int fio_options_parse(struct thread_data *, char **, int, int); 439 extern void fio_keywords_init(void); 440 extern int fio_cmd_option_parse(struct thread_data *, const char *, char *); 441 extern int fio_cmd_ioengine_option_parse(struct thread_data *, const char *, char *); 442 extern void fio_fill_default_options(struct thread_data *); 443 extern int fio_show_option_help(const char *); 444 extern void fio_options_set_ioengine_opts(struct option *long_options, struct thread_data *td); 445 extern void fio_options_dup_and_init(struct option *); 446 extern void fio_options_mem_dupe(struct thread_data *); 447 extern void options_mem_dupe(void *data, struct fio_option *options); 448 extern void td_fill_rand_seeds(struct thread_data *); 449 extern void add_job_opts(const char **, int); 450 extern char *num2str(uint64_t, int, int, int, int); 451 extern int ioengine_load(struct thread_data *); 452 extern int parse_dryrun(void); 453 extern int fio_running_or_pending_io_threads(void); 454 extern int fio_set_fd_nonblocking(int, const char *); 455 456 extern uintptr_t page_mask; 457 extern uintptr_t page_size; 458 extern int initialize_fio(char *envp[]); 459 460 #define FIO_GETOPT_JOB 0x89000000 461 #define FIO_GETOPT_IOENGINE 0x98000000 462 #define FIO_NR_OPTIONS (FIO_MAX_OPTS + 128) 463 464 /* 465 * ETA/status stuff 466 */ 467 extern void print_thread_status(void); 468 extern void print_status_init(int); 469 extern char *fio_uint_to_kmg(unsigned int val); 470 471 /* 472 * Thread life cycle. Once a thread has a runstate beyond TD_INITIALIZED, it 473 * will never back again. It may cycle between running/verififying/fsyncing. 474 * Once the thread reaches TD_EXITED, it is just waiting for the core to 475 * reap it. 476 */ 477 enum { 478 TD_NOT_CREATED = 0, 479 TD_CREATED, 480 TD_INITIALIZED, 481 TD_RAMP, 482 TD_SETTING_UP, 483 TD_RUNNING, 484 TD_PRE_READING, 485 TD_VERIFYING, 486 TD_FSYNCING, 487 TD_FINISHING, 488 TD_EXITED, 489 TD_REAPED, 490 }; 491 492 extern void td_set_runstate(struct thread_data *, int); 493 extern int td_bump_runstate(struct thread_data *, int); 494 extern void td_restore_runstate(struct thread_data *, int); 495 496 /* 497 * Allow 60 seconds for a job to quit on its own, otherwise reap with 498 * a vengeance. 499 */ 500 #define FIO_REAP_TIMEOUT 60 501 502 #define TERMINATE_ALL (-1) 503 extern void fio_terminate_threads(int); 504 extern void fio_mark_td_terminate(struct thread_data *); 505 506 /* 507 * Memory helpers 508 */ 509 extern int __must_check fio_pin_memory(struct thread_data *); 510 extern void fio_unpin_memory(struct thread_data *); 511 extern int __must_check allocate_io_mem(struct thread_data *); 512 extern void free_io_mem(struct thread_data *); 513 extern void free_threads_shm(void); 514 515 /* 516 * Reset stats after ramp time completes 517 */ 518 extern void reset_all_stats(struct thread_data *); 519 520 /* 521 * blktrace support 522 */ 523 #ifdef FIO_HAVE_BLKTRACE 524 extern int is_blktrace(const char *, int *); 525 extern int load_blktrace(struct thread_data *, const char *, int); 526 #endif 527 528 /* 529 * Latency target helpers 530 */ 531 extern void lat_target_check(struct thread_data *); 532 extern void lat_target_init(struct thread_data *); 533 extern void lat_target_reset(struct thread_data *); 534 535 #define for_each_td(td, i) \ 536 for ((i) = 0, (td) = &threads[0]; (i) < (int) thread_number; (i)++, (td)++) 537 #define for_each_file(td, f, i) \ 538 if ((td)->files_index) \ 539 for ((i) = 0, (f) = (td)->files[0]; \ 540 (i) < (td)->o.nr_files && ((f) = (td)->files[i]) != NULL; \ 541 (i)++) 542 543 #define fio_assert(td, cond) do { \ 544 if (!(cond)) { \ 545 int *__foo = NULL; \ 546 fprintf(stderr, "file:%s:%d, assert %s failed\n", __FILE__, __LINE__, #cond); \ 547 td_set_runstate((td), TD_EXITED); \ 548 (td)->error = EFAULT; \ 549 *__foo = 0; \ 550 } \ 551 } while (0) 552 553 static inline int fio_fill_issue_time(struct thread_data *td) 554 { 555 if (td->o.read_iolog_file || 556 !td->o.disable_clat || !td->o.disable_slat || !td->o.disable_bw) 557 return 1; 558 559 return 0; 560 } 561 562 static inline int __should_check_rate(struct thread_data *td, 563 enum fio_ddir ddir) 564 { 565 struct thread_options *o = &td->o; 566 567 /* 568 * If some rate setting was given, we need to check it 569 */ 570 if (o->rate[ddir] || o->ratemin[ddir] || o->rate_iops[ddir] || 571 o->rate_iops_min[ddir]) 572 return 1; 573 574 return 0; 575 } 576 577 static inline int should_check_rate(struct thread_data *td, 578 uint64_t *bytes_done) 579 { 580 int ret = 0; 581 582 if (bytes_done[DDIR_READ]) 583 ret |= __should_check_rate(td, DDIR_READ); 584 if (bytes_done[DDIR_WRITE]) 585 ret |= __should_check_rate(td, DDIR_WRITE); 586 if (bytes_done[DDIR_TRIM]) 587 ret |= __should_check_rate(td, DDIR_TRIM); 588 589 return ret; 590 } 591 592 static inline unsigned int td_max_bs(struct thread_data *td) 593 { 594 unsigned int max_bs; 595 596 max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]); 597 return max(td->o.max_bs[DDIR_TRIM], max_bs); 598 } 599 600 static inline unsigned int td_min_bs(struct thread_data *td) 601 { 602 unsigned int min_bs; 603 604 min_bs = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]); 605 return min(td->o.min_bs[DDIR_TRIM], min_bs); 606 } 607 608 static inline int is_power_of_2(uint64_t val) 609 { 610 return (val != 0 && ((val & (val - 1)) == 0)); 611 } 612 613 /* 614 * We currently only need to do locking if we have verifier threads 615 * accessing our internal structures too 616 */ 617 static inline void td_io_u_lock(struct thread_data *td) 618 { 619 if (td->o.verify_async) 620 pthread_mutex_lock(&td->io_u_lock); 621 } 622 623 static inline void td_io_u_unlock(struct thread_data *td) 624 { 625 if (td->o.verify_async) 626 pthread_mutex_unlock(&td->io_u_lock); 627 } 628 629 static inline void td_io_u_free_notify(struct thread_data *td) 630 { 631 if (td->o.verify_async) 632 pthread_cond_signal(&td->free_cond); 633 } 634 635 extern const char *fio_get_arch_string(int); 636 extern const char *fio_get_os_string(int); 637 638 #ifdef FIO_INTERNAL 639 #define ARRAY_SIZE(x) (sizeof((x)) / (sizeof((x)[0]))) 640 #endif 641 642 enum { 643 FIO_OUTPUT_TERSE = 0, 644 FIO_OUTPUT_JSON, 645 FIO_OUTPUT_NORMAL, 646 }; 647 648 enum { 649 FIO_RAND_DIST_RANDOM = 0, 650 FIO_RAND_DIST_ZIPF, 651 FIO_RAND_DIST_PARETO, 652 }; 653 654 #define FIO_DEF_ZIPF 1.1 655 #define FIO_DEF_PARETO 0.2 656 657 enum { 658 FIO_RAND_GEN_TAUSWORTHE = 0, 659 FIO_RAND_GEN_LFSR, 660 }; 661 662 enum { 663 FIO_CPUS_SHARED = 0, 664 FIO_CPUS_SPLIT, 665 }; 666 667 extern void exec_trigger(const char *); 668 extern void check_trigger_file(void); 669 670 #endif 671
