1 /* 2 * mmap engine 3 * 4 * IO engine that reads/writes from files by doing memcpy to/from 5 * a memory mapped region of the file. 6 * 7 */ 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <unistd.h> 11 #include <errno.h> 12 #include <sys/mman.h> 13 14 #include "../fio.h" 15 #include "../verify.h" 16 17 /* 18 * Limits us to 1GiB of mapped files in total 19 */ 20 #define MMAP_TOTAL_SZ (1 * 1024 * 1024 * 1024UL) 21 22 static unsigned long mmap_map_size; 23 24 struct fio_mmap_data { 25 void *mmap_ptr; 26 size_t mmap_sz; 27 off_t mmap_off; 28 }; 29 30 static int fio_mmap_file(struct thread_data *td, struct fio_file *f, 31 size_t length, off_t off) 32 { 33 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 34 int flags = 0; 35 36 if (td_rw(td)) 37 flags = PROT_READ | PROT_WRITE; 38 else if (td_write(td)) { 39 flags = PROT_WRITE; 40 41 if (td->o.verify != VERIFY_NONE) 42 flags |= PROT_READ; 43 } else 44 flags = PROT_READ; 45 46 fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off); 47 if (fmd->mmap_ptr == MAP_FAILED) { 48 fmd->mmap_ptr = NULL; 49 td_verror(td, errno, "mmap"); 50 goto err; 51 } 52 53 if (!td_random(td)) { 54 if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) { 55 td_verror(td, errno, "madvise"); 56 goto err; 57 } 58 } else { 59 if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) { 60 td_verror(td, errno, "madvise"); 61 goto err; 62 } 63 } 64 if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) { 65 td_verror(td, errno, "madvise"); 66 goto err; 67 } 68 69 #ifdef FIO_MADV_FREE 70 if (f->filetype == FIO_TYPE_BLOCK) 71 (void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE); 72 #endif 73 74 err: 75 if (td->error && fmd->mmap_ptr) 76 munmap(fmd->mmap_ptr, length); 77 78 return td->error; 79 } 80 81 /* 82 * Just mmap an appropriate portion, we cannot mmap the full extent 83 */ 84 static int fio_mmapio_prep_limited(struct thread_data *td, struct io_u *io_u) 85 { 86 struct fio_file *f = io_u->file; 87 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 88 89 if (io_u->buflen > mmap_map_size) { 90 log_err("fio: bs too big for mmap engine\n"); 91 return EIO; 92 } 93 94 fmd->mmap_sz = mmap_map_size; 95 if (fmd->mmap_sz > f->io_size) 96 fmd->mmap_sz = f->io_size; 97 98 fmd->mmap_off = io_u->offset; 99 100 return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off); 101 } 102 103 /* 104 * Attempt to mmap the entire file 105 */ 106 static int fio_mmapio_prep_full(struct thread_data *td, struct io_u *io_u) 107 { 108 struct fio_file *f = io_u->file; 109 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 110 int ret; 111 112 if (fio_file_partial_mmap(f)) 113 return EINVAL; 114 if (io_u->offset != (size_t) io_u->offset || 115 f->io_size != (size_t) f->io_size) { 116 fio_file_set_partial_mmap(f); 117 return EINVAL; 118 } 119 120 fmd->mmap_sz = f->io_size; 121 fmd->mmap_off = 0; 122 123 ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off); 124 if (ret) 125 fio_file_set_partial_mmap(f); 126 127 return ret; 128 } 129 130 static int fio_mmapio_prep(struct thread_data *td, struct io_u *io_u) 131 { 132 struct fio_file *f = io_u->file; 133 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 134 int ret; 135 136 /* 137 * It fits within existing mapping, use it 138 */ 139 if (io_u->offset >= fmd->mmap_off && 140 io_u->offset + io_u->buflen < fmd->mmap_off + fmd->mmap_sz) 141 goto done; 142 143 /* 144 * unmap any existing mapping 145 */ 146 if (fmd->mmap_ptr) { 147 if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0) 148 return errno; 149 fmd->mmap_ptr = NULL; 150 } 151 152 if (fio_mmapio_prep_full(td, io_u)) { 153 td_clear_error(td); 154 ret = fio_mmapio_prep_limited(td, io_u); 155 if (ret) 156 return ret; 157 } 158 159 done: 160 io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off - 161 f->file_offset; 162 return 0; 163 } 164 165 static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) 166 { 167 struct fio_file *f = io_u->file; 168 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 169 170 fio_ro_check(td, io_u); 171 172 if (io_u->ddir == DDIR_READ) 173 memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen); 174 else if (io_u->ddir == DDIR_WRITE) 175 memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen); 176 else if (ddir_sync(io_u->ddir)) { 177 if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) { 178 io_u->error = errno; 179 td_verror(td, io_u->error, "msync"); 180 } 181 } else if (io_u->ddir == DDIR_TRIM) { 182 int ret = do_io_u_trim(td, io_u); 183 184 if (!ret) 185 td_verror(td, io_u->error, "trim"); 186 } 187 188 189 /* 190 * not really direct, but should drop the pages from the cache 191 */ 192 if (td->o.odirect && ddir_rw(io_u->ddir)) { 193 if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) { 194 io_u->error = errno; 195 td_verror(td, io_u->error, "msync"); 196 } 197 if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) { 198 io_u->error = errno; 199 td_verror(td, io_u->error, "madvise"); 200 } 201 } 202 203 return FIO_Q_COMPLETED; 204 } 205 206 static int fio_mmapio_init(struct thread_data *td) 207 { 208 struct thread_options *o = &td->o; 209 210 if ((o->rw_min_bs & page_mask) && 211 (o->odirect || o->fsync_blocks || o->fdatasync_blocks)) { 212 log_err("fio: mmap options dictate a minimum block size of " 213 "%llu bytes\n", (unsigned long long) page_size); 214 return 1; 215 } 216 217 mmap_map_size = MMAP_TOTAL_SZ / o->nr_files; 218 return 0; 219 } 220 221 static int fio_mmapio_open_file(struct thread_data *td, struct fio_file *f) 222 { 223 struct fio_mmap_data *fmd; 224 int ret; 225 226 ret = generic_open_file(td, f); 227 if (ret) 228 return ret; 229 230 fmd = calloc(1, sizeof(*fmd)); 231 if (!fmd) { 232 int fio_unused __ret; 233 __ret = generic_close_file(td, f); 234 return 1; 235 } 236 237 FILE_SET_ENG_DATA(f, fmd); 238 return 0; 239 } 240 241 static int fio_mmapio_close_file(struct thread_data *td, struct fio_file *f) 242 { 243 struct fio_mmap_data *fmd = FILE_ENG_DATA(f); 244 245 FILE_SET_ENG_DATA(f, NULL); 246 free(fmd); 247 fio_file_clear_partial_mmap(f); 248 249 return generic_close_file(td, f); 250 } 251 252 static struct ioengine_ops ioengine = { 253 .name = "mmap", 254 .version = FIO_IOOPS_VERSION, 255 .init = fio_mmapio_init, 256 .prep = fio_mmapio_prep, 257 .queue = fio_mmapio_queue, 258 .open_file = fio_mmapio_open_file, 259 .close_file = fio_mmapio_close_file, 260 .get_file_size = generic_get_file_size, 261 .flags = FIO_SYNCIO | FIO_NOEXTEND, 262 }; 263 264 static void fio_init fio_mmapio_register(void) 265 { 266 register_ioengine(&ioengine); 267 } 268 269 static void fio_exit fio_mmapio_unregister(void) 270 { 271 unregister_ioengine(&ioengine); 272 } 273
