Home | History | Annotate | Download | only in fio
      1 /*
      2  * Memory helpers
      3  */
      4 #include <sys/types.h>
      5 #include <sys/stat.h>
      6 #include <fcntl.h>
      7 #include <unistd.h>
      8 #include <sys/mman.h>
      9 
     10 #include "fio.h"
     11 #ifndef FIO_NO_HAVE_SHM_H
     12 #include <sys/shm.h>
     13 #endif
     14 
     15 void fio_unpin_memory(struct thread_data *td)
     16 {
     17 	if (td->pinned_mem) {
     18 		dprint(FD_MEM, "unpinning %llu bytes\n", td->o.lockmem);
     19 		if (munlock(td->pinned_mem, td->o.lockmem) < 0)
     20 			perror("munlock");
     21 		munmap(td->pinned_mem, td->o.lockmem);
     22 		td->pinned_mem = NULL;
     23 	}
     24 }
     25 
     26 int fio_pin_memory(struct thread_data *td)
     27 {
     28 	unsigned long long phys_mem;
     29 
     30 	if (!td->o.lockmem)
     31 		return 0;
     32 
     33 	dprint(FD_MEM, "pinning %llu bytes\n", td->o.lockmem);
     34 
     35 	/*
     36 	 * Don't allow mlock of more than real_mem-128MB
     37 	 */
     38 	phys_mem = os_phys_mem();
     39 	if (phys_mem) {
     40 		if ((td->o.lockmem + 128 * 1024 * 1024) > phys_mem) {
     41 			td->o.lockmem = phys_mem - 128 * 1024 * 1024;
     42 			log_info("fio: limiting mlocked memory to %lluMB\n",
     43 							td->o.lockmem >> 20);
     44 		}
     45 	}
     46 
     47 	td->pinned_mem = mmap(NULL, td->o.lockmem, PROT_READ | PROT_WRITE,
     48 				MAP_PRIVATE | OS_MAP_ANON, -1, 0);
     49 	if (td->pinned_mem == MAP_FAILED) {
     50 		perror("malloc locked mem");
     51 		td->pinned_mem = NULL;
     52 		return 1;
     53 	}
     54 	if (mlock(td->pinned_mem, td->o.lockmem) < 0) {
     55 		perror("mlock");
     56 		munmap(td->pinned_mem, td->o.lockmem);
     57 		td->pinned_mem = NULL;
     58 		return 1;
     59 	}
     60 
     61 	return 0;
     62 }
     63 
     64 static int alloc_mem_shm(struct thread_data *td, unsigned int total_mem)
     65 {
     66 	int flags = IPC_CREAT | S_IRUSR | S_IWUSR;
     67 
     68 	if (td->o.mem_type == MEM_SHMHUGE) {
     69 		unsigned long mask = td->o.hugepage_size - 1;
     70 
     71 		flags |= SHM_HUGETLB;
     72 		total_mem = (total_mem + mask) & ~mask;
     73 	}
     74 
     75 	td->shm_id = shmget(IPC_PRIVATE, total_mem, flags);
     76 	dprint(FD_MEM, "shmget %u, %d\n", total_mem, td->shm_id);
     77 	if (td->shm_id < 0) {
     78 		td_verror(td, errno, "shmget");
     79 		if (geteuid() != 0 && (errno == ENOMEM || errno == EPERM))
     80 			log_err("fio: you may need to run this job as root\n");
     81 		if (td->o.mem_type == MEM_SHMHUGE) {
     82 			if (errno == EINVAL) {
     83 				log_err("fio: check that you have free huge"
     84 					" pages and that hugepage-size is"
     85 					" correct.\n");
     86 			} else if (errno == ENOSYS) {
     87 				log_err("fio: your system does not appear to"
     88 					" support huge pages.\n");
     89 			} else if (errno == ENOMEM) {
     90 				log_err("fio: no huge pages available, do you"
     91 					" need to alocate some? See HOWTO.\n");
     92 			}
     93 		}
     94 
     95 		return 1;
     96 	}
     97 
     98 	td->orig_buffer = shmat(td->shm_id, NULL, 0);
     99 	dprint(FD_MEM, "shmat %d, %p\n", td->shm_id, td->orig_buffer);
    100 	if (td->orig_buffer == (void *) -1) {
    101 		td_verror(td, errno, "shmat");
    102 		td->orig_buffer = NULL;
    103 		return 1;
    104 	}
    105 
    106 	return 0;
    107 }
    108 
    109 static void free_mem_shm(struct thread_data *td)
    110 {
    111 	struct shmid_ds sbuf;
    112 
    113 	dprint(FD_MEM, "shmdt/ctl %d %p\n", td->shm_id, td->orig_buffer);
    114 	shmdt(td->orig_buffer);
    115 	shmctl(td->shm_id, IPC_RMID, &sbuf);
    116 }
    117 
    118 static int alloc_mem_mmap(struct thread_data *td, size_t total_mem)
    119 {
    120 	int flags = 0;
    121 
    122 	td->mmapfd = 1;
    123 
    124 	if (td->o.mem_type == MEM_MMAPHUGE) {
    125 		unsigned long mask = td->o.hugepage_size - 1;
    126 
    127 		/* TODO: make sure the file is a real hugetlbfs file */
    128 		if (!td->o.mmapfile)
    129 			flags |= MAP_HUGETLB;
    130 		total_mem = (total_mem + mask) & ~mask;
    131 	}
    132 
    133 	if (td->o.mmapfile) {
    134 		td->mmapfd = open(td->o.mmapfile, O_RDWR|O_CREAT, 0644);
    135 
    136 		if (td->mmapfd < 0) {
    137 			td_verror(td, errno, "open mmap file");
    138 			td->orig_buffer = NULL;
    139 			return 1;
    140 		}
    141 		if (td->o.mem_type != MEM_MMAPHUGE &&
    142 		    ftruncate(td->mmapfd, total_mem) < 0) {
    143 			td_verror(td, errno, "truncate mmap file");
    144 			td->orig_buffer = NULL;
    145 			return 1;
    146 		}
    147 		if (td->o.mem_type == MEM_MMAPHUGE)
    148 			flags |= MAP_SHARED;
    149 		else
    150 			flags |= MAP_PRIVATE;
    151 	} else
    152 		flags |= OS_MAP_ANON | MAP_PRIVATE;
    153 
    154 	td->orig_buffer = mmap(NULL, total_mem, PROT_READ | PROT_WRITE, flags,
    155 				td->mmapfd, 0);
    156 	dprint(FD_MEM, "mmap %llu/%d %p\n", (unsigned long long) total_mem,
    157 						td->mmapfd, td->orig_buffer);
    158 	if (td->orig_buffer == MAP_FAILED) {
    159 		td_verror(td, errno, "mmap");
    160 		td->orig_buffer = NULL;
    161 		if (td->mmapfd != 1) {
    162 			close(td->mmapfd);
    163 			if (td->o.mmapfile)
    164 				unlink(td->o.mmapfile);
    165 		}
    166 
    167 		return 1;
    168 	}
    169 
    170 	return 0;
    171 }
    172 
    173 static void free_mem_mmap(struct thread_data *td, size_t total_mem)
    174 {
    175 	dprint(FD_MEM, "munmap %llu %p\n", (unsigned long long) total_mem,
    176 						td->orig_buffer);
    177 	munmap(td->orig_buffer, td->orig_buffer_size);
    178 	if (td->o.mmapfile) {
    179 		close(td->mmapfd);
    180 		unlink(td->o.mmapfile);
    181 		free(td->o.mmapfile);
    182 	}
    183 }
    184 
    185 static int alloc_mem_malloc(struct thread_data *td, size_t total_mem)
    186 {
    187 	td->orig_buffer = malloc(total_mem);
    188 	dprint(FD_MEM, "malloc %llu %p\n", (unsigned long long) total_mem,
    189 							td->orig_buffer);
    190 
    191 	return td->orig_buffer == NULL;
    192 }
    193 
    194 static void free_mem_malloc(struct thread_data *td)
    195 {
    196 	dprint(FD_MEM, "free malloc mem %p\n", td->orig_buffer);
    197 	free(td->orig_buffer);
    198 }
    199 
    200 /*
    201  * Set up the buffer area we need for io.
    202  */
    203 int allocate_io_mem(struct thread_data *td)
    204 {
    205 	size_t total_mem;
    206 	int ret = 0;
    207 
    208 	if (td->io_ops->flags & FIO_NOIO)
    209 		return 0;
    210 
    211 	total_mem = td->orig_buffer_size;
    212 
    213 	if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
    214 	    (td->io_ops->flags & FIO_MEMALIGN)) {
    215 		total_mem += page_mask;
    216 		if (td->o.mem_align && td->o.mem_align > page_size)
    217 			total_mem += td->o.mem_align - page_size;
    218 	}
    219 
    220 	dprint(FD_MEM, "Alloc %llu for buffers\n", (unsigned long long) total_mem);
    221 
    222 	if (td->o.mem_type == MEM_MALLOC)
    223 		ret = alloc_mem_malloc(td, total_mem);
    224 	else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
    225 		ret = alloc_mem_shm(td, total_mem);
    226 	else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
    227 		ret = alloc_mem_mmap(td, total_mem);
    228 	else {
    229 		log_err("fio: bad mem type: %d\n", td->o.mem_type);
    230 		ret = 1;
    231 	}
    232 
    233 	if (ret)
    234 		td_verror(td, ENOMEM, "iomem allocation");
    235 
    236 	return ret;
    237 }
    238 
    239 void free_io_mem(struct thread_data *td)
    240 {
    241 	unsigned int total_mem;
    242 
    243 	total_mem = td->orig_buffer_size;
    244 	if (td->o.odirect || td->o.oatomic)
    245 		total_mem += page_mask;
    246 
    247 	if (td->o.mem_type == MEM_MALLOC)
    248 		free_mem_malloc(td);
    249 	else if (td->o.mem_type == MEM_SHM || td->o.mem_type == MEM_SHMHUGE)
    250 		free_mem_shm(td);
    251 	else if (td->o.mem_type == MEM_MMAP || td->o.mem_type == MEM_MMAPHUGE)
    252 		free_mem_mmap(td, total_mem);
    253 	else
    254 		log_err("Bad memory type %u\n", td->o.mem_type);
    255 
    256 	td->orig_buffer = NULL;
    257 	td->orig_buffer_size = 0;
    258 }
    259