1 /* 2 ** Copyright 2010 The Android Open Source Project 3 ** 4 ** Licensed under the Apache License, Version 2.0 (the "License"); 5 ** you may not use this file except in compliance with the License. 6 ** You may obtain a copy of the License at 7 ** 8 ** http://www.apache.org/licenses/LICENSE-2.0 9 ** 10 ** Unless required by applicable law or agreed to in writing, software 11 ** distributed under the License is distributed on an "AS IS" BASIS, 12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 ** See the License for the specific language governing permissions and 14 ** limitations under the License. 15 */ 16 17 /* 18 * Micro-benchmarking of sleep/cpu speed/memcpy/memset/memory reads/strcmp. 19 */ 20 21 #include <stdio.h> 22 #include <stdlib.h> 23 #include <string.h> 24 #include <ctype.h> 25 #include <math.h> 26 #include <sched.h> 27 #include <sys/resource.h> 28 #include <time.h> 29 #include <unistd.h> 30 31 // The default size of data that will be manipulated in each iteration of 32 // a memory benchmark. Can be modified with the --data_size option. 33 #define DEFAULT_DATA_SIZE 1000000000 34 35 // The amount of memory allocated for the cold benchmarks to use. 36 #define DEFAULT_COLD_DATA_SIZE 128*1024*1024 37 38 // The default size of the stride between each buffer for cold benchmarks. 39 #define DEFAULT_COLD_STRIDE_SIZE 4096 40 41 // Number of nanoseconds in a second. 42 #define NS_PER_SEC 1000000000 43 44 // The maximum number of arguments that a benchmark will accept. 45 #define MAX_ARGS 2 46 47 // Default memory alignment of malloc. 48 #define DEFAULT_MALLOC_MEMORY_ALIGNMENT 8 49 50 // Contains information about benchmark options. 51 typedef struct { 52 bool print_average; 53 bool print_each_iter; 54 55 int dst_align; 56 int dst_or_mask; 57 int src_align; 58 int src_or_mask; 59 60 int cpu_to_lock; 61 62 int data_size; 63 int dst_str_size; 64 int cold_data_size; 65 int cold_stride_size; 66 67 int args[MAX_ARGS]; 68 int num_args; 69 } command_data_t; 70 71 typedef void *(*void_func_t)(); 72 typedef void *(*memcpy_func_t)(void *, const void *, size_t); 73 typedef void *(*memset_func_t)(void *, int, size_t); 74 typedef int (*strcmp_func_t)(const char *, const char *); 75 typedef char *(*str_func_t)(char *, const char *); 76 typedef size_t (*strlen_func_t)(const char *); 77 78 // Struct that contains a mapping of benchmark name to benchmark function. 79 typedef struct { 80 const char *name; 81 int (*ptr)(const char *, const command_data_t &, void_func_t func); 82 void_func_t func; 83 } function_t; 84 85 // Get the current time in nanoseconds. 86 uint64_t nanoTime() { 87 struct timespec t; 88 89 t.tv_sec = t.tv_nsec = 0; 90 clock_gettime(CLOCK_MONOTONIC, &t); 91 return static_cast<uint64_t>(t.tv_sec) * NS_PER_SEC + t.tv_nsec; 92 } 93 94 // Static analyzer warns about potential memory leak of orig_ptr 95 // in getAlignedMemory. That is true and the callers in this program 96 // do not free orig_ptr. But, we don't care about that in this 97 // going-obsolete test program. So, here is a hack to trick the 98 // static analyzer. 99 static void *saved_orig_ptr; 100 101 // Allocate memory with a specific alignment and return that pointer. 102 // This function assumes an alignment value that is a power of 2. 103 // If the alignment is 0, then use the pointer returned by malloc. 104 uint8_t *getAlignedMemory(uint8_t *orig_ptr, int alignment, int or_mask) { 105 uint64_t ptr = reinterpret_cast<uint64_t>(orig_ptr); 106 saved_orig_ptr = orig_ptr; 107 if (alignment > 0) { 108 // When setting the alignment, set it to exactly the alignment chosen. 109 // The pointer returned will be guaranteed not to be aligned to anything 110 // more than that. 111 ptr += alignment - (ptr & (alignment - 1)); 112 ptr |= alignment | or_mask; 113 } 114 115 return reinterpret_cast<uint8_t*>(ptr); 116 } 117 118 // Allocate memory with a specific alignment and return that pointer. 119 // This function assumes an alignment value that is a power of 2. 120 // If the alignment is 0, then use the pointer returned by malloc. 121 uint8_t *allocateAlignedMemory(size_t size, int alignment, int or_mask) { 122 uint64_t ptr = reinterpret_cast<uint64_t>(malloc(size + 3 * alignment)); 123 if (!ptr) 124 return NULL; 125 return getAlignedMemory((uint8_t*)ptr, alignment, or_mask); 126 } 127 128 void initString(uint8_t *buf, size_t size) { 129 for (size_t i = 0; i < size - 1; i++) { 130 buf[i] = static_cast<char>(32 + (i % 96)); 131 } 132 buf[size-1] = '\0'; 133 } 134 135 static inline double computeAverage(uint64_t time_ns, size_t size, size_t copies) { 136 return ((size/1024.0) * copies) / ((double)time_ns/NS_PER_SEC); 137 } 138 139 static inline double computeRunningAvg(double avg, double running_avg, size_t cur_idx) { 140 return (running_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)); 141 } 142 143 static inline double computeRunningSquareAvg(double avg, double square_avg, size_t cur_idx) { 144 return (square_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)) * avg; 145 } 146 147 static inline double computeStdDev(double square_avg, double running_avg) { 148 return sqrt(square_avg - running_avg * running_avg); 149 } 150 151 static inline void printIter(uint64_t time_ns, const char *name, size_t size, size_t copies, double avg) { 152 printf("%s %zux%zu bytes took %.06f seconds (%f MB/s)\n", 153 name, copies, size, (double)time_ns/NS_PER_SEC, avg/1024.0); 154 } 155 156 static inline void printSummary(uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, double running_avg, double std_dev, double min, double max) { 157 printf(" %s %zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", 158 name, copies, size, running_avg/1024.0, std_dev/1024.0, min/1024.0, 159 max/1024.0); 160 } 161 162 // For the cold benchmarks, a large buffer will be created which 163 // contains many "size" buffers. This function will figure out the increment 164 // needed between each buffer so that each one is aligned to "alignment". 165 int getAlignmentIncrement(size_t size, int alignment) { 166 if (alignment == 0) { 167 alignment = DEFAULT_MALLOC_MEMORY_ALIGNMENT; 168 } 169 alignment *= 2; 170 return size + alignment - (size % alignment); 171 } 172 173 uint8_t *getColdBuffer(int num_buffers, size_t incr, int alignment, int or_mask) { 174 uint8_t *buffers = reinterpret_cast<uint8_t*>(malloc(num_buffers * incr + 3 * alignment)); 175 if (!buffers) { 176 return NULL; 177 } 178 return getAlignedMemory(buffers, alignment, or_mask); 179 } 180 181 static inline double computeColdAverage(uint64_t time_ns, size_t size, size_t copies, size_t num_buffers) { 182 return ((size/1024.0) * copies * num_buffers) / ((double)time_ns/NS_PER_SEC); 183 } 184 185 static void inline printColdIter(uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, double avg) { 186 printf("%s %zux%zux%zu bytes took %.06f seconds (%f MB/s)\n", 187 name, copies, num_buffers, size, (double)time_ns/NS_PER_SEC, avg/1024.0); 188 } 189 190 static void inline printColdSummary( 191 uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, size_t num_buffers, 192 double running_avg, double square_avg, double min, double max) { 193 printf(" %s %zux%zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", 194 name, copies, num_buffers, size, running_avg/1024.0, 195 computeStdDev(running_avg, square_avg)/1024.0, min/1024.0, max/1024.0); 196 } 197 198 #define MAINLOOP(cmd_data, BENCH, COMPUTE_AVG, PRINT_ITER, PRINT_AVG) \ 199 uint64_t time_ns; \ 200 int iters = cmd_data.args[1]; \ 201 bool print_average = cmd_data.print_average; \ 202 bool print_each_iter = cmd_data.print_each_iter; \ 203 double min = 0.0, max = 0.0, running_avg = 0.0, square_avg = 0.0; \ 204 double avg; \ 205 for (int i = 0; iters == -1 || i < iters; i++) { \ 206 time_ns = nanoTime(); \ 207 BENCH; \ 208 time_ns = nanoTime() - time_ns; \ 209 avg = COMPUTE_AVG; \ 210 if (print_average) { \ 211 running_avg = computeRunningAvg(avg, running_avg, i); \ 212 square_avg = computeRunningSquareAvg(avg, square_avg, i); \ 213 if (min == 0.0 || avg < min) { \ 214 min = avg; \ 215 } \ 216 if (avg > max) { \ 217 max = avg; \ 218 } \ 219 } \ 220 if (print_each_iter) { \ 221 PRINT_ITER; \ 222 } \ 223 } \ 224 if (print_average) { \ 225 PRINT_AVG; \ 226 } 227 228 #define MAINLOOP_DATA(name, cmd_data, size, BENCH) \ 229 size_t copies = cmd_data.data_size/size; \ 230 size_t j; \ 231 MAINLOOP(cmd_data, \ 232 for (j = 0; j < copies; j++) { \ 233 BENCH; \ 234 }, \ 235 computeAverage(time_ns, size, copies), \ 236 printIter(time_ns, name, size, copies, avg), \ 237 double std_dev = computeStdDev(square_avg, running_avg); \ 238 printSummary(time_ns, name, size, copies, running_avg, \ 239 std_dev, min, max)); 240 241 #define MAINLOOP_COLD(name, cmd_data, size, num_incrs, BENCH) \ 242 size_t num_strides = num_buffers / num_incrs; \ 243 if ((num_buffers % num_incrs) != 0) { \ 244 num_strides--; \ 245 } \ 246 size_t copies = 1; \ 247 num_buffers = num_incrs * num_strides; \ 248 if (num_buffers * size < static_cast<size_t>(cmd_data.data_size)) { \ 249 copies = cmd_data.data_size / (num_buffers * size); \ 250 } \ 251 if (num_strides == 0) { \ 252 printf("%s: Chosen options lead to no copies, aborting.\n", name); \ 253 return -1; \ 254 } \ 255 size_t j, k; \ 256 MAINLOOP(cmd_data, \ 257 for (j = 0; j < copies; j++) { \ 258 for (k = 0; k < num_incrs; k++) { \ 259 BENCH; \ 260 } \ 261 }, \ 262 computeColdAverage(time_ns, size, copies, num_buffers), \ 263 printColdIter(time_ns, name, size, copies, num_buffers, avg), \ 264 printColdSummary(time_ns, name, size, copies, num_buffers, \ 265 running_avg, square_avg, min, max)); 266 267 // This version of the macro creates a single buffer of the given size and 268 // alignment. The variable "buf" will be a pointer to the buffer and should 269 // be used by the BENCH code. 270 // INIT - Any specialized code needed to initialize the data. This will only 271 // be executed once. 272 // BENCH - The actual code to benchmark and is timed. 273 #define BENCH_ONE_BUF(name, cmd_data, INIT, BENCH) \ 274 size_t size = cmd_data.args[0]; \ 275 uint8_t *buf = allocateAlignedMemory(size, cmd_data.dst_align, cmd_data.dst_or_mask); \ 276 if (!buf) \ 277 return -1; \ 278 INIT; \ 279 MAINLOOP_DATA(name, cmd_data, size, BENCH); 280 281 // This version of the macro creates two buffers of the given sizes and 282 // alignments. The variables "buf1" and "buf2" will be pointers to the 283 // buffers and should be used by the BENCH code. 284 // INIT - Any specialized code needed to initialize the data. This will only 285 // be executed once. 286 // BENCH - The actual code to benchmark and is timed. 287 #define BENCH_TWO_BUFS(name, cmd_data, INIT, BENCH) \ 288 size_t size = cmd_data.args[0]; \ 289 uint8_t *buf1 = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); \ 290 if (!buf1) \ 291 return -1; \ 292 size_t total_size = size; \ 293 if (cmd_data.dst_str_size > 0) \ 294 total_size += cmd_data.dst_str_size; \ 295 uint8_t *buf2 = allocateAlignedMemory(total_size, cmd_data.dst_align, cmd_data.dst_or_mask); \ 296 if (!buf2) \ 297 return -1; \ 298 INIT; \ 299 MAINLOOP_DATA(name, cmd_data, size, BENCH); 300 301 // This version of the macro attempts to benchmark code when the data 302 // being manipulated is not in the cache, thus the cache is cold. It does 303 // this by creating a single large buffer that is designed to be larger than 304 // the largest cache in the system. The variable "buf" will be one slice 305 // of the buffer that the BENCH code should use that is of the correct size 306 // and alignment. In order to avoid any algorithms that prefetch past the end 307 // of their "buf" and into the next sequential buffer, the code strides 308 // through the buffer. Specifically, as "buf" values are iterated in BENCH 309 // code, the end of "buf" is guaranteed to be at least "stride_size" away 310 // from the next "buf". 311 // INIT - Any specialized code needed to initialize the data. This will only 312 // be executed once. 313 // BENCH - The actual code to benchmark and is timed. 314 #define COLD_ONE_BUF(name, cmd_data, INIT, BENCH) \ 315 size_t size = cmd_data.args[0]; \ 316 size_t incr = getAlignmentIncrement(size, cmd_data.dst_align); \ 317 size_t num_buffers = cmd_data.cold_data_size / incr; \ 318 size_t buffer_size = num_buffers * incr; \ 319 uint8_t *buffer = getColdBuffer(num_buffers, incr, cmd_data.dst_align, cmd_data.dst_or_mask); \ 320 if (!buffer) \ 321 return -1; \ 322 size_t num_incrs = cmd_data.cold_stride_size / incr + 1; \ 323 size_t stride_incr = incr * num_incrs; \ 324 uint8_t *buf; \ 325 size_t l; \ 326 INIT; \ 327 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \ 328 buf = buffer + k * incr; \ 329 for (l = 0; l < num_strides; l++) { \ 330 BENCH; \ 331 buf += stride_incr; \ 332 }); 333 334 // This version of the macro attempts to benchmark code when the data 335 // being manipulated is not in the cache, thus the cache is cold. It does 336 // this by creating two large buffers each of which is designed to be 337 // larger than the largest cache in the system. Two variables "buf1" and 338 // "buf2" will be the two buffers that BENCH code should use. In order 339 // to avoid any algorithms that prefetch past the end of either "buf1" 340 // or "buf2" and into the next sequential buffer, the code strides through 341 // both buffers. Specifically, as "buf1" and "buf2" values are iterated in 342 // BENCH code, the end of "buf1" and "buf2" is guaranteed to be at least 343 // "stride_size" away from the next "buf1" and "buf2". 344 // INIT - Any specialized code needed to initialize the data. This will only 345 // be executed once. 346 // BENCH - The actual code to benchmark and is timed. 347 #define COLD_TWO_BUFS(name, cmd_data, INIT, BENCH) \ 348 size_t size = cmd_data.args[0]; \ 349 size_t buf1_incr = getAlignmentIncrement(size, cmd_data.src_align); \ 350 size_t total_size = size; \ 351 if (cmd_data.dst_str_size > 0) \ 352 total_size += cmd_data.dst_str_size; \ 353 size_t buf2_incr = getAlignmentIncrement(total_size, cmd_data.dst_align); \ 354 size_t max_incr = (buf1_incr > buf2_incr) ? buf1_incr : buf2_incr; \ 355 size_t num_buffers = cmd_data.cold_data_size / max_incr; \ 356 size_t buffer1_size = num_buffers * buf1_incr; \ 357 size_t buffer2_size = num_buffers * buf2_incr; \ 358 uint8_t *buffer1 = getColdBuffer(num_buffers, buf1_incr, cmd_data.src_align, cmd_data.src_or_mask); \ 359 if (!buffer1) \ 360 return -1; \ 361 uint8_t *buffer2 = getColdBuffer(num_buffers, buf2_incr, cmd_data.dst_align, cmd_data.dst_or_mask); \ 362 if (!buffer2) \ 363 return -1; \ 364 size_t min_incr = (buf1_incr < buf2_incr) ? buf1_incr : buf2_incr; \ 365 size_t num_incrs = cmd_data.cold_stride_size / min_incr + 1; \ 366 size_t buf1_stride_incr = buf1_incr * num_incrs; \ 367 size_t buf2_stride_incr = buf2_incr * num_incrs; \ 368 size_t l; \ 369 uint8_t *buf1; \ 370 uint8_t *buf2; \ 371 INIT; \ 372 MAINLOOP_COLD(name, cmd_data, size, num_incrs, \ 373 buf1 = buffer1 + k * buf1_incr; \ 374 buf2 = buffer2 + k * buf2_incr; \ 375 for (l = 0; l < num_strides; l++) { \ 376 BENCH; \ 377 buf1 += buf1_stride_incr; \ 378 buf2 += buf2_stride_incr; \ 379 }); 380 381 int benchmarkSleep(const char* /*name*/, const command_data_t &cmd_data, void_func_t /*func*/) { 382 int delay = cmd_data.args[0]; 383 MAINLOOP(cmd_data, sleep(delay), 384 (double)time_ns/NS_PER_SEC, 385 printf("sleep(%d) took %.06f seconds\n", delay, avg);, 386 printf(" sleep(%d) average %.06f seconds std dev %f min %.06f seconds max %0.6f seconds\n", \ 387 delay, running_avg, computeStdDev(square_avg, running_avg), \ 388 min, max)); 389 390 return 0; 391 } 392 393 int benchmarkMemset(const char *name, const command_data_t &cmd_data, void_func_t func) { 394 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); 395 BENCH_ONE_BUF(name, cmd_data, ;, memset_func(buf, i, size)); 396 397 return 0; 398 } 399 400 int benchmarkMemsetCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 401 memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); 402 COLD_ONE_BUF(name, cmd_data, ;, memset_func(buf, l, size)); 403 404 return 0; 405 } 406 407 int benchmarkMemcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { 408 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); 409 410 BENCH_TWO_BUFS(name, cmd_data, 411 memset(buf1, 0xff, size); \ 412 memset(buf2, 0, size), 413 memcpy_func(buf2, buf1, size)); 414 415 return 0; 416 } 417 418 int benchmarkMemcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 419 memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); 420 421 COLD_TWO_BUFS(name, cmd_data, 422 memset(buffer1, 0xff, buffer1_size); \ 423 memset(buffer2, 0x0, buffer2_size), 424 memcpy_func(buf2, buf1, size)); 425 426 return 0; 427 } 428 429 int benchmarkMemmoveBackwards(const char *name, const command_data_t &cmd_data, void_func_t func) { 430 memcpy_func_t memmove_func = reinterpret_cast<memcpy_func_t>(func); 431 432 size_t size = cmd_data.args[0]; 433 size_t alloc_size = size * 2 + 3 * cmd_data.dst_align; 434 uint8_t* src = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); 435 if (!src) 436 return -1; 437 // Force memmove to do a backwards copy by getting a pointer into the source buffer. 438 uint8_t* dst = getAlignedMemory(src+1, cmd_data.dst_align, cmd_data.dst_or_mask); 439 if (!dst) 440 return -1; 441 MAINLOOP_DATA(name, cmd_data, size, memmove_func(dst, src, size)); 442 return 0; 443 } 444 445 int benchmarkMemread(const char *name, const command_data_t &cmd_data, void_func_t /*func*/) { 446 int size = cmd_data.args[0]; 447 448 uint32_t *src = reinterpret_cast<uint32_t*>(malloc(size)); 449 if (!src) 450 return -1; 451 memset(src, 0xff, size); 452 453 // Use volatile so the compiler does not optimize away the reads. 454 volatile int foo; 455 size_t k; 456 MAINLOOP_DATA(name, cmd_data, size, 457 for (k = 0; k < size/sizeof(uint32_t); k++) foo = src[k]); 458 free(src); 459 460 return 0; 461 } 462 463 int benchmarkStrcmp(const char *name, const command_data_t &cmd_data, void_func_t func) { 464 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); 465 466 int retval; 467 BENCH_TWO_BUFS(name, cmd_data, 468 initString(buf1, size); \ 469 initString(buf2, size), 470 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ 471 if (retval != 0) printf("%s failed, return value %d\n", name, retval)); 472 473 return 0; 474 } 475 476 int benchmarkStrcmpCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 477 strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); 478 479 int retval; 480 COLD_TWO_BUFS(name, cmd_data, 481 memset(buffer1, 'a', buffer1_size); \ 482 memset(buffer2, 'a', buffer2_size); \ 483 for (size_t i =0; i < num_buffers; i++) { \ 484 buffer1[size-1+buf1_incr*i] = '\0'; \ 485 buffer2[size-1+buf2_incr*i] = '\0'; \ 486 }, 487 retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ 488 if (retval != 0) printf("%s failed, return value %d\n", name, retval)); 489 490 return 0; 491 } 492 493 int benchmarkStrlen(const char *name, const command_data_t &cmd_data, void_func_t func) { 494 size_t real_size; 495 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); 496 BENCH_ONE_BUF(name, cmd_data, 497 initString(buf, size), 498 real_size = strlen_func(reinterpret_cast<char*>(buf)); \ 499 if (real_size + 1 != size) { \ 500 printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \ 501 return -1; \ 502 }); 503 504 return 0; 505 } 506 507 int benchmarkStrlenCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 508 strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); 509 size_t real_size; 510 COLD_ONE_BUF(name, cmd_data, 511 memset(buffer, 'a', buffer_size); \ 512 for (size_t i = 0; i < num_buffers; i++) { \ 513 buffer[size-1+incr*i] = '\0'; \ 514 }, 515 real_size = strlen_func(reinterpret_cast<char*>(buf)); \ 516 if (real_size + 1 != size) { \ 517 printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \ 518 return -1; \ 519 }); 520 return 0; 521 } 522 523 int benchmarkStrcat(const char *name, const command_data_t &cmd_data, void_func_t func) { 524 str_func_t str_func = reinterpret_cast<str_func_t>(func); 525 526 int dst_str_size = cmd_data.dst_str_size; 527 if (dst_str_size <= 0) { 528 printf("%s requires --dst_str_size to be set to a non-zero value.\n", 529 name); 530 return -1; 531 } 532 BENCH_TWO_BUFS(name, cmd_data, 533 initString(buf1, size); \ 534 initString(buf2, dst_str_size), 535 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); 536 537 return 0; 538 } 539 540 int benchmarkStrcatCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 541 str_func_t str_func = reinterpret_cast<str_func_t>(func); 542 543 int dst_str_size = cmd_data.dst_str_size; 544 if (dst_str_size <= 0) { 545 printf("%s requires --dst_str_size to be set to a non-zero value.\n", 546 name); 547 return -1; 548 } 549 COLD_TWO_BUFS(name, cmd_data, 550 memset(buffer1, 'a', buffer1_size); \ 551 memset(buffer2, 'b', buffer2_size); \ 552 for (size_t i = 0; i < num_buffers; i++) { \ 553 buffer1[size-1+buf1_incr*i] = '\0'; \ 554 buffer2[dst_str_size-1+buf2_incr*i] = '\0'; \ 555 }, 556 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); 557 558 return 0; 559 } 560 561 562 int benchmarkStrcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { 563 str_func_t str_func = reinterpret_cast<str_func_t>(func); 564 565 BENCH_TWO_BUFS(name, cmd_data, 566 initString(buf1, size); \ 567 memset(buf2, 0, size), 568 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); 569 570 return 0; 571 } 572 573 int benchmarkStrcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { 574 str_func_t str_func = reinterpret_cast<str_func_t>(func); 575 576 COLD_TWO_BUFS(name, cmd_data, 577 memset(buffer1, 'a', buffer1_size); \ 578 for (size_t i = 0; i < num_buffers; i++) { \ 579 buffer1[size-1+buf1_incr*i] = '\0'; \ 580 } \ 581 memset(buffer2, 0, buffer2_size), 582 str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); 583 584 return 0; 585 } 586 587 // Create the mapping structure. 588 function_t function_table[] = { 589 { "memcpy", benchmarkMemcpy, reinterpret_cast<void_func_t>(memcpy) }, 590 { "memcpy_cold", benchmarkMemcpyCold, reinterpret_cast<void_func_t>(memcpy) }, 591 { "memmove_forward", benchmarkMemcpy, reinterpret_cast<void_func_t>(memmove) }, 592 { "memmove_backward", benchmarkMemmoveBackwards, reinterpret_cast<void_func_t>(memmove) }, 593 { "memread", benchmarkMemread, NULL }, 594 { "memset", benchmarkMemset, reinterpret_cast<void_func_t>(memset) }, 595 { "memset_cold", benchmarkMemsetCold, reinterpret_cast<void_func_t>(memset) }, 596 { "sleep", benchmarkSleep, NULL }, 597 { "strcat", benchmarkStrcat, reinterpret_cast<void_func_t>(strcat) }, 598 { "strcat_cold", benchmarkStrcatCold, reinterpret_cast<void_func_t>(strcat) }, 599 { "strcmp", benchmarkStrcmp, reinterpret_cast<void_func_t>(strcmp) }, 600 { "strcmp_cold", benchmarkStrcmpCold, reinterpret_cast<void_func_t>(strcmp) }, 601 { "strcpy", benchmarkStrcpy, reinterpret_cast<void_func_t>(strcpy) }, 602 { "strcpy_cold", benchmarkStrcpyCold, reinterpret_cast<void_func_t>(strcpy) }, 603 { "strlen", benchmarkStrlen, reinterpret_cast<void_func_t>(strlen) }, 604 { "strlen_cold", benchmarkStrlenCold, reinterpret_cast<void_func_t>(strlen) }, 605 }; 606 607 void usage() { 608 printf("Usage:\n"); 609 printf(" micro_bench [--data_size DATA_BYTES] [--print_average]\n"); 610 printf(" [--no_print_each_iter] [--lock_to_cpu CORE]\n"); 611 printf(" [--src_align ALIGN] [--src_or_mask OR_MASK]\n"); 612 printf(" [--dst_align ALIGN] [--dst_or_mask OR_MASK]\n"); 613 printf(" [--dst_str_size SIZE] [--cold_data_size DATA_BYTES]\n"); 614 printf(" [--cold_stride_size SIZE]\n"); 615 printf(" --data_size DATA_BYTES\n"); 616 printf(" For the data benchmarks (memcpy/memset/memread) the approximate\n"); 617 printf(" size of data, in bytes, that will be manipulated in each iteration.\n"); 618 printf(" --print_average\n"); 619 printf(" Print the average and standard deviation of all iterations.\n"); 620 printf(" --no_print_each_iter\n"); 621 printf(" Do not print any values in each iteration.\n"); 622 printf(" --lock_to_cpu CORE\n"); 623 printf(" Lock to the specified CORE. The default is to use the last core found.\n"); 624 printf(" --dst_align ALIGN\n"); 625 printf(" If the command supports it, align the destination pointer to ALIGN.\n"); 626 printf(" The default is to use the value returned by malloc.\n"); 627 printf(" --dst_or_mask OR_MASK\n"); 628 printf(" If the command supports it, or in the OR_MASK on to the destination pointer.\n"); 629 printf(" The OR_MASK must be smaller than the dst_align value.\n"); 630 printf(" The default value is 0.\n"); 631 632 printf(" --src_align ALIGN\n"); 633 printf(" If the command supports it, align the source pointer to ALIGN. The default is to use the\n"); 634 printf(" value returned by malloc.\n"); 635 printf(" --src_or_mask OR_MASK\n"); 636 printf(" If the command supports it, or in the OR_MASK on to the source pointer.\n"); 637 printf(" The OR_MASK must be smaller than the src_align value.\n"); 638 printf(" The default value is 0.\n"); 639 printf(" --dst_str_size SIZE\n"); 640 printf(" If the command supports it, create a destination string of this length.\n"); 641 printf(" The default is to not update the destination string.\n"); 642 printf(" --cold_data_size DATA_SIZE\n"); 643 printf(" For _cold benchmarks, use this as the total amount of memory to use.\n"); 644 printf(" The default is 128MB, and the number should be larger than the cache on the chip.\n"); 645 printf(" This value is specified in bytes.\n"); 646 printf(" --cold_stride_size SIZE\n"); 647 printf(" For _cold benchmarks, use this as the minimum stride between iterations.\n"); 648 printf(" The default is 4096 bytes and the number should be larger than the amount of data\n"); 649 printf(" pulled in to the cache by each run of the benchmark.\n"); 650 printf(" ITERS\n"); 651 printf(" The number of iterations to execute each benchmark. If not\n"); 652 printf(" passed in then run forever.\n"); 653 printf(" micro_bench cpu UNUSED [ITERS]\n"); 654 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memcpy NUM_BYTES [ITERS]\n"); 655 printf(" micro_bench memread NUM_BYTES [ITERS]\n"); 656 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memset NUM_BYTES [ITERS]\n"); 657 printf(" micro_bench sleep TIME_TO_SLEEP [ITERS]\n"); 658 printf(" TIME_TO_SLEEP\n"); 659 printf(" The time in seconds to sleep.\n"); 660 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] [--dst_str_size SIZE] strcat NUM_BYTES [ITERS]\n"); 661 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask OR_MASK] strcmp NUM_BYTES [ITERS]\n"); 662 printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] strcpy NUM_BYTES [ITERS]\n"); 663 printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] strlen NUM_BYTES [ITERS]\n"); 664 printf("\n"); 665 printf(" In addition, memcpy/memcpy/memset/strcat/strcpy/strlen have _cold versions\n"); 666 printf(" that will execute the function on a buffer not in the cache.\n"); 667 } 668 669 function_t *processOptions(int argc, char **argv, command_data_t *cmd_data) { 670 function_t *command = NULL; 671 672 // Initialize the command_flags. 673 cmd_data->print_average = false; 674 cmd_data->print_each_iter = true; 675 cmd_data->dst_align = 0; 676 cmd_data->src_align = 0; 677 cmd_data->src_or_mask = 0; 678 cmd_data->dst_or_mask = 0; 679 cmd_data->num_args = 0; 680 cmd_data->cpu_to_lock = -1; 681 cmd_data->data_size = DEFAULT_DATA_SIZE; 682 cmd_data->dst_str_size = -1; 683 cmd_data->cold_data_size = DEFAULT_COLD_DATA_SIZE; 684 cmd_data->cold_stride_size = DEFAULT_COLD_STRIDE_SIZE; 685 for (int i = 0; i < MAX_ARGS; i++) { 686 cmd_data->args[i] = -1; 687 } 688 689 for (int i = 1; i < argc; i++) { 690 if (argv[i][0] == '-') { 691 int *save_value = NULL; 692 if (strcmp(argv[i], "--print_average") == 0) { 693 cmd_data->print_average = true; 694 } else if (strcmp(argv[i], "--no_print_each_iter") == 0) { 695 cmd_data->print_each_iter = false; 696 } else if (strcmp(argv[i], "--dst_align") == 0) { 697 save_value = &cmd_data->dst_align; 698 } else if (strcmp(argv[i], "--src_align") == 0) { 699 save_value = &cmd_data->src_align; 700 } else if (strcmp(argv[i], "--dst_or_mask") == 0) { 701 save_value = &cmd_data->dst_or_mask; 702 } else if (strcmp(argv[i], "--src_or_mask") == 0) { 703 save_value = &cmd_data->src_or_mask; 704 } else if (strcmp(argv[i], "--lock_to_cpu") == 0) { 705 save_value = &cmd_data->cpu_to_lock; 706 } else if (strcmp(argv[i], "--data_size") == 0) { 707 save_value = &cmd_data->data_size; 708 } else if (strcmp(argv[i], "--dst_str_size") == 0) { 709 save_value = &cmd_data->dst_str_size; 710 } else if (strcmp(argv[i], "--cold_data_size") == 0) { 711 save_value = &cmd_data->cold_data_size; 712 } else if (strcmp(argv[i], "--cold_stride_size") == 0) { 713 save_value = &cmd_data->cold_stride_size; 714 } else { 715 printf("Unknown option %s\n", argv[i]); 716 return NULL; 717 } 718 if (save_value) { 719 // Checking both characters without a strlen() call should be 720 // safe since as long as the argument exists, one character will 721 // be present (\0). And if the first character is '-', then 722 // there will always be a second character (\0 again). 723 if (i == argc - 1 || (argv[i + 1][0] == '-' && !isdigit(argv[i + 1][1]))) { 724 printf("The option %s requires one argument.\n", 725 argv[i]); 726 return NULL; 727 } 728 *save_value = (int)strtol(argv[++i], NULL, 0); 729 } 730 } else if (!command) { 731 for (size_t j = 0; j < sizeof(function_table)/sizeof(function_t); j++) { 732 if (strcmp(argv[i], function_table[j].name) == 0) { 733 command = &function_table[j]; 734 break; 735 } 736 } 737 if (!command) { 738 printf("Uknown command %s\n", argv[i]); 739 return NULL; 740 } 741 } else if (cmd_data->num_args > MAX_ARGS) { 742 printf("More than %d number arguments passed in.\n", MAX_ARGS); 743 return NULL; 744 } else { 745 cmd_data->args[cmd_data->num_args++] = atoi(argv[i]); 746 } 747 } 748 749 // Check the arguments passed in make sense. 750 if (cmd_data->num_args != 1 && cmd_data->num_args != 2) { 751 printf("Not enough arguments passed in.\n"); 752 return NULL; 753 } else if (cmd_data->dst_align < 0) { 754 printf("The --dst_align option must be greater than or equal to 0.\n"); 755 return NULL; 756 } else if (cmd_data->src_align < 0) { 757 printf("The --src_align option must be greater than or equal to 0.\n"); 758 return NULL; 759 } else if (cmd_data->data_size <= 0) { 760 printf("The --data_size option must be a positive number.\n"); 761 return NULL; 762 } else if ((cmd_data->dst_align & (cmd_data->dst_align - 1))) { 763 printf("The --dst_align option must be a power of 2.\n"); 764 return NULL; 765 } else if ((cmd_data->src_align & (cmd_data->src_align - 1))) { 766 printf("The --src_align option must be a power of 2.\n"); 767 return NULL; 768 } else if (!cmd_data->src_align && cmd_data->src_or_mask) { 769 printf("The --src_or_mask option requires that --src_align be set.\n"); 770 return NULL; 771 } else if (!cmd_data->dst_align && cmd_data->dst_or_mask) { 772 printf("The --dst_or_mask option requires that --dst_align be set.\n"); 773 return NULL; 774 } else if (cmd_data->src_or_mask > cmd_data->src_align) { 775 printf("The value of --src_or_mask cannot be larger that --src_align.\n"); 776 return NULL; 777 } else if (cmd_data->dst_or_mask > cmd_data->dst_align) { 778 printf("The value of --src_or_mask cannot be larger that --src_align.\n"); 779 return NULL; 780 } 781 782 return command; 783 } 784 785 bool raisePriorityAndLock(int cpu_to_lock) { 786 cpu_set_t cpuset; 787 788 if (setpriority(PRIO_PROCESS, 0, -20)) { 789 perror("Unable to raise priority of process.\n"); 790 return false; 791 } 792 793 CPU_ZERO(&cpuset); 794 if (sched_getaffinity(0, sizeof(cpuset), &cpuset) != 0) { 795 perror("sched_getaffinity failed"); 796 return false; 797 } 798 799 if (cpu_to_lock < 0) { 800 // Lock to the last active core we find. 801 for (int i = 0; i < CPU_SETSIZE; i++) { 802 if (CPU_ISSET(i, &cpuset)) { 803 cpu_to_lock = i; 804 } 805 } 806 } else if (!CPU_ISSET(cpu_to_lock, &cpuset)) { 807 printf("Cpu %d does not exist.\n", cpu_to_lock); 808 return false; 809 } 810 811 if (cpu_to_lock < 0) { 812 printf("Cannot find any valid cpu to lock.\n"); 813 return false; 814 } 815 816 CPU_ZERO(&cpuset); 817 CPU_SET(cpu_to_lock, &cpuset); 818 if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) { 819 perror("sched_setaffinity failed"); 820 return false; 821 } 822 823 return true; 824 } 825 826 int main(int argc, char **argv) { 827 command_data_t cmd_data; 828 829 function_t *command = processOptions(argc, argv, &cmd_data); 830 if (!command) { 831 usage(); 832 return -1; 833 } 834 835 if (!raisePriorityAndLock(cmd_data.cpu_to_lock)) { 836 return -1; 837 } 838 839 printf("%s\n", command->name); 840 return (*command->ptr)(command->name, cmd_data, command->func); 841 } 842
