Home | History | Annotate | Download | only in micro_bench
      1 /*
      2 ** Copyright 2010 The Android Open Source Project
      3 **
      4 ** Licensed under the Apache License, Version 2.0 (the "License");
      5 ** you may not use this file except in compliance with the License.
      6 ** You may obtain a copy of the License at
      7 **
      8 **     http://www.apache.org/licenses/LICENSE-2.0
      9 **
     10 ** Unless required by applicable law or agreed to in writing, software
     11 ** distributed under the License is distributed on an "AS IS" BASIS,
     12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13 ** See the License for the specific language governing permissions and
     14 ** limitations under the License.
     15 */
     16 
     17 /*
     18  * Micro-benchmarking of sleep/cpu speed/memcpy/memset/memory reads/strcmp.
     19  */
     20 
     21 #include <stdio.h>
     22 #include <stdlib.h>
     23 #include <string.h>
     24 #include <ctype.h>
     25 #include <math.h>
     26 #include <sched.h>
     27 #include <sys/resource.h>
     28 #include <time.h>
     29 #include <unistd.h>
     30 
     31 // The default size of data that will be manipulated in each iteration of
     32 // a memory benchmark. Can be modified with the --data_size option.
     33 #define DEFAULT_DATA_SIZE       1000000000
     34 
     35 // The amount of memory allocated for the cold benchmarks to use.
     36 #define DEFAULT_COLD_DATA_SIZE  128*1024*1024
     37 
     38 // The default size of the stride between each buffer for cold benchmarks.
     39 #define DEFAULT_COLD_STRIDE_SIZE  4096
     40 
     41 // Number of nanoseconds in a second.
     42 #define NS_PER_SEC              1000000000
     43 
     44 // The maximum number of arguments that a benchmark will accept.
     45 #define MAX_ARGS    2
     46 
     47 // Default memory alignment of malloc.
     48 #define DEFAULT_MALLOC_MEMORY_ALIGNMENT   8
     49 
     50 // Contains information about benchmark options.
     51 typedef struct {
     52     bool print_average;
     53     bool print_each_iter;
     54 
     55     int dst_align;
     56     int dst_or_mask;
     57     int src_align;
     58     int src_or_mask;
     59 
     60     int cpu_to_lock;
     61 
     62     int data_size;
     63     int dst_str_size;
     64     int cold_data_size;
     65     int cold_stride_size;
     66 
     67     int args[MAX_ARGS];
     68     int num_args;
     69 } command_data_t;
     70 
     71 typedef void *(*void_func_t)();
     72 typedef void *(*memcpy_func_t)(void *, const void *, size_t);
     73 typedef void *(*memset_func_t)(void *, int, size_t);
     74 typedef int (*strcmp_func_t)(const char *, const char *);
     75 typedef char *(*str_func_t)(char *, const char *);
     76 typedef size_t (*strlen_func_t)(const char *);
     77 
     78 // Struct that contains a mapping of benchmark name to benchmark function.
     79 typedef struct {
     80     const char *name;
     81     int (*ptr)(const char *, const command_data_t &, void_func_t func);
     82     void_func_t func;
     83 } function_t;
     84 
     85 // Get the current time in nanoseconds.
     86 uint64_t nanoTime() {
     87   struct timespec t;
     88 
     89   t.tv_sec = t.tv_nsec = 0;
     90   clock_gettime(CLOCK_MONOTONIC, &t);
     91   return static_cast<uint64_t>(t.tv_sec) * NS_PER_SEC + t.tv_nsec;
     92 }
     93 
     94 // Static analyzer warns about potential memory leak of orig_ptr
     95 // in getAlignedMemory. That is true and the callers in this program
     96 // do not free orig_ptr. But, we don't care about that in this
     97 // going-obsolete test program. So, here is a hack to trick the
     98 // static analyzer.
     99 static void *saved_orig_ptr;
    100 
    101 // Allocate memory with a specific alignment and return that pointer.
    102 // This function assumes an alignment value that is a power of 2.
    103 // If the alignment is 0, then use the pointer returned by malloc.
    104 uint8_t *getAlignedMemory(uint8_t *orig_ptr, int alignment, int or_mask) {
    105   uint64_t ptr = reinterpret_cast<uint64_t>(orig_ptr);
    106   saved_orig_ptr = orig_ptr;
    107   if (alignment > 0) {
    108       // When setting the alignment, set it to exactly the alignment chosen.
    109       // The pointer returned will be guaranteed not to be aligned to anything
    110       // more than that.
    111       ptr += alignment - (ptr & (alignment - 1));
    112       ptr |= alignment | or_mask;
    113   }
    114 
    115   return reinterpret_cast<uint8_t*>(ptr);
    116 }
    117 
    118 // Allocate memory with a specific alignment and return that pointer.
    119 // This function assumes an alignment value that is a power of 2.
    120 // If the alignment is 0, then use the pointer returned by malloc.
    121 uint8_t *allocateAlignedMemory(size_t size, int alignment, int or_mask) {
    122   uint64_t ptr = reinterpret_cast<uint64_t>(malloc(size + 3 * alignment));
    123   if (!ptr)
    124       return NULL;
    125   return getAlignedMemory((uint8_t*)ptr, alignment, or_mask);
    126 }
    127 
    128 void initString(uint8_t *buf, size_t size) {
    129     for (size_t i = 0; i < size - 1; i++) {
    130         buf[i] = static_cast<char>(32 + (i % 96));
    131     }
    132     buf[size-1] = '\0';
    133 }
    134 
    135 static inline double computeAverage(uint64_t time_ns, size_t size, size_t copies) {
    136     return ((size/1024.0) * copies) / ((double)time_ns/NS_PER_SEC);
    137 }
    138 
    139 static inline double computeRunningAvg(double avg, double running_avg, size_t cur_idx) {
    140     return (running_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1));
    141 }
    142 
    143 static inline double computeRunningSquareAvg(double avg, double square_avg, size_t cur_idx) {
    144     return (square_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)) * avg;
    145 }
    146 
    147 static inline double computeStdDev(double square_avg, double running_avg) {
    148     return sqrt(square_avg - running_avg * running_avg);
    149 }
    150 
    151 static inline void printIter(uint64_t time_ns, const char *name, size_t size, size_t copies, double avg) {
    152     printf("%s %zux%zu bytes took %.06f seconds (%f MB/s)\n",
    153            name, copies, size, (double)time_ns/NS_PER_SEC, avg/1024.0);
    154 }
    155 
    156 static inline void printSummary(uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, double running_avg, double std_dev, double min, double max) {
    157     printf("  %s %zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n",
    158            name, copies, size, running_avg/1024.0, std_dev/1024.0, min/1024.0,
    159            max/1024.0);
    160 }
    161 
    162 // For the cold benchmarks, a large buffer will be created which
    163 // contains many "size" buffers. This function will figure out the increment
    164 // needed between each buffer so that each one is aligned to "alignment".
    165 int getAlignmentIncrement(size_t size, int alignment) {
    166     if (alignment == 0) {
    167         alignment = DEFAULT_MALLOC_MEMORY_ALIGNMENT;
    168     }
    169     alignment *= 2;
    170     return size + alignment - (size % alignment);
    171 }
    172 
    173 uint8_t *getColdBuffer(int num_buffers, size_t incr, int alignment, int or_mask) {
    174     uint8_t *buffers = reinterpret_cast<uint8_t*>(malloc(num_buffers * incr + 3 * alignment));
    175     if (!buffers) {
    176         return NULL;
    177     }
    178     return getAlignedMemory(buffers, alignment, or_mask);
    179 }
    180 
    181 static inline double computeColdAverage(uint64_t time_ns, size_t size, size_t copies, size_t num_buffers) {
    182     return ((size/1024.0) * copies * num_buffers) / ((double)time_ns/NS_PER_SEC);
    183 }
    184 
    185 static void inline printColdIter(uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, double avg) {
    186     printf("%s %zux%zux%zu bytes took %.06f seconds (%f MB/s)\n",
    187            name, copies, num_buffers, size, (double)time_ns/NS_PER_SEC, avg/1024.0);
    188 }
    189 
    190 static void inline printColdSummary(
    191         uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, size_t num_buffers,
    192         double running_avg, double square_avg, double min, double max) {
    193     printf("  %s %zux%zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n",
    194            name, copies, num_buffers, size, running_avg/1024.0,
    195            computeStdDev(running_avg, square_avg)/1024.0, min/1024.0, max/1024.0);
    196 }
    197 
    198 #define MAINLOOP(cmd_data, BENCH, COMPUTE_AVG, PRINT_ITER, PRINT_AVG) \
    199     uint64_t time_ns;                                                 \
    200     int iters = cmd_data.args[1];                                     \
    201     bool print_average = cmd_data.print_average;                      \
    202     bool print_each_iter = cmd_data.print_each_iter;                  \
    203     double min = 0.0, max = 0.0, running_avg = 0.0, square_avg = 0.0; \
    204     double avg;                                                       \
    205     for (int i = 0; iters == -1 || i < iters; i++) {                  \
    206         time_ns = nanoTime();                                         \
    207         BENCH;                                                        \
    208         time_ns = nanoTime() - time_ns;                               \
    209         avg = COMPUTE_AVG;                                            \
    210         if (print_average) {                                          \
    211             running_avg = computeRunningAvg(avg, running_avg, i);     \
    212             square_avg = computeRunningSquareAvg(avg, square_avg, i); \
    213             if (min == 0.0 || avg < min) {                            \
    214                 min = avg;                                            \
    215             }                                                         \
    216             if (avg > max) {                                          \
    217                 max = avg;                                            \
    218             }                                                         \
    219         }                                                             \
    220         if (print_each_iter) {                                        \
    221             PRINT_ITER;                                               \
    222         }                                                             \
    223     }                                                                 \
    224     if (print_average) {                                              \
    225         PRINT_AVG;                                                    \
    226     }
    227 
    228 #define MAINLOOP_DATA(name, cmd_data, size, BENCH)                    \
    229     size_t copies = cmd_data.data_size/size;                          \
    230     size_t j;                                                         \
    231     MAINLOOP(cmd_data,                                                \
    232              for (j = 0; j < copies; j++) {                           \
    233                  BENCH;                                               \
    234              },                                                       \
    235              computeAverage(time_ns, size, copies),                   \
    236              printIter(time_ns, name, size, copies, avg),             \
    237              double std_dev = computeStdDev(square_avg, running_avg); \
    238              printSummary(time_ns, name, size, copies, running_avg,   \
    239                           std_dev, min, max));
    240 
    241 #define MAINLOOP_COLD(name, cmd_data, size, num_incrs, BENCH)                 \
    242     size_t num_strides = num_buffers / num_incrs;                             \
    243     if ((num_buffers % num_incrs) != 0) {                                     \
    244         num_strides--;                                                        \
    245     }                                                                         \
    246     size_t copies = 1;                                                        \
    247     num_buffers = num_incrs * num_strides;                                    \
    248     if (num_buffers * size < static_cast<size_t>(cmd_data.data_size)) {       \
    249         copies = cmd_data.data_size / (num_buffers * size);                   \
    250     }                                                                         \
    251     if (num_strides == 0) {                                                   \
    252         printf("%s: Chosen options lead to no copies, aborting.\n", name);    \
    253         return -1;                                                            \
    254     }                                                                         \
    255     size_t j, k;                                                              \
    256     MAINLOOP(cmd_data,                                                        \
    257              for (j = 0; j < copies; j++) {                                   \
    258                  for (k = 0; k < num_incrs; k++) {                            \
    259                      BENCH;                                                   \
    260                 }                                                             \
    261             },                                                                \
    262             computeColdAverage(time_ns, size, copies, num_buffers),           \
    263             printColdIter(time_ns, name, size, copies, num_buffers, avg),     \
    264             printColdSummary(time_ns, name, size, copies, num_buffers,        \
    265                              running_avg, square_avg, min, max));
    266 
    267 // This version of the macro creates a single buffer of the given size and
    268 // alignment. The variable "buf" will be a pointer to the buffer and should
    269 // be used by the BENCH code.
    270 // INIT - Any specialized code needed to initialize the data. This will only
    271 //        be executed once.
    272 // BENCH - The actual code to benchmark and is timed.
    273 #define BENCH_ONE_BUF(name, cmd_data, INIT, BENCH)                            \
    274     size_t size = cmd_data.args[0]; \
    275     uint8_t *buf = allocateAlignedMemory(size, cmd_data.dst_align, cmd_data.dst_or_mask); \
    276     if (!buf)                                                                 \
    277         return -1;                                                            \
    278     INIT;                                                                     \
    279     MAINLOOP_DATA(name, cmd_data, size, BENCH);
    280 
    281 // This version of the macro creates two buffers of the given sizes and
    282 // alignments. The variables "buf1" and "buf2" will be pointers to the
    283 // buffers and should be used by the BENCH code.
    284 // INIT - Any specialized code needed to initialize the data. This will only
    285 //        be executed once.
    286 // BENCH - The actual code to benchmark and is timed.
    287 #define BENCH_TWO_BUFS(name, cmd_data, INIT, BENCH)                           \
    288     size_t size = cmd_data.args[0];                                           \
    289     uint8_t *buf1 = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); \
    290     if (!buf1)                                                                \
    291         return -1;                                                            \
    292     size_t total_size = size;                                                 \
    293     if (cmd_data.dst_str_size > 0)                                            \
    294         total_size += cmd_data.dst_str_size;                                  \
    295     uint8_t *buf2 = allocateAlignedMemory(total_size, cmd_data.dst_align, cmd_data.dst_or_mask); \
    296     if (!buf2)                                                                \
    297         return -1;                                                            \
    298     INIT;                                                                     \
    299     MAINLOOP_DATA(name, cmd_data, size, BENCH);
    300 
    301 // This version of the macro attempts to benchmark code when the data
    302 // being manipulated is not in the cache, thus the cache is cold. It does
    303 // this by creating a single large buffer that is designed to be larger than
    304 // the largest cache in the system. The variable "buf" will be one slice
    305 // of the buffer that the BENCH code should use that is of the correct size
    306 // and alignment. In order to avoid any algorithms that prefetch past the end
    307 // of their "buf" and into the next sequential buffer, the code strides
    308 // through the buffer. Specifically, as "buf" values are iterated in BENCH
    309 // code, the end of "buf" is guaranteed to be at least "stride_size" away
    310 // from the next "buf".
    311 // INIT - Any specialized code needed to initialize the data. This will only
    312 //        be executed once.
    313 // BENCH - The actual code to benchmark and is timed.
    314 #define COLD_ONE_BUF(name, cmd_data, INIT, BENCH)                             \
    315     size_t size = cmd_data.args[0];                                           \
    316     size_t incr = getAlignmentIncrement(size, cmd_data.dst_align);            \
    317     size_t num_buffers = cmd_data.cold_data_size / incr;                      \
    318     size_t buffer_size = num_buffers * incr;                                  \
    319     uint8_t *buffer = getColdBuffer(num_buffers, incr, cmd_data.dst_align, cmd_data.dst_or_mask); \
    320     if (!buffer)                                                              \
    321         return -1;                                                            \
    322     size_t num_incrs = cmd_data.cold_stride_size / incr + 1;                  \
    323     size_t stride_incr = incr * num_incrs;                                    \
    324     uint8_t *buf;                                                             \
    325     size_t l;                                                                 \
    326     INIT;                                                                     \
    327     MAINLOOP_COLD(name, cmd_data, size, num_incrs,                            \
    328                   buf = buffer + k * incr;                                    \
    329                   for (l = 0; l < num_strides; l++) {                         \
    330                       BENCH;                                                  \
    331                       buf += stride_incr;                                     \
    332                   });
    333 
    334 // This version of the macro attempts to benchmark code when the data
    335 // being manipulated is not in the cache, thus the cache is cold. It does
    336 // this by creating two large buffers each of which is designed to be
    337 // larger than the largest cache in the system. Two variables "buf1" and
    338 // "buf2" will be the two buffers that BENCH code should use. In order
    339 // to avoid any algorithms that prefetch past the end of either "buf1"
    340 // or "buf2" and into the next sequential buffer, the code strides through
    341 // both buffers. Specifically, as "buf1" and "buf2" values are iterated in
    342 // BENCH code, the end of "buf1" and "buf2" is guaranteed to be at least
    343 // "stride_size" away from the next "buf1" and "buf2".
    344 // INIT - Any specialized code needed to initialize the data. This will only
    345 //        be executed once.
    346 // BENCH - The actual code to benchmark and is timed.
    347 #define COLD_TWO_BUFS(name, cmd_data, INIT, BENCH)                            \
    348     size_t size = cmd_data.args[0];                                           \
    349     size_t buf1_incr = getAlignmentIncrement(size, cmd_data.src_align);       \
    350     size_t total_size = size;                                                 \
    351     if (cmd_data.dst_str_size > 0)                                            \
    352         total_size += cmd_data.dst_str_size;                                  \
    353     size_t buf2_incr = getAlignmentIncrement(total_size, cmd_data.dst_align); \
    354     size_t max_incr = (buf1_incr > buf2_incr) ? buf1_incr : buf2_incr;        \
    355     size_t num_buffers = cmd_data.cold_data_size / max_incr;                  \
    356     size_t buffer1_size = num_buffers * buf1_incr;                            \
    357     size_t buffer2_size = num_buffers * buf2_incr;                            \
    358     uint8_t *buffer1 = getColdBuffer(num_buffers, buf1_incr, cmd_data.src_align, cmd_data.src_or_mask); \
    359     if (!buffer1)                                                             \
    360         return -1;                                                            \
    361     uint8_t *buffer2 = getColdBuffer(num_buffers, buf2_incr, cmd_data.dst_align, cmd_data.dst_or_mask); \
    362     if (!buffer2)                                                             \
    363         return -1;                                                            \
    364     size_t min_incr = (buf1_incr < buf2_incr) ? buf1_incr : buf2_incr;        \
    365     size_t num_incrs = cmd_data.cold_stride_size / min_incr + 1;              \
    366     size_t buf1_stride_incr = buf1_incr * num_incrs;                          \
    367     size_t buf2_stride_incr = buf2_incr * num_incrs;                          \
    368     size_t l;                                                                 \
    369     uint8_t *buf1;                                                            \
    370     uint8_t *buf2;                                                            \
    371     INIT;                                                                     \
    372     MAINLOOP_COLD(name, cmd_data, size, num_incrs,                            \
    373                   buf1 = buffer1 + k * buf1_incr;                             \
    374                   buf2 = buffer2 + k * buf2_incr;                             \
    375                   for (l = 0; l < num_strides; l++) {                         \
    376                       BENCH;                                                  \
    377                       buf1 += buf1_stride_incr;                               \
    378                       buf2 += buf2_stride_incr;                               \
    379                   });
    380 
    381 int benchmarkSleep(const char* /*name*/, const command_data_t &cmd_data, void_func_t /*func*/) {
    382     int delay = cmd_data.args[0];
    383     MAINLOOP(cmd_data, sleep(delay),
    384              (double)time_ns/NS_PER_SEC,
    385              printf("sleep(%d) took %.06f seconds\n", delay, avg);,
    386              printf("  sleep(%d) average %.06f seconds std dev %f min %.06f seconds max %0.6f seconds\n", \
    387                     delay, running_avg, computeStdDev(square_avg, running_avg), \
    388                     min, max));
    389 
    390     return 0;
    391 }
    392 
    393 int benchmarkMemset(const char *name, const command_data_t &cmd_data, void_func_t func) {
    394     memset_func_t memset_func = reinterpret_cast<memset_func_t>(func);
    395     BENCH_ONE_BUF(name, cmd_data, ;, memset_func(buf, i, size));
    396 
    397     return 0;
    398 }
    399 
    400 int benchmarkMemsetCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    401     memset_func_t memset_func = reinterpret_cast<memset_func_t>(func);
    402     COLD_ONE_BUF(name, cmd_data, ;, memset_func(buf, l, size));
    403 
    404     return 0;
    405 }
    406 
    407 int benchmarkMemcpy(const char *name, const command_data_t &cmd_data, void_func_t func) {
    408     memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func);
    409 
    410     BENCH_TWO_BUFS(name, cmd_data,
    411                    memset(buf1, 0xff, size); \
    412                    memset(buf2, 0, size),
    413                    memcpy_func(buf2, buf1, size));
    414 
    415     return 0;
    416 }
    417 
    418 int benchmarkMemcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    419     memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func);
    420 
    421     COLD_TWO_BUFS(name, cmd_data,
    422                   memset(buffer1, 0xff, buffer1_size); \
    423                   memset(buffer2, 0x0, buffer2_size),
    424                   memcpy_func(buf2, buf1, size));
    425 
    426     return 0;
    427 }
    428 
    429 int benchmarkMemmoveBackwards(const char *name, const command_data_t &cmd_data, void_func_t func) {
    430     memcpy_func_t memmove_func = reinterpret_cast<memcpy_func_t>(func);
    431 
    432     size_t size = cmd_data.args[0];
    433     size_t alloc_size = size * 2 + 3 * cmd_data.dst_align;
    434     uint8_t* src = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask);
    435     if (!src)
    436         return -1;
    437     // Force memmove to do a backwards copy by getting a pointer into the source buffer.
    438     uint8_t* dst = getAlignedMemory(src+1, cmd_data.dst_align, cmd_data.dst_or_mask);
    439     if (!dst)
    440         return -1;
    441     MAINLOOP_DATA(name, cmd_data, size, memmove_func(dst, src, size));
    442     return 0;
    443 }
    444 
    445 int benchmarkMemread(const char *name, const command_data_t &cmd_data, void_func_t /*func*/) {
    446     int size = cmd_data.args[0];
    447 
    448     uint32_t *src = reinterpret_cast<uint32_t*>(malloc(size));
    449     if (!src)
    450         return -1;
    451     memset(src, 0xff, size);
    452 
    453     // Use volatile so the compiler does not optimize away the reads.
    454     volatile int foo;
    455     size_t k;
    456     MAINLOOP_DATA(name, cmd_data, size,
    457                   for (k = 0; k < size/sizeof(uint32_t); k++) foo = src[k]);
    458     free(src);
    459 
    460     return 0;
    461 }
    462 
    463 int benchmarkStrcmp(const char *name, const command_data_t &cmd_data, void_func_t func) {
    464     strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func);
    465 
    466     int retval;
    467     BENCH_TWO_BUFS(name, cmd_data,
    468                    initString(buf1, size); \
    469                    initString(buf2, size),
    470                    retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \
    471                    if (retval != 0) printf("%s failed, return value %d\n", name, retval));
    472 
    473     return 0;
    474 }
    475 
    476 int benchmarkStrcmpCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    477     strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func);
    478 
    479     int retval;
    480     COLD_TWO_BUFS(name, cmd_data,
    481                   memset(buffer1, 'a', buffer1_size); \
    482                   memset(buffer2, 'a', buffer2_size); \
    483                   for (size_t i =0; i < num_buffers; i++) { \
    484                       buffer1[size-1+buf1_incr*i] = '\0'; \
    485                       buffer2[size-1+buf2_incr*i] = '\0'; \
    486                   },
    487                   retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \
    488                   if (retval != 0) printf("%s failed, return value %d\n", name, retval));
    489 
    490     return 0;
    491 }
    492 
    493 int benchmarkStrlen(const char *name, const command_data_t &cmd_data, void_func_t func) {
    494     size_t real_size;
    495     strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func);
    496     BENCH_ONE_BUF(name, cmd_data,
    497                   initString(buf, size),
    498                   real_size = strlen_func(reinterpret_cast<char*>(buf)); \
    499                   if (real_size + 1 != size) { \
    500                       printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \
    501                       return -1; \
    502                   });
    503 
    504     return 0;
    505 }
    506 
    507 int benchmarkStrlenCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    508     strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func);
    509     size_t real_size;
    510     COLD_ONE_BUF(name, cmd_data,
    511                  memset(buffer, 'a', buffer_size); \
    512                  for (size_t i = 0; i < num_buffers; i++) { \
    513                      buffer[size-1+incr*i] = '\0'; \
    514                  },
    515                  real_size = strlen_func(reinterpret_cast<char*>(buf)); \
    516                  if (real_size + 1 != size) { \
    517                      printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \
    518                      return -1; \
    519                  });
    520     return 0;
    521 }
    522 
    523 int benchmarkStrcat(const char *name, const command_data_t &cmd_data, void_func_t func) {
    524     str_func_t str_func = reinterpret_cast<str_func_t>(func);
    525 
    526     int dst_str_size = cmd_data.dst_str_size;
    527     if (dst_str_size <= 0) {
    528         printf("%s requires --dst_str_size to be set to a non-zero value.\n",
    529                name);
    530         return -1;
    531     }
    532     BENCH_TWO_BUFS(name, cmd_data,
    533                    initString(buf1, size); \
    534                    initString(buf2, dst_str_size),
    535                    str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0');
    536 
    537     return 0;
    538 }
    539 
    540 int benchmarkStrcatCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    541     str_func_t str_func = reinterpret_cast<str_func_t>(func);
    542 
    543     int dst_str_size = cmd_data.dst_str_size;
    544     if (dst_str_size <= 0) {
    545         printf("%s requires --dst_str_size to be set to a non-zero value.\n",
    546                name);
    547         return -1;
    548     }
    549     COLD_TWO_BUFS(name, cmd_data,
    550                   memset(buffer1, 'a', buffer1_size); \
    551                   memset(buffer2, 'b', buffer2_size); \
    552                   for (size_t i = 0; i < num_buffers; i++) { \
    553                       buffer1[size-1+buf1_incr*i] = '\0'; \
    554                       buffer2[dst_str_size-1+buf2_incr*i] = '\0'; \
    555                   },
    556                   str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0');
    557 
    558     return 0;
    559 }
    560 
    561 
    562 int benchmarkStrcpy(const char *name, const command_data_t &cmd_data, void_func_t func) {
    563     str_func_t str_func = reinterpret_cast<str_func_t>(func);
    564 
    565     BENCH_TWO_BUFS(name, cmd_data,
    566                    initString(buf1, size); \
    567                    memset(buf2, 0, size),
    568                    str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)));
    569 
    570     return 0;
    571 }
    572 
    573 int benchmarkStrcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) {
    574     str_func_t str_func = reinterpret_cast<str_func_t>(func);
    575 
    576     COLD_TWO_BUFS(name, cmd_data,
    577                   memset(buffer1, 'a', buffer1_size); \
    578                   for (size_t i = 0; i < num_buffers; i++) { \
    579                      buffer1[size-1+buf1_incr*i] = '\0'; \
    580                   } \
    581                   memset(buffer2, 0, buffer2_size),
    582                   str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)));
    583 
    584     return 0;
    585 }
    586 
    587 // Create the mapping structure.
    588 function_t function_table[] = {
    589     { "memcpy", benchmarkMemcpy, reinterpret_cast<void_func_t>(memcpy) },
    590     { "memcpy_cold", benchmarkMemcpyCold, reinterpret_cast<void_func_t>(memcpy) },
    591     { "memmove_forward", benchmarkMemcpy, reinterpret_cast<void_func_t>(memmove) },
    592     { "memmove_backward", benchmarkMemmoveBackwards, reinterpret_cast<void_func_t>(memmove) },
    593     { "memread", benchmarkMemread, NULL },
    594     { "memset", benchmarkMemset, reinterpret_cast<void_func_t>(memset) },
    595     { "memset_cold", benchmarkMemsetCold, reinterpret_cast<void_func_t>(memset) },
    596     { "sleep", benchmarkSleep, NULL },
    597     { "strcat", benchmarkStrcat, reinterpret_cast<void_func_t>(strcat) },
    598     { "strcat_cold", benchmarkStrcatCold, reinterpret_cast<void_func_t>(strcat) },
    599     { "strcmp", benchmarkStrcmp, reinterpret_cast<void_func_t>(strcmp) },
    600     { "strcmp_cold", benchmarkStrcmpCold, reinterpret_cast<void_func_t>(strcmp) },
    601     { "strcpy", benchmarkStrcpy, reinterpret_cast<void_func_t>(strcpy) },
    602     { "strcpy_cold", benchmarkStrcpyCold, reinterpret_cast<void_func_t>(strcpy) },
    603     { "strlen", benchmarkStrlen, reinterpret_cast<void_func_t>(strlen) },
    604     { "strlen_cold", benchmarkStrlenCold, reinterpret_cast<void_func_t>(strlen) },
    605 };
    606 
    607 void usage() {
    608     printf("Usage:\n");
    609     printf("  micro_bench [--data_size DATA_BYTES] [--print_average]\n");
    610     printf("              [--no_print_each_iter] [--lock_to_cpu CORE]\n");
    611     printf("              [--src_align ALIGN] [--src_or_mask OR_MASK]\n");
    612     printf("              [--dst_align ALIGN] [--dst_or_mask OR_MASK]\n");
    613     printf("              [--dst_str_size SIZE] [--cold_data_size DATA_BYTES]\n");
    614     printf("              [--cold_stride_size SIZE]\n");
    615     printf("    --data_size DATA_BYTES\n");
    616     printf("      For the data benchmarks (memcpy/memset/memread) the approximate\n");
    617     printf("      size of data, in bytes, that will be manipulated in each iteration.\n");
    618     printf("    --print_average\n");
    619     printf("      Print the average and standard deviation of all iterations.\n");
    620     printf("    --no_print_each_iter\n");
    621     printf("      Do not print any values in each iteration.\n");
    622     printf("    --lock_to_cpu CORE\n");
    623     printf("      Lock to the specified CORE. The default is to use the last core found.\n");
    624     printf("    --dst_align ALIGN\n");
    625     printf("      If the command supports it, align the destination pointer to ALIGN.\n");
    626     printf("      The default is to use the value returned by malloc.\n");
    627     printf("    --dst_or_mask OR_MASK\n");
    628     printf("      If the command supports it, or in the OR_MASK on to the destination pointer.\n");
    629     printf("      The OR_MASK must be smaller than the dst_align value.\n");
    630     printf("      The default value is 0.\n");
    631 
    632     printf("    --src_align ALIGN\n");
    633     printf("      If the command supports it, align the source pointer to ALIGN. The default is to use the\n");
    634     printf("      value returned by malloc.\n");
    635     printf("    --src_or_mask OR_MASK\n");
    636     printf("      If the command supports it, or in the OR_MASK on to the source pointer.\n");
    637     printf("      The OR_MASK must be smaller than the src_align value.\n");
    638     printf("      The default value is 0.\n");
    639     printf("    --dst_str_size SIZE\n");
    640     printf("      If the command supports it, create a destination string of this length.\n");
    641     printf("      The default is to not update the destination string.\n");
    642     printf("    --cold_data_size DATA_SIZE\n");
    643     printf("      For _cold benchmarks, use this as the total amount of memory to use.\n");
    644     printf("      The default is 128MB, and the number should be larger than the cache on the chip.\n");
    645     printf("      This value is specified in bytes.\n");
    646     printf("    --cold_stride_size SIZE\n");
    647     printf("      For _cold benchmarks, use this as the minimum stride between iterations.\n");
    648     printf("      The default is 4096 bytes and the number should be larger than the amount of data\n");
    649     printf("      pulled in to the cache by each run of the benchmark.\n");
    650     printf("    ITERS\n");
    651     printf("      The number of iterations to execute each benchmark. If not\n");
    652     printf("      passed in then run forever.\n");
    653     printf("  micro_bench cpu UNUSED [ITERS]\n");
    654     printf("  micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memcpy NUM_BYTES [ITERS]\n");
    655     printf("  micro_bench memread NUM_BYTES [ITERS]\n");
    656     printf("  micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memset NUM_BYTES [ITERS]\n");
    657     printf("  micro_bench sleep TIME_TO_SLEEP [ITERS]\n");
    658     printf("    TIME_TO_SLEEP\n");
    659     printf("      The time in seconds to sleep.\n");
    660     printf("  micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] [--dst_str_size SIZE] strcat NUM_BYTES [ITERS]\n");
    661     printf("  micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask OR_MASK] strcmp NUM_BYTES [ITERS]\n");
    662     printf("  micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] strcpy NUM_BYTES [ITERS]\n");
    663     printf("  micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] strlen NUM_BYTES [ITERS]\n");
    664     printf("\n");
    665     printf("  In addition, memcpy/memcpy/memset/strcat/strcpy/strlen have _cold versions\n");
    666     printf("  that will execute the function on a buffer not in the cache.\n");
    667 }
    668 
    669 function_t *processOptions(int argc, char **argv, command_data_t *cmd_data) {
    670     function_t *command = NULL;
    671 
    672     // Initialize the command_flags.
    673     cmd_data->print_average = false;
    674     cmd_data->print_each_iter = true;
    675     cmd_data->dst_align = 0;
    676     cmd_data->src_align = 0;
    677     cmd_data->src_or_mask = 0;
    678     cmd_data->dst_or_mask = 0;
    679     cmd_data->num_args = 0;
    680     cmd_data->cpu_to_lock = -1;
    681     cmd_data->data_size = DEFAULT_DATA_SIZE;
    682     cmd_data->dst_str_size = -1;
    683     cmd_data->cold_data_size = DEFAULT_COLD_DATA_SIZE;
    684     cmd_data->cold_stride_size = DEFAULT_COLD_STRIDE_SIZE;
    685     for (int i = 0; i < MAX_ARGS; i++) {
    686         cmd_data->args[i] = -1;
    687     }
    688 
    689     for (int i = 1; i < argc; i++) {
    690         if (argv[i][0] == '-') {
    691             int *save_value = NULL;
    692             if (strcmp(argv[i], "--print_average") == 0) {
    693                 cmd_data->print_average = true;
    694             } else if (strcmp(argv[i], "--no_print_each_iter") == 0) {
    695                 cmd_data->print_each_iter = false;
    696             } else if (strcmp(argv[i], "--dst_align") == 0) {
    697                 save_value = &cmd_data->dst_align;
    698             } else if (strcmp(argv[i], "--src_align") == 0) {
    699                 save_value = &cmd_data->src_align;
    700             } else if (strcmp(argv[i], "--dst_or_mask") == 0) {
    701                 save_value = &cmd_data->dst_or_mask;
    702             } else if (strcmp(argv[i], "--src_or_mask") == 0) {
    703                 save_value = &cmd_data->src_or_mask;
    704             } else if (strcmp(argv[i], "--lock_to_cpu") == 0) {
    705                 save_value = &cmd_data->cpu_to_lock;
    706             } else if (strcmp(argv[i], "--data_size") == 0) {
    707                 save_value = &cmd_data->data_size;
    708             } else if (strcmp(argv[i], "--dst_str_size") == 0) {
    709                 save_value = &cmd_data->dst_str_size;
    710             } else if (strcmp(argv[i], "--cold_data_size") == 0) {
    711                 save_value = &cmd_data->cold_data_size;
    712             } else if (strcmp(argv[i], "--cold_stride_size") == 0) {
    713                 save_value = &cmd_data->cold_stride_size;
    714             } else {
    715                 printf("Unknown option %s\n", argv[i]);
    716                 return NULL;
    717             }
    718             if (save_value) {
    719                 // Checking both characters without a strlen() call should be
    720                 // safe since as long as the argument exists, one character will
    721                 // be present (\0). And if the first character is '-', then
    722                 // there will always be a second character (\0 again).
    723                 if (i == argc - 1 || (argv[i + 1][0] == '-' && !isdigit(argv[i + 1][1]))) {
    724                     printf("The option %s requires one argument.\n",
    725                            argv[i]);
    726                     return NULL;
    727                 }
    728                 *save_value = (int)strtol(argv[++i], NULL, 0);
    729             }
    730         } else if (!command) {
    731             for (size_t j = 0; j < sizeof(function_table)/sizeof(function_t); j++) {
    732                 if (strcmp(argv[i], function_table[j].name) == 0) {
    733                     command = &function_table[j];
    734                     break;
    735                 }
    736             }
    737             if (!command) {
    738                 printf("Uknown command %s\n", argv[i]);
    739                 return NULL;
    740             }
    741         } else if (cmd_data->num_args > MAX_ARGS) {
    742             printf("More than %d number arguments passed in.\n", MAX_ARGS);
    743             return NULL;
    744         } else {
    745             cmd_data->args[cmd_data->num_args++] = atoi(argv[i]);
    746         }
    747     }
    748 
    749     // Check the arguments passed in make sense.
    750     if (cmd_data->num_args != 1 && cmd_data->num_args != 2) {
    751         printf("Not enough arguments passed in.\n");
    752         return NULL;
    753     } else if (cmd_data->dst_align < 0) {
    754         printf("The --dst_align option must be greater than or equal to 0.\n");
    755         return NULL;
    756     } else if (cmd_data->src_align < 0) {
    757         printf("The --src_align option must be greater than or equal to 0.\n");
    758         return NULL;
    759     } else if (cmd_data->data_size <= 0) {
    760         printf("The --data_size option must be a positive number.\n");
    761         return NULL;
    762     } else if ((cmd_data->dst_align & (cmd_data->dst_align - 1))) {
    763         printf("The --dst_align option must be a power of 2.\n");
    764         return NULL;
    765     } else if ((cmd_data->src_align & (cmd_data->src_align - 1))) {
    766         printf("The --src_align option must be a power of 2.\n");
    767         return NULL;
    768     } else if (!cmd_data->src_align && cmd_data->src_or_mask) {
    769         printf("The --src_or_mask option requires that --src_align be set.\n");
    770         return NULL;
    771     } else if (!cmd_data->dst_align && cmd_data->dst_or_mask) {
    772         printf("The --dst_or_mask option requires that --dst_align be set.\n");
    773         return NULL;
    774     } else if (cmd_data->src_or_mask > cmd_data->src_align) {
    775         printf("The value of --src_or_mask cannot be larger that --src_align.\n");
    776         return NULL;
    777     } else if (cmd_data->dst_or_mask > cmd_data->dst_align) {
    778         printf("The value of --src_or_mask cannot be larger that --src_align.\n");
    779         return NULL;
    780     }
    781 
    782     return command;
    783 }
    784 
    785 bool raisePriorityAndLock(int cpu_to_lock) {
    786     cpu_set_t cpuset;
    787 
    788     if (setpriority(PRIO_PROCESS, 0, -20)) {
    789         perror("Unable to raise priority of process.\n");
    790         return false;
    791     }
    792 
    793     CPU_ZERO(&cpuset);
    794     if (sched_getaffinity(0, sizeof(cpuset), &cpuset) != 0) {
    795         perror("sched_getaffinity failed");
    796         return false;
    797     }
    798 
    799     if (cpu_to_lock < 0) {
    800         // Lock to the last active core we find.
    801         for (int i = 0; i < CPU_SETSIZE; i++) {
    802             if (CPU_ISSET(i, &cpuset)) {
    803                 cpu_to_lock = i;
    804             }
    805         }
    806     } else if (!CPU_ISSET(cpu_to_lock, &cpuset)) {
    807         printf("Cpu %d does not exist.\n", cpu_to_lock);
    808         return false;
    809     }
    810 
    811     if (cpu_to_lock < 0) {
    812         printf("Cannot find any valid cpu to lock.\n");
    813         return false;
    814     }
    815 
    816     CPU_ZERO(&cpuset);
    817     CPU_SET(cpu_to_lock, &cpuset);
    818     if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) {
    819         perror("sched_setaffinity failed");
    820         return false;
    821     }
    822 
    823     return true;
    824 }
    825 
    826 int main(int argc, char **argv) {
    827     command_data_t cmd_data;
    828 
    829     function_t *command = processOptions(argc, argv, &cmd_data);
    830     if (!command) {
    831       usage();
    832       return -1;
    833     }
    834 
    835     if (!raisePriorityAndLock(cmd_data.cpu_to_lock)) {
    836       return -1;
    837     }
    838 
    839     printf("%s\n", command->name);
    840     return (*command->ptr)(command->name, cmd_data, command->func);
    841 }
    842