Home | History | Annotate | Download | only in test 
      1 
      2 #undef NDEBUG
      3 #include <utility>
      4 
      5 #include "benchmark/benchmark.h"
      6 #include "output_test.h"
      7 
      8 // ========================================================================= //
      9 // ---------------------- Testing Prologue Output -------------------------- //
     10 // ========================================================================= //
     11 
     12 ADD_CASES(TC_ConsoleOut,
     13           {{"^[-]+$", MR_Next},
     14            {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
     15            {"^[-]+$", MR_Next}});
     16 static int AddContextCases() {
     17   AddCases(TC_ConsoleErr,
     18            {
     19                {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
     20                {"Run on \\(%int X %float MHz CPU s\\)", MR_Next},
     21            });
     22   AddCases(TC_JSONOut, {{"^\\{", MR_Default},
     23                         {"\"context\":", MR_Next},
     24                         {"\"date\": \"", MR_Next},
     25                         {"\"num_cpus\": %int,$", MR_Next},
     26                         {"\"mhz_per_cpu\": %float,$", MR_Next},
     27                         {"\"cpu_scaling_enabled\": ", MR_Next},
     28                         {"\"caches\": \\[$", MR_Next}});
     29   auto const& Caches = benchmark::CPUInfo::Get().caches;
     30   if (!Caches.empty()) {
     31     AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
     32   }
     33   for (size_t I = 0; I < Caches.size(); ++I) {
     34     std::string num_caches_str =
     35         Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
     36     AddCases(
     37         TC_ConsoleErr,
     38         {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}});
     39     AddCases(TC_JSONOut, {{"\\{$", MR_Next},
     40                           {"\"type\": \"", MR_Next},
     41                           {"\"level\": %int,$", MR_Next},
     42                           {"\"size\": %int,$", MR_Next},
     43                           {"\"num_sharing\": %int$", MR_Next},
     44                           {"}[,]{0,1}$", MR_Next}});
     45   }
     46 
     47   AddCases(TC_JSONOut, {{"],$"}});
     48   return 0;
     49 }
     50 int dummy_register = AddContextCases();
     51 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
     52 
     53 // ========================================================================= //
     54 // ------------------------ Testing Basic Output --------------------------- //
     55 // ========================================================================= //
     56 
     57 void BM_basic(benchmark::State& state) {
     58   for (auto _ : state) {
     59   }
     60 }
     61 BENCHMARK(BM_basic);
     62 
     63 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
     64 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
     65                        {"\"iterations\": %int,$", MR_Next},
     66                        {"\"real_time\": %float,$", MR_Next},
     67                        {"\"cpu_time\": %float,$", MR_Next},
     68                        {"\"time_unit\": \"ns\"$", MR_Next},
     69                        {"}", MR_Next}});
     70 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
     71 
     72 // ========================================================================= //
     73 // ------------------------ Testing Bytes per Second Output ---------------- //
     74 // ========================================================================= //
     75 
     76 void BM_bytes_per_second(benchmark::State& state) {
     77   for (auto _ : state) {
     78   }
     79   state.SetBytesProcessed(1);
     80 }
     81 BENCHMARK(BM_bytes_per_second);
     82 
     83 ADD_CASES(TC_ConsoleOut,
     84           {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
     85 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
     86                        {"\"iterations\": %int,$", MR_Next},
     87                        {"\"real_time\": %float,$", MR_Next},
     88                        {"\"cpu_time\": %float,$", MR_Next},
     89                        {"\"time_unit\": \"ns\",$", MR_Next},
     90                        {"\"bytes_per_second\": %float$", MR_Next},
     91                        {"}", MR_Next}});
     92 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
     93 
     94 // ========================================================================= //
     95 // ------------------------ Testing Items per Second Output ---------------- //
     96 // ========================================================================= //
     97 
     98 void BM_items_per_second(benchmark::State& state) {
     99   for (auto _ : state) {
    100   }
    101   state.SetItemsProcessed(1);
    102 }
    103 BENCHMARK(BM_items_per_second);
    104 
    105 ADD_CASES(TC_ConsoleOut,
    106           {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
    107 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
    108                        {"\"iterations\": %int,$", MR_Next},
    109                        {"\"real_time\": %float,$", MR_Next},
    110                        {"\"cpu_time\": %float,$", MR_Next},
    111                        {"\"time_unit\": \"ns\",$", MR_Next},
    112                        {"\"items_per_second\": %float$", MR_Next},
    113                        {"}", MR_Next}});
    114 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
    115 
    116 // ========================================================================= //
    117 // ------------------------ Testing Label Output --------------------------- //
    118 // ========================================================================= //
    119 
    120 void BM_label(benchmark::State& state) {
    121   for (auto _ : state) {
    122   }
    123   state.SetLabel("some label");
    124 }
    125 BENCHMARK(BM_label);
    126 
    127 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
    128 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
    129                        {"\"iterations\": %int,$", MR_Next},
    130                        {"\"real_time\": %float,$", MR_Next},
    131                        {"\"cpu_time\": %float,$", MR_Next},
    132                        {"\"time_unit\": \"ns\",$", MR_Next},
    133                        {"\"label\": \"some label\"$", MR_Next},
    134                        {"}", MR_Next}});
    135 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
    136                        "label\"%csv_label_report_end$"}});
    137 
    138 // ========================================================================= //
    139 // ------------------------ Testing Error Output --------------------------- //
    140 // ========================================================================= //
    141 
    142 void BM_error(benchmark::State& state) {
    143   state.SkipWithError("message");
    144   for (auto _ : state) {
    145   }
    146 }
    147 BENCHMARK(BM_error);
    148 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
    149 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
    150                        {"\"error_occurred\": true,$", MR_Next},
    151                        {"\"error_message\": \"message\",$", MR_Next}});
    152 
    153 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
    154 
    155 // ========================================================================= //
    156 // ------------------------ Testing No Arg Name Output -----------------------
    157 // //
    158 // ========================================================================= //
    159 
    160 void BM_no_arg_name(benchmark::State& state) {
    161   for (auto _ : state) {
    162   }
    163 }
    164 BENCHMARK(BM_no_arg_name)->Arg(3);
    165 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
    166 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
    167 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
    168 
    169 // ========================================================================= //
    170 // ------------------------ Testing Arg Name Output ----------------------- //
    171 // ========================================================================= //
    172 
    173 void BM_arg_name(benchmark::State& state) {
    174   for (auto _ : state) {
    175   }
    176 }
    177 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
    178 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
    179 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
    180 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
    181 
    182 // ========================================================================= //
    183 // ------------------------ Testing Arg Names Output ----------------------- //
    184 // ========================================================================= //
    185 
    186 void BM_arg_names(benchmark::State& state) {
    187   for (auto _ : state) {
    188   }
    189 }
    190 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
    191 ADD_CASES(TC_ConsoleOut,
    192           {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
    193 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
    194 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
    195 
    196 // ========================================================================= //
    197 // ----------------------- Testing Complexity Output ----------------------- //
    198 // ========================================================================= //
    199 
    200 void BM_Complexity_O1(benchmark::State& state) {
    201   for (auto _ : state) {
    202   }
    203   state.SetComplexityN(state.range(0));
    204 }
    205 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
    206 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
    207                    {"%RMS", "[ ]*[0-9]+ %"}});
    208 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
    209                           {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
    210 
    211 // ========================================================================= //
    212 // ----------------------- Testing Aggregate Output ------------------------ //
    213 // ========================================================================= //
    214 
    215 // Test that non-aggregate data is printed by default
    216 void BM_Repeat(benchmark::State& state) {
    217   for (auto _ : state) {
    218   }
    219 }
    220 // need two repetitions min to be able to output any aggregate output
    221 BENCHMARK(BM_Repeat)->Repetitions(2);
    222 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
    223                           {"^BM_Repeat/repeats:2 %console_report$"},
    224                           {"^BM_Repeat/repeats:2_mean %console_report$"},
    225                           {"^BM_Repeat/repeats:2_median %console_report$"},
    226                           {"^BM_Repeat/repeats:2_stddev %console_report$"}});
    227 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
    228                        {"\"name\": \"BM_Repeat/repeats:2\",$"},
    229                        {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
    230                        {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
    231                        {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}});
    232 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
    233                       {"^\"BM_Repeat/repeats:2\",%csv_report$"},
    234                       {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
    235                       {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
    236                       {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
    237 // but for two repetitions, mean and median is the same, so let's repeat..
    238 BENCHMARK(BM_Repeat)->Repetitions(3);
    239 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
    240                           {"^BM_Repeat/repeats:3 %console_report$"},
    241                           {"^BM_Repeat/repeats:3 %console_report$"},
    242                           {"^BM_Repeat/repeats:3_mean %console_report$"},
    243                           {"^BM_Repeat/repeats:3_median %console_report$"},
    244                           {"^BM_Repeat/repeats:3_stddev %console_report$"}});
    245 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
    246                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
    247                        {"\"name\": \"BM_Repeat/repeats:3\",$"},
    248                        {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
    249                        {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
    250                        {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
    251 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
    252                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
    253                       {"^\"BM_Repeat/repeats:3\",%csv_report$"},
    254                       {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
    255                       {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
    256                       {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
    257 // median differs between even/odd number of repetitions, so just to be sure
    258 BENCHMARK(BM_Repeat)->Repetitions(4);
    259 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
    260                           {"^BM_Repeat/repeats:4 %console_report$"},
    261                           {"^BM_Repeat/repeats:4 %console_report$"},
    262                           {"^BM_Repeat/repeats:4 %console_report$"},
    263                           {"^BM_Repeat/repeats:4_mean %console_report$"},
    264                           {"^BM_Repeat/repeats:4_median %console_report$"},
    265                           {"^BM_Repeat/repeats:4_stddev %console_report$"}});
    266 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
    267                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
    268                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
    269                        {"\"name\": \"BM_Repeat/repeats:4\",$"},
    270                        {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
    271                        {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
    272                        {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}});
    273 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
    274                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
    275                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
    276                       {"^\"BM_Repeat/repeats:4\",%csv_report$"},
    277                       {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
    278                       {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
    279                       {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
    280 
    281 // Test that a non-repeated test still prints non-aggregate results even when
    282 // only-aggregate reports have been requested
    283 void BM_RepeatOnce(benchmark::State& state) {
    284   for (auto _ : state) {
    285   }
    286 }
    287 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
    288 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
    289 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
    290 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
    291 
    292 // Test that non-aggregate data is not reported
    293 void BM_SummaryRepeat(benchmark::State& state) {
    294   for (auto _ : state) {
    295   }
    296 }
    297 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
    298 ADD_CASES(TC_ConsoleOut,
    299           {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
    300            {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
    301            {"^BM_SummaryRepeat/repeats:3_median %console_report$"},
    302            {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
    303 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
    304                        {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
    305                        {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
    306                        {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
    307 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
    308                       {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
    309                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
    310                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
    311 
    312 void BM_RepeatTimeUnit(benchmark::State& state) {
    313   for (auto _ : state) {
    314   }
    315 }
    316 BENCHMARK(BM_RepeatTimeUnit)
    317     ->Repetitions(3)
    318     ->ReportAggregatesOnly()
    319     ->Unit(benchmark::kMicrosecond);
    320 ADD_CASES(TC_ConsoleOut,
    321           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
    322            {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
    323            {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
    324            {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
    325 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
    326                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
    327                        {"\"time_unit\": \"us\",?$"},
    328                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
    329                        {"\"time_unit\": \"us\",?$"},
    330                        {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
    331                        {"\"time_unit\": \"us\",?$"}});
    332 ADD_CASES(TC_CSVOut,
    333           {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
    334            {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
    335            {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
    336            {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
    337 
    338 // ========================================================================= //
    339 // -------------------- Testing user-provided statistics ------------------- //
    340 // ========================================================================= //
    341 
    342 const auto UserStatistics = [](const std::vector<double>& v) {
    343   return v.back();
    344 };
    345 void BM_UserStats(benchmark::State& state) {
    346   for (auto _ : state) {
    347   }
    348 }
    349 BENCHMARK(BM_UserStats)
    350     ->Repetitions(3)
    351     ->ComputeStatistics("", UserStatistics);
    352 // check that user-provided stats is calculated, and is after the default-ones
    353 // empty string as name is intentional, it would sort before anything else
    354 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
    355                           {"^BM_UserStats/repeats:3 %console_report$"},
    356                           {"^BM_UserStats/repeats:3 %console_report$"},
    357                           {"^BM_UserStats/repeats:3_mean %console_report$"},
    358                           {"^BM_UserStats/repeats:3_median %console_report$"},
    359                           {"^BM_UserStats/repeats:3_stddev %console_report$"},
    360                           {"^BM_UserStats/repeats:3_ %console_report$"}});
    361 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
    362                        {"\"name\": \"BM_UserStats/repeats:3\",$"},
    363                        {"\"name\": \"BM_UserStats/repeats:3\",$"},
    364                        {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
    365                        {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
    366                        {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
    367                        {"\"name\": \"BM_UserStats/repeats:3_\",$"}});
    368 ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
    369                       {"^\"BM_UserStats/repeats:3\",%csv_report$"},
    370                       {"^\"BM_UserStats/repeats:3\",%csv_report$"},
    371                       {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
    372                       {"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
    373                       {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
    374                       {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
    375 
    376 // ========================================================================= //
    377 // --------------------------- TEST CASES END ------------------------------ //
    378 // ========================================================================= //
    379 
    380 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
    381