1 2 #undef NDEBUG 3 #include <utility> 4 5 #include "benchmark/benchmark.h" 6 #include "output_test.h" 7 8 // ========================================================================= // 9 // ---------------------- Testing Prologue Output -------------------------- // 10 // ========================================================================= // 11 12 ADD_CASES(TC_ConsoleOut, 13 {{"^[-]+$", MR_Next}, 14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, 15 {"^[-]+$", MR_Next}}); 16 ADD_CASES(TC_CSVOut, 17 {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second," 18 "items_per_second,label,error_occurred,error_message"}}); 19 20 // ========================================================================= // 21 // ------------------------ Testing Basic Output --------------------------- // 22 // ========================================================================= // 23 24 void BM_basic(benchmark::State& state) { 25 while (state.KeepRunning()) { 26 } 27 } 28 BENCHMARK(BM_basic); 29 30 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); 31 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, 32 {"\"iterations\": %int,$", MR_Next}, 33 {"\"real_time\": %int,$", MR_Next}, 34 {"\"cpu_time\": %int,$", MR_Next}, 35 {"\"time_unit\": \"ns\"$", MR_Next}, 36 {"}", MR_Next}}); 37 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); 38 39 // ========================================================================= // 40 // ------------------------ Testing Bytes per Second Output ---------------- // 41 // ========================================================================= // 42 43 void BM_bytes_per_second(benchmark::State& state) { 44 while (state.KeepRunning()) { 45 } 46 state.SetBytesProcessed(1); 47 } 48 BENCHMARK(BM_bytes_per_second); 49 50 ADD_CASES(TC_ConsoleOut, 51 {{"^BM_bytes_per_second %console_report +%floatB/s$"}}); 52 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, 53 {"\"iterations\": %int,$", MR_Next}, 54 {"\"real_time\": %int,$", MR_Next}, 55 {"\"cpu_time\": %int,$", MR_Next}, 56 {"\"time_unit\": \"ns\",$", MR_Next}, 57 {"\"bytes_per_second\": %int$", MR_Next}, 58 {"}", MR_Next}}); 59 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); 60 61 // ========================================================================= // 62 // ------------------------ Testing Items per Second Output ---------------- // 63 // ========================================================================= // 64 65 void BM_items_per_second(benchmark::State& state) { 66 while (state.KeepRunning()) { 67 } 68 state.SetItemsProcessed(1); 69 } 70 BENCHMARK(BM_items_per_second); 71 72 ADD_CASES(TC_ConsoleOut, 73 {{"^BM_items_per_second %console_report +%float items/s$"}}); 74 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, 75 {"\"iterations\": %int,$", MR_Next}, 76 {"\"real_time\": %int,$", MR_Next}, 77 {"\"cpu_time\": %int,$", MR_Next}, 78 {"\"time_unit\": \"ns\",$", MR_Next}, 79 {"\"items_per_second\": %int$", MR_Next}, 80 {"}", MR_Next}}); 81 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); 82 83 // ========================================================================= // 84 // ------------------------ Testing Label Output --------------------------- // 85 // ========================================================================= // 86 87 void BM_label(benchmark::State& state) { 88 while (state.KeepRunning()) { 89 } 90 state.SetLabel("some label"); 91 } 92 BENCHMARK(BM_label); 93 94 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); 95 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, 96 {"\"iterations\": %int,$", MR_Next}, 97 {"\"real_time\": %int,$", MR_Next}, 98 {"\"cpu_time\": %int,$", MR_Next}, 99 {"\"time_unit\": \"ns\",$", MR_Next}, 100 {"\"label\": \"some label\"$", MR_Next}, 101 {"}", MR_Next}}); 102 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " 103 "label\"%csv_label_report_end$"}}); 104 105 // ========================================================================= // 106 // ------------------------ Testing Error Output --------------------------- // 107 // ========================================================================= // 108 109 void BM_error(benchmark::State& state) { 110 state.SkipWithError("message"); 111 while (state.KeepRunning()) { 112 } 113 } 114 BENCHMARK(BM_error); 115 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); 116 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, 117 {"\"error_occurred\": true,$", MR_Next}, 118 {"\"error_message\": \"message\",$", MR_Next}}); 119 120 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); 121 122 // ========================================================================= // 123 // ------------------------ Testing No Arg Name Output ----------------------- 124 // // 125 // ========================================================================= // 126 127 void BM_no_arg_name(benchmark::State& state) { 128 while (state.KeepRunning()) { 129 } 130 } 131 BENCHMARK(BM_no_arg_name)->Arg(3); 132 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); 133 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); 134 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); 135 136 // ========================================================================= // 137 // ------------------------ Testing Arg Name Output ----------------------- // 138 // ========================================================================= // 139 140 void BM_arg_name(benchmark::State& state) { 141 while (state.KeepRunning()) { 142 } 143 } 144 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); 145 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); 146 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); 147 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); 148 149 // ========================================================================= // 150 // ------------------------ Testing Arg Names Output ----------------------- // 151 // ========================================================================= // 152 153 void BM_arg_names(benchmark::State& state) { 154 while (state.KeepRunning()) { 155 } 156 } 157 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); 158 ADD_CASES(TC_ConsoleOut, 159 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); 160 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); 161 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); 162 163 // ========================================================================= // 164 // ----------------------- Testing Complexity Output ----------------------- // 165 // ========================================================================= // 166 167 void BM_Complexity_O1(benchmark::State& state) { 168 while (state.KeepRunning()) { 169 } 170 state.SetComplexityN(state.range(0)); 171 } 172 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 173 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, 174 {"%RMS", "[ ]*[0-9]+ %"}}); 175 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, 176 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); 177 178 // ========================================================================= // 179 // ----------------------- Testing Aggregate Output ------------------------ // 180 // ========================================================================= // 181 182 // Test that non-aggregate data is printed by default 183 void BM_Repeat(benchmark::State& state) { 184 while (state.KeepRunning()) { 185 } 186 } 187 BENCHMARK(BM_Repeat)->Repetitions(3); 188 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, 189 {"^BM_Repeat/repeats:3 %console_report$"}, 190 {"^BM_Repeat/repeats:3 %console_report$"}, 191 {"^BM_Repeat/repeats:3_mean %console_report$"}, 192 {"^BM_Repeat/repeats:3_stddev %console_report$"}}); 193 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, 194 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 195 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 196 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, 197 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); 198 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, 199 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 200 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 201 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, 202 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); 203 204 // Test that a non-repeated test still prints non-aggregate results even when 205 // only-aggregate reports have been requested 206 void BM_RepeatOnce(benchmark::State& state) { 207 while (state.KeepRunning()) { 208 } 209 } 210 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); 211 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); 212 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); 213 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); 214 215 // Test that non-aggregate data is not reported 216 void BM_SummaryRepeat(benchmark::State& state) { 217 while (state.KeepRunning()) { 218 } 219 } 220 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); 221 ADD_CASES(TC_ConsoleOut, 222 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 223 {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, 224 {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); 225 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 226 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, 227 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); 228 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 229 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, 230 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); 231 232 void BM_RepeatTimeUnit(benchmark::State& state) { 233 while (state.KeepRunning()) { 234 } 235 } 236 BENCHMARK(BM_RepeatTimeUnit) 237 ->Repetitions(3) 238 ->ReportAggregatesOnly() 239 ->Unit(benchmark::kMicrosecond); 240 ADD_CASES(TC_ConsoleOut, 241 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 242 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, 243 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); 244 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 245 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, 246 {"\"time_unit\": \"us\",?$"}, 247 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, 248 {"\"time_unit\": \"us\",?$"}}); 249 ADD_CASES(TC_CSVOut, 250 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 251 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, 252 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); 253 254 // ========================================================================= // 255 // --------------------------- TEST CASES END ------------------------------ // 256 // ========================================================================= // 257 258 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 259
