1 2 #undef NDEBUG 3 #include <utility> 4 5 #include "benchmark/benchmark.h" 6 #include "output_test.h" 7 8 // ========================================================================= // 9 // ---------------------- Testing Prologue Output -------------------------- // 10 // ========================================================================= // 11 12 ADD_CASES(TC_ConsoleOut, {{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, 13 {"^[-]+$", MR_Next}}); 14 ADD_CASES(TC_CSVOut, 15 {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second," 16 "items_per_second,label,error_occurred,error_message"}}); 17 18 // ========================================================================= // 19 // ------------------------ Testing Basic Output --------------------------- // 20 // ========================================================================= // 21 22 void BM_basic(benchmark::State& state) { 23 while (state.KeepRunning()) { 24 } 25 } 26 BENCHMARK(BM_basic); 27 28 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); 29 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, 30 {"\"iterations\": %int,$", MR_Next}, 31 {"\"real_time\": %int,$", MR_Next}, 32 {"\"cpu_time\": %int,$", MR_Next}, 33 {"\"time_unit\": \"ns\"$", MR_Next}, 34 {"}", MR_Next}}); 35 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); 36 37 // ========================================================================= // 38 // ------------------------ Testing Bytes per Second Output ---------------- // 39 // ========================================================================= // 40 41 void BM_bytes_per_second(benchmark::State& state) { 42 while (state.KeepRunning()) { 43 } 44 state.SetBytesProcessed(1); 45 } 46 BENCHMARK(BM_bytes_per_second); 47 48 ADD_CASES(TC_ConsoleOut, 49 {{"^BM_bytes_per_second %console_report +%floatB/s$"}}); 50 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, 51 {"\"iterations\": %int,$", MR_Next}, 52 {"\"real_time\": %int,$", MR_Next}, 53 {"\"cpu_time\": %int,$", MR_Next}, 54 {"\"time_unit\": \"ns\",$", MR_Next}, 55 {"\"bytes_per_second\": %int$", MR_Next}, 56 {"}", MR_Next}}); 57 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); 58 59 // ========================================================================= // 60 // ------------------------ Testing Items per Second Output ---------------- // 61 // ========================================================================= // 62 63 void BM_items_per_second(benchmark::State& state) { 64 while (state.KeepRunning()) { 65 } 66 state.SetItemsProcessed(1); 67 } 68 BENCHMARK(BM_items_per_second); 69 70 ADD_CASES(TC_ConsoleOut, 71 {{"^BM_items_per_second %console_report +%float items/s$"}}); 72 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, 73 {"\"iterations\": %int,$", MR_Next}, 74 {"\"real_time\": %int,$", MR_Next}, 75 {"\"cpu_time\": %int,$", MR_Next}, 76 {"\"time_unit\": \"ns\",$", MR_Next}, 77 {"\"items_per_second\": %int$", MR_Next}, 78 {"}", MR_Next}}); 79 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); 80 81 // ========================================================================= // 82 // ------------------------ Testing Label Output --------------------------- // 83 // ========================================================================= // 84 85 void BM_label(benchmark::State& state) { 86 while (state.KeepRunning()) { 87 } 88 state.SetLabel("some label"); 89 } 90 BENCHMARK(BM_label); 91 92 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); 93 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, 94 {"\"iterations\": %int,$", MR_Next}, 95 {"\"real_time\": %int,$", MR_Next}, 96 {"\"cpu_time\": %int,$", MR_Next}, 97 {"\"time_unit\": \"ns\",$", MR_Next}, 98 {"\"label\": \"some label\"$", MR_Next}, 99 {"}", MR_Next}}); 100 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " 101 "label\"%csv_label_report_end$"}}); 102 103 // ========================================================================= // 104 // ------------------------ Testing Error Output --------------------------- // 105 // ========================================================================= // 106 107 void BM_error(benchmark::State& state) { 108 state.SkipWithError("message"); 109 while (state.KeepRunning()) { 110 } 111 } 112 BENCHMARK(BM_error); 113 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); 114 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, 115 {"\"error_occurred\": true,$", MR_Next}, 116 {"\"error_message\": \"message\",$", MR_Next}}); 117 118 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); 119 120 // ========================================================================= // 121 // ------------------------ Testing No Arg Name Output ----------------------- 122 // // 123 // ========================================================================= // 124 125 void BM_no_arg_name(benchmark::State& state) { 126 while (state.KeepRunning()) { 127 } 128 } 129 BENCHMARK(BM_no_arg_name)->Arg(3); 130 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); 131 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); 132 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); 133 134 // ========================================================================= // 135 // ------------------------ Testing Arg Name Output ----------------------- // 136 // ========================================================================= // 137 138 void BM_arg_name(benchmark::State& state) { 139 while (state.KeepRunning()) { 140 } 141 } 142 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); 143 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); 144 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); 145 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); 146 147 // ========================================================================= // 148 // ------------------------ Testing Arg Names Output ----------------------- // 149 // ========================================================================= // 150 151 void BM_arg_names(benchmark::State& state) { 152 while (state.KeepRunning()) { 153 } 154 } 155 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); 156 ADD_CASES(TC_ConsoleOut, 157 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); 158 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); 159 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); 160 161 // ========================================================================= // 162 // ----------------------- Testing Complexity Output ----------------------- // 163 // ========================================================================= // 164 165 void BM_Complexity_O1(benchmark::State& state) { 166 while (state.KeepRunning()) { 167 } 168 state.SetComplexityN(state.range(0)); 169 } 170 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 171 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, 172 {"%RMS", "[ ]*[0-9]+ %"}}); 173 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, 174 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); 175 176 // ========================================================================= // 177 // ----------------------- Testing Aggregate Output ------------------------ // 178 // ========================================================================= // 179 180 // Test that non-aggregate data is printed by default 181 void BM_Repeat(benchmark::State& state) { 182 while (state.KeepRunning()) { 183 } 184 } 185 BENCHMARK(BM_Repeat)->Repetitions(3); 186 ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, 187 {"^BM_Repeat/repeats:3 %console_report$"}, 188 {"^BM_Repeat/repeats:3 %console_report$"}, 189 {"^BM_Repeat/repeats:3_mean %console_report$"}, 190 {"^BM_Repeat/repeats:3_stddev %console_report$"}}); 191 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, 192 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 193 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 194 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, 195 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); 196 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, 197 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 198 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 199 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, 200 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); 201 202 // Test that a non-repeated test still prints non-aggregate results even when 203 // only-aggregate reports have been requested 204 void BM_RepeatOnce(benchmark::State& state) { 205 while (state.KeepRunning()) { 206 } 207 } 208 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); 209 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); 210 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); 211 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); 212 213 // Test that non-aggregate data is not reported 214 void BM_SummaryRepeat(benchmark::State& state) { 215 while (state.KeepRunning()) { 216 } 217 } 218 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); 219 ADD_CASES(TC_ConsoleOut, 220 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 221 {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, 222 {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); 223 ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 224 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, 225 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); 226 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 227 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, 228 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); 229 230 void BM_RepeatTimeUnit(benchmark::State& state) { 231 while (state.KeepRunning()) { 232 } 233 } 234 BENCHMARK(BM_RepeatTimeUnit) 235 ->Repetitions(3) 236 ->ReportAggregatesOnly() 237 ->Unit(benchmark::kMicrosecond); 238 ADD_CASES(TC_ConsoleOut, 239 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 240 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, 241 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); 242 ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 243 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, 244 {"\"time_unit\": \"us\",?$"}, 245 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, 246 {"\"time_unit\": \"us\",?$"}}); 247 ADD_CASES(TC_CSVOut, 248 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 249 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, 250 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); 251 252 // ========================================================================= // 253 // --------------------------- TEST CASES END ------------------------------ // 254 // ========================================================================= // 255 256 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 257
