1 2 #undef NDEBUG 3 4 #include "benchmark/benchmark.h" 5 #include "output_test.h" 6 7 // @todo: <jpmag> this checks the full output at once; the rule for 8 // CounterSet1 was failing because it was not matching "^[-]+$". 9 // @todo: <jpmag> check that the counters are vertically aligned. 10 ADD_CASES( 11 TC_ConsoleOut, 12 { 13 // keeping these lines long improves readability, so: 14 // clang-format off 15 {"^[-]+$", MR_Next}, 16 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, 17 {"^[-]+$", MR_Next}, 18 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 19 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 20 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 21 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 22 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 23 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 24 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 25 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 26 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 27 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 28 {"^[-]+$", MR_Next}, 29 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next}, 30 {"^[-]+$", MR_Next}, 31 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 32 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 33 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 34 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 35 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 36 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 37 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 38 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 39 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 40 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 41 {"^[-]+$", MR_Next}, 42 {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next}, 43 {"^[-]+$", MR_Next}, 44 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 45 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 46 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 47 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 48 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}, 49 // clang-format on 50 }); 51 ADD_CASES(TC_CSVOut, {{"%csv_header," 52 "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); 53 54 // ========================================================================= // 55 // ------------------------- Tabular Counters Output ----------------------- // 56 // ========================================================================= // 57 58 void BM_Counters_Tabular(benchmark::State& state) { 59 for (auto _ : state) { 60 } 61 namespace bm = benchmark; 62 state.counters.insert({ 63 {"Foo", {1, bm::Counter::kAvgThreads}}, 64 {"Bar", {2, bm::Counter::kAvgThreads}}, 65 {"Baz", {4, bm::Counter::kAvgThreads}}, 66 {"Bat", {8, bm::Counter::kAvgThreads}}, 67 {"Frob", {16, bm::Counter::kAvgThreads}}, 68 {"Lob", {32, bm::Counter::kAvgThreads}}, 69 }); 70 } 71 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); 72 ADD_CASES(TC_JSONOut, 73 {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, 74 {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, 75 {"\"run_type\": \"iteration\",$", MR_Next}, 76 {"\"iterations\": %int,$", MR_Next}, 77 {"\"real_time\": %float,$", MR_Next}, 78 {"\"cpu_time\": %float,$", MR_Next}, 79 {"\"time_unit\": \"ns\",$", MR_Next}, 80 {"\"Bar\": %float,$", MR_Next}, 81 {"\"Bat\": %float,$", MR_Next}, 82 {"\"Baz\": %float,$", MR_Next}, 83 {"\"Foo\": %float,$", MR_Next}, 84 {"\"Frob\": %float,$", MR_Next}, 85 {"\"Lob\": %float$", MR_Next}, 86 {"}", MR_Next}}); 87 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," 88 "%float,%float,%float,%float,%float,%float$"}}); 89 // VS2013 does not allow this function to be passed as a lambda argument 90 // to CHECK_BENCHMARK_RESULTS() 91 void CheckTabular(Results const& e) { 92 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1); 93 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2); 94 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4); 95 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8); 96 CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); 97 CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32); 98 } 99 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); 100 101 // ========================================================================= // 102 // -------------------- Tabular+Rate Counters Output ----------------------- // 103 // ========================================================================= // 104 105 void BM_CounterRates_Tabular(benchmark::State& state) { 106 for (auto _ : state) { 107 } 108 namespace bm = benchmark; 109 state.counters.insert({ 110 {"Foo", {1, bm::Counter::kAvgThreadsRate}}, 111 {"Bar", {2, bm::Counter::kAvgThreadsRate}}, 112 {"Baz", {4, bm::Counter::kAvgThreadsRate}}, 113 {"Bat", {8, bm::Counter::kAvgThreadsRate}}, 114 {"Frob", {16, bm::Counter::kAvgThreadsRate}}, 115 {"Lob", {32, bm::Counter::kAvgThreadsRate}}, 116 }); 117 } 118 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); 119 ADD_CASES(TC_JSONOut, 120 {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, 121 {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", 122 MR_Next}, 123 {"\"run_type\": \"iteration\",$", MR_Next}, 124 {"\"iterations\": %int,$", MR_Next}, 125 {"\"real_time\": %float,$", MR_Next}, 126 {"\"cpu_time\": %float,$", MR_Next}, 127 {"\"time_unit\": \"ns\",$", MR_Next}, 128 {"\"Bar\": %float,$", MR_Next}, 129 {"\"Bat\": %float,$", MR_Next}, 130 {"\"Baz\": %float,$", MR_Next}, 131 {"\"Foo\": %float,$", MR_Next}, 132 {"\"Frob\": %float,$", MR_Next}, 133 {"\"Lob\": %float$", MR_Next}, 134 {"}", MR_Next}}); 135 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," 136 "%float,%float,%float,%float,%float,%float$"}}); 137 // VS2013 does not allow this function to be passed as a lambda argument 138 // to CHECK_BENCHMARK_RESULTS() 139 void CheckTabularRate(Results const& e) { 140 double t = e.DurationCPUTime(); 141 CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1. / t, 0.001); 142 CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2. / t, 0.001); 143 CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4. / t, 0.001); 144 CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8. / t, 0.001); 145 CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16. / t, 0.001); 146 CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32. / t, 0.001); 147 } 148 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int", 149 &CheckTabularRate); 150 151 // ========================================================================= // 152 // ------------------------- Tabular Counters Output ----------------------- // 153 // ========================================================================= // 154 155 // set only some of the counters 156 void BM_CounterSet0_Tabular(benchmark::State& state) { 157 for (auto _ : state) { 158 } 159 namespace bm = benchmark; 160 state.counters.insert({ 161 {"Foo", {10, bm::Counter::kAvgThreads}}, 162 {"Bar", {20, bm::Counter::kAvgThreads}}, 163 {"Baz", {40, bm::Counter::kAvgThreads}}, 164 }); 165 } 166 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); 167 ADD_CASES(TC_JSONOut, 168 {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, 169 {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, 170 {"\"run_type\": \"iteration\",$", MR_Next}, 171 {"\"iterations\": %int,$", MR_Next}, 172 {"\"real_time\": %float,$", MR_Next}, 173 {"\"cpu_time\": %float,$", MR_Next}, 174 {"\"time_unit\": \"ns\",$", MR_Next}, 175 {"\"Bar\": %float,$", MR_Next}, 176 {"\"Baz\": %float,$", MR_Next}, 177 {"\"Foo\": %float$", MR_Next}, 178 {"}", MR_Next}}); 179 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," 180 "%float,,%float,%float,,"}}); 181 // VS2013 does not allow this function to be passed as a lambda argument 182 // to CHECK_BENCHMARK_RESULTS() 183 void CheckSet0(Results const& e) { 184 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); 185 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20); 186 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); 187 } 188 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0); 189 190 // again. 191 void BM_CounterSet1_Tabular(benchmark::State& state) { 192 for (auto _ : state) { 193 } 194 namespace bm = benchmark; 195 state.counters.insert({ 196 {"Foo", {15, bm::Counter::kAvgThreads}}, 197 {"Bar", {25, bm::Counter::kAvgThreads}}, 198 {"Baz", {45, bm::Counter::kAvgThreads}}, 199 }); 200 } 201 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); 202 ADD_CASES(TC_JSONOut, 203 {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, 204 {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, 205 {"\"run_type\": \"iteration\",$", MR_Next}, 206 {"\"iterations\": %int,$", MR_Next}, 207 {"\"real_time\": %float,$", MR_Next}, 208 {"\"cpu_time\": %float,$", MR_Next}, 209 {"\"time_unit\": \"ns\",$", MR_Next}, 210 {"\"Bar\": %float,$", MR_Next}, 211 {"\"Baz\": %float,$", MR_Next}, 212 {"\"Foo\": %float$", MR_Next}, 213 {"}", MR_Next}}); 214 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," 215 "%float,,%float,%float,,"}}); 216 // VS2013 does not allow this function to be passed as a lambda argument 217 // to CHECK_BENCHMARK_RESULTS() 218 void CheckSet1(Results const& e) { 219 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15); 220 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25); 221 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45); 222 } 223 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1); 224 225 // ========================================================================= // 226 // ------------------------- Tabular Counters Output ----------------------- // 227 // ========================================================================= // 228 229 // set only some of the counters, different set now. 230 void BM_CounterSet2_Tabular(benchmark::State& state) { 231 for (auto _ : state) { 232 } 233 namespace bm = benchmark; 234 state.counters.insert({ 235 {"Foo", {10, bm::Counter::kAvgThreads}}, 236 {"Bat", {30, bm::Counter::kAvgThreads}}, 237 {"Baz", {40, bm::Counter::kAvgThreads}}, 238 }); 239 } 240 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); 241 ADD_CASES(TC_JSONOut, 242 {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, 243 {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, 244 {"\"run_type\": \"iteration\",$", MR_Next}, 245 {"\"iterations\": %int,$", MR_Next}, 246 {"\"real_time\": %float,$", MR_Next}, 247 {"\"cpu_time\": %float,$", MR_Next}, 248 {"\"time_unit\": \"ns\",$", MR_Next}, 249 {"\"Bat\": %float,$", MR_Next}, 250 {"\"Baz\": %float,$", MR_Next}, 251 {"\"Foo\": %float$", MR_Next}, 252 {"}", MR_Next}}); 253 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," 254 ",%float,%float,%float,,"}}); 255 // VS2013 does not allow this function to be passed as a lambda argument 256 // to CHECK_BENCHMARK_RESULTS() 257 void CheckSet2(Results const& e) { 258 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); 259 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30); 260 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); 261 } 262 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2); 263 264 // ========================================================================= // 265 // --------------------------- TEST CASES END ------------------------------ // 266 // ========================================================================= // 267 268 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 269
