1 2 #undef NDEBUG 3 4 #include "benchmark/benchmark.h" 5 #include "output_test.h" 6 7 // @todo: <jpmag> this checks the full output at once; the rule for 8 // CounterSet1 was failing because it was not matching "^[-]+$". 9 // @todo: <jpmag> check that the counters are vertically aligned. 10 ADD_CASES(TC_ConsoleOut, { 11 // keeping these lines long improves readability, so: 12 // clang-format off 13 {"^[-]+$", MR_Next}, 14 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, 15 {"^[-]+$", MR_Next}, 16 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 17 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 18 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 19 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 20 {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 21 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 22 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 23 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 24 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 25 {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, 26 {"^[-]+$", MR_Next}, 27 {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next}, 28 {"^[-]+$", MR_Next}, 29 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 30 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 31 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 32 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 33 {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 34 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 35 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 36 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 37 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 38 {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 39 {"^[-]+$", MR_Next}, 40 {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next}, 41 {"^[-]+$", MR_Next}, 42 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 43 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 44 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 45 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, 46 {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}, 47 // clang-format on 48 }); 49 ADD_CASES(TC_CSVOut, {{"%csv_header," 50 "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); 51 52 // ========================================================================= // 53 // ------------------------- Tabular Counters Output ----------------------- // 54 // ========================================================================= // 55 56 void BM_Counters_Tabular(benchmark::State& state) { 57 for (auto _ : state) { 58 } 59 namespace bm = benchmark; 60 state.counters.insert({ 61 {"Foo", { 1, bm::Counter::kAvgThreads}}, 62 {"Bar", { 2, bm::Counter::kAvgThreads}}, 63 {"Baz", { 4, bm::Counter::kAvgThreads}}, 64 {"Bat", { 8, bm::Counter::kAvgThreads}}, 65 {"Frob", {16, bm::Counter::kAvgThreads}}, 66 {"Lob", {32, bm::Counter::kAvgThreads}}, 67 }); 68 } 69 BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); 70 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, 71 {"\"iterations\": %int,$", MR_Next}, 72 {"\"real_time\": %float,$", MR_Next}, 73 {"\"cpu_time\": %float,$", MR_Next}, 74 {"\"time_unit\": \"ns\",$", MR_Next}, 75 {"\"Bar\": %float,$", MR_Next}, 76 {"\"Bat\": %float,$", MR_Next}, 77 {"\"Baz\": %float,$", MR_Next}, 78 {"\"Foo\": %float,$", MR_Next}, 79 {"\"Frob\": %float,$", MR_Next}, 80 {"\"Lob\": %float$", MR_Next}, 81 {"}", MR_Next}}); 82 ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," 83 "%float,%float,%float,%float,%float,%float$"}}); 84 // VS2013 does not allow this function to be passed as a lambda argument 85 // to CHECK_BENCHMARK_RESULTS() 86 void CheckTabular(Results const& e) { 87 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1); 88 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2); 89 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4); 90 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8); 91 CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); 92 CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32); 93 } 94 CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); 95 96 // ========================================================================= // 97 // -------------------- Tabular+Rate Counters Output ----------------------- // 98 // ========================================================================= // 99 100 void BM_CounterRates_Tabular(benchmark::State& state) { 101 for (auto _ : state) { 102 } 103 namespace bm = benchmark; 104 state.counters.insert({ 105 {"Foo", { 1, bm::Counter::kAvgThreadsRate}}, 106 {"Bar", { 2, bm::Counter::kAvgThreadsRate}}, 107 {"Baz", { 4, bm::Counter::kAvgThreadsRate}}, 108 {"Bat", { 8, bm::Counter::kAvgThreadsRate}}, 109 {"Frob", {16, bm::Counter::kAvgThreadsRate}}, 110 {"Lob", {32, bm::Counter::kAvgThreadsRate}}, 111 }); 112 } 113 BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); 114 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, 115 {"\"iterations\": %int,$", MR_Next}, 116 {"\"real_time\": %float,$", MR_Next}, 117 {"\"cpu_time\": %float,$", MR_Next}, 118 {"\"time_unit\": \"ns\",$", MR_Next}, 119 {"\"Bar\": %float,$", MR_Next}, 120 {"\"Bat\": %float,$", MR_Next}, 121 {"\"Baz\": %float,$", MR_Next}, 122 {"\"Foo\": %float,$", MR_Next}, 123 {"\"Frob\": %float,$", MR_Next}, 124 {"\"Lob\": %float$", MR_Next}, 125 {"}", MR_Next}}); 126 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," 127 "%float,%float,%float,%float,%float,%float$"}}); 128 // VS2013 does not allow this function to be passed as a lambda argument 129 // to CHECK_BENCHMARK_RESULTS() 130 void CheckTabularRate(Results const& e) { 131 double t = e.DurationCPUTime(); 132 CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001); 133 CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001); 134 CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001); 135 CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001); 136 CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001); 137 CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001); 138 } 139 CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int", 140 &CheckTabularRate); 141 142 // ========================================================================= // 143 // ------------------------- Tabular Counters Output ----------------------- // 144 // ========================================================================= // 145 146 // set only some of the counters 147 void BM_CounterSet0_Tabular(benchmark::State& state) { 148 for (auto _ : state) { 149 } 150 namespace bm = benchmark; 151 state.counters.insert({ 152 {"Foo", {10, bm::Counter::kAvgThreads}}, 153 {"Bar", {20, bm::Counter::kAvgThreads}}, 154 {"Baz", {40, bm::Counter::kAvgThreads}}, 155 }); 156 } 157 BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); 158 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, 159 {"\"iterations\": %int,$", MR_Next}, 160 {"\"real_time\": %float,$", MR_Next}, 161 {"\"cpu_time\": %float,$", MR_Next}, 162 {"\"time_unit\": \"ns\",$", MR_Next}, 163 {"\"Bar\": %float,$", MR_Next}, 164 {"\"Baz\": %float,$", MR_Next}, 165 {"\"Foo\": %float$", MR_Next}, 166 {"}", MR_Next}}); 167 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," 168 "%float,,%float,%float,,"}}); 169 // VS2013 does not allow this function to be passed as a lambda argument 170 // to CHECK_BENCHMARK_RESULTS() 171 void CheckSet0(Results const& e) { 172 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); 173 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20); 174 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); 175 } 176 CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0); 177 178 // again. 179 void BM_CounterSet1_Tabular(benchmark::State& state) { 180 for (auto _ : state) { 181 } 182 namespace bm = benchmark; 183 state.counters.insert({ 184 {"Foo", {15, bm::Counter::kAvgThreads}}, 185 {"Bar", {25, bm::Counter::kAvgThreads}}, 186 {"Baz", {45, bm::Counter::kAvgThreads}}, 187 }); 188 } 189 BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); 190 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, 191 {"\"iterations\": %int,$", MR_Next}, 192 {"\"real_time\": %float,$", MR_Next}, 193 {"\"cpu_time\": %float,$", MR_Next}, 194 {"\"time_unit\": \"ns\",$", MR_Next}, 195 {"\"Bar\": %float,$", MR_Next}, 196 {"\"Baz\": %float,$", MR_Next}, 197 {"\"Foo\": %float$", MR_Next}, 198 {"}", MR_Next}}); 199 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," 200 "%float,,%float,%float,,"}}); 201 // VS2013 does not allow this function to be passed as a lambda argument 202 // to CHECK_BENCHMARK_RESULTS() 203 void CheckSet1(Results const& e) { 204 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15); 205 CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25); 206 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45); 207 } 208 CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1); 209 210 // ========================================================================= // 211 // ------------------------- Tabular Counters Output ----------------------- // 212 // ========================================================================= // 213 214 // set only some of the counters, different set now. 215 void BM_CounterSet2_Tabular(benchmark::State& state) { 216 for (auto _ : state) { 217 } 218 namespace bm = benchmark; 219 state.counters.insert({ 220 {"Foo", {10, bm::Counter::kAvgThreads}}, 221 {"Bat", {30, bm::Counter::kAvgThreads}}, 222 {"Baz", {40, bm::Counter::kAvgThreads}}, 223 }); 224 } 225 BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); 226 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, 227 {"\"iterations\": %int,$", MR_Next}, 228 {"\"real_time\": %float,$", MR_Next}, 229 {"\"cpu_time\": %float,$", MR_Next}, 230 {"\"time_unit\": \"ns\",$", MR_Next}, 231 {"\"Bat\": %float,$", MR_Next}, 232 {"\"Baz\": %float,$", MR_Next}, 233 {"\"Foo\": %float$", MR_Next}, 234 {"}", MR_Next}}); 235 ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," 236 ",%float,%float,%float,,"}}); 237 // VS2013 does not allow this function to be passed as a lambda argument 238 // to CHECK_BENCHMARK_RESULTS() 239 void CheckSet2(Results const& e) { 240 CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); 241 CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30); 242 CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); 243 } 244 CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2); 245 246 // ========================================================================= // 247 // --------------------------- TEST CASES END ------------------------------ // 248 // ========================================================================= // 249 250 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 251
