Home | History | Annotate | Download | only in test 
      1 
      2 #undef NDEBUG
      3 #include <cassert>
      4 #include <vector>
      5 
      6 #include "../src/check.h"  // NOTE: check.h is for internal use only!
      7 #include "benchmark/benchmark.h"
      8 
      9 namespace {
     10 
     11 class TestReporter : public benchmark::ConsoleReporter {
     12  public:
     13   virtual bool ReportContext(const Context& context) {
     14     return ConsoleReporter::ReportContext(context);
     15   };
     16 
     17   virtual void ReportRuns(const std::vector<Run>& report) {
     18     all_runs_.insert(all_runs_.end(), begin(report), end(report));
     19     ConsoleReporter::ReportRuns(report);
     20   }
     21 
     22   TestReporter() {}
     23   virtual ~TestReporter() {}
     24 
     25   mutable std::vector<Run> all_runs_;
     26 };
     27 
     28 struct TestCase {
     29   std::string name;
     30   bool error_occurred;
     31   std::string error_message;
     32 
     33   typedef benchmark::BenchmarkReporter::Run Run;
     34 
     35   void CheckRun(Run const& run) const {
     36     CHECK(name == run.benchmark_name) << "expected " << name << " got "
     37                                       << run.benchmark_name;
     38     CHECK(error_occurred == run.error_occurred);
     39     CHECK(error_message == run.error_message);
     40     if (error_occurred) {
     41       // CHECK(run.iterations == 0);
     42     } else {
     43       CHECK(run.iterations != 0);
     44     }
     45   }
     46 };
     47 
     48 std::vector<TestCase> ExpectedResults;
     49 
     50 int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
     51   for (auto TC : v) {
     52     TC.name = base_name + TC.name;
     53     ExpectedResults.push_back(std::move(TC));
     54   }
     55   return 0;
     56 }
     57 
     58 #define CONCAT(x, y) CONCAT2(x, y)
     59 #define CONCAT2(x, y) x##y
     60 #define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
     61 
     62 }  // end namespace
     63 
     64 void BM_error_before_running(benchmark::State& state) {
     65   state.SkipWithError("error message");
     66   while (state.KeepRunning()) {
     67     assert(false);
     68   }
     69 }
     70 BENCHMARK(BM_error_before_running);
     71 ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
     72 
     73 void BM_error_during_running(benchmark::State& state) {
     74   int first_iter = true;
     75   while (state.KeepRunning()) {
     76     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
     77       assert(first_iter);
     78       first_iter = false;
     79       state.SkipWithError("error message");
     80     } else {
     81       state.PauseTiming();
     82       state.ResumeTiming();
     83     }
     84   }
     85 }
     86 BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
     87 ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
     88                                       {"/1/threads:2", true, "error message"},
     89                                       {"/1/threads:4", true, "error message"},
     90                                       {"/1/threads:8", true, "error message"},
     91                                       {"/2/threads:1", false, ""},
     92                                       {"/2/threads:2", false, ""},
     93                                       {"/2/threads:4", false, ""},
     94                                       {"/2/threads:8", false, ""}});
     95 
     96 void BM_error_after_running(benchmark::State& state) {
     97   while (state.KeepRunning()) {
     98     benchmark::DoNotOptimize(state.iterations());
     99   }
    100   if (state.thread_index <= (state.threads / 2))
    101     state.SkipWithError("error message");
    102 }
    103 BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
    104 ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
    105                                      {"/threads:2", true, "error message"},
    106                                      {"/threads:4", true, "error message"},
    107                                      {"/threads:8", true, "error message"}});
    108 
    109 void BM_error_while_paused(benchmark::State& state) {
    110   bool first_iter = true;
    111   while (state.KeepRunning()) {
    112     if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
    113       assert(first_iter);
    114       first_iter = false;
    115       state.PauseTiming();
    116       state.SkipWithError("error message");
    117     } else {
    118       state.PauseTiming();
    119       state.ResumeTiming();
    120     }
    121   }
    122 }
    123 BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
    124 ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
    125                                     {"/1/threads:2", true, "error message"},
    126                                     {"/1/threads:4", true, "error message"},
    127                                     {"/1/threads:8", true, "error message"},
    128                                     {"/2/threads:1", false, ""},
    129                                     {"/2/threads:2", false, ""},
    130                                     {"/2/threads:4", false, ""},
    131                                     {"/2/threads:8", false, ""}});
    132 
    133 int main(int argc, char* argv[]) {
    134   benchmark::Initialize(&argc, argv);
    135 
    136   TestReporter test_reporter;
    137   benchmark::RunSpecifiedBenchmarks(&test_reporter);
    138 
    139   typedef benchmark::BenchmarkReporter::Run Run;
    140   auto EB = ExpectedResults.begin();
    141 
    142   for (Run const& run : test_reporter.all_runs_) {
    143     assert(EB != ExpectedResults.end());
    144     EB->CheckRun(run);
    145     ++EB;
    146   }
    147   assert(EB == ExpectedResults.end());
    148 
    149   return 0;
    150 }
    151