Home | History | Annotate | Download | only in aot 
      1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
      2 
      3 Licensed under the Apache License, Version 2.0 (the "License");
      4 you may not use this file except in compliance with the License.
      5 You may obtain a copy of the License at
      6 
      7     http://www.apache.org/licenses/LICENSE-2.0
      8 
      9 Unless required by applicable law or agreed to in writing, software
     10 distributed under the License is distributed on an "AS IS" BASIS,
     11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 See the License for the specific language governing permissions and
     13 limitations under the License.
     14 ==============================================================================*/
     15 
     16 // The purpose of the benchmark library is to support building an aot binary
     17 // with minimal dependencies, to demonstrate small binary sizes.
     18 //
     19 // KEEP THE DEPENDENCIES MINIMAL.
     20 
     21 #include "tensorflow/compiler/aot/benchmark.h"
     22 
     23 #include <sys/time.h>
     24 
     25 #include <algorithm>
     26 #include <functional>
     27 #include <string>
     28 #include <utility>
     29 #include <vector>
     30 
     31 #include "tensorflow/core/platform/types.h"
     32 
     33 namespace tensorflow {
     34 namespace tfcompile {
     35 namespace benchmark {
     36 
     37 // Returns current wall time in micros.
     38 //
     39 // TODO(b/33546473): Refactor tensorflow::Env::NowMicros() so that we can re-use
     40 // the implementation without pulling in all of the Env dependencies.
     41 static double NowMicros() {
     42   struct timeval tv;
     43   gettimeofday(&tv, nullptr);
     44   return static_cast<uint64>(tv.tv_sec) * 1000000 + tv.tv_usec;
     45 }
     46 
     47 void DumpStatsToStdout(const Stats& stats) {
     48   // Compute stats.
     49   std::vector<int64> sorted_us(stats.per_iter_us);
     50   std::sort(sorted_us.begin(), sorted_us.end());
     51   const size_t count_us = sorted_us.size();
     52   double sum_us = 0;
     53   size_t count_us_trimmed = 0;
     54   double sum_us_trimmed = 0;
     55   size_t count_us_best = 0;
     56   double sum_us_best = 0;
     57   static constexpr float trim_ratio = 0.25;
     58   static constexpr float best_ratio = 0.1;
     59   const size_t count_trimmed = count_us * trim_ratio;
     60   const size_t count_best = count_us * best_ratio;
     61   for (size_t i = 0; i < sorted_us.size(); ++i) {
     62     const int64 us = sorted_us[i];
     63     sum_us += us;
     64     if (i >= count_trimmed && i < count_us - count_trimmed) {
     65       sum_us_trimmed += us;
     66       ++count_us_trimmed;
     67     }
     68     if (i < count_best) {
     69       sum_us_best += us;
     70       ++count_us_best;
     71     }
     72   }
     73   // Prepare nicely-formatted data.
     74   const int kBufSize = 1000;
     75   char buf[kBufSize];
     76   snprintf(buf, kBufSize, "Mean with %2.0f%% trimmed:", trim_ratio * 100);
     77   const string label_trimmed(buf);
     78   snprintf(buf, kBufSize, "Mean of %2.0f%% best:", best_ratio * 100);
     79   const string label_best(buf);
     80   std::vector<std::pair<string, double>> groups = {
     81       {"Best:", sorted_us.front()},
     82       {"Worst:", sorted_us.back()},
     83       {"Median:", sorted_us[count_us / 2]},
     84       {"Mean:", sum_us / count_us},
     85       {label_trimmed, sum_us_trimmed / count_us_trimmed},
     86       {label_best, sum_us_best / count_us_best},
     87   };
     88   int max_label_size = 0;
     89   double max_us = 0;
     90   for (const auto& g : groups) {
     91     if (g.first.size() > max_label_size) {
     92       max_label_size = g.first.size();
     93     }
     94     if (g.second > max_us) {
     95       max_us = g.second;
     96     }
     97   }
     98   int max_digits = 1;
     99   while (max_us >= 10.0) {
    100     max_us /= 10.0;
    101     ++max_digits;
    102   }
    103   // Dump stats out.
    104   printf("Benchmark ran %zu iterations over %lld us\n", count_us,
    105          stats.total_us);
    106   for (const auto& g : groups) {
    107     printf("  %-*s %*.3f us\n", max_label_size, g.first.c_str(), max_digits + 4,
    108            g.second);
    109   }
    110 }
    111 
    112 void Benchmark(const Options& options, const BenchmarkFn& fn, Stats* stats) {
    113   // If neither max_seconds or max_iters is set, stop at kDefaultMicros.
    114   const int64 max_us = (options.max_micros <= 0 && options.max_iters <= 0)
    115                            ? Options::kDefaultMicros
    116                            : options.max_micros;
    117   printf("Running benchmark for %lld us\n", max_us);
    118   const int64 start_us = NowMicros();
    119   int64 iters = 0;
    120   while (true) {
    121     const int64 iter_start_us = NowMicros();
    122     fn();
    123     const int64 end_us = NowMicros();
    124     // Collect stats and decide whether to stop.
    125     stats->per_iter_us.push_back(end_us - iter_start_us);
    126     const int64 total_us = end_us - start_us;
    127     ++iters;
    128     if ((max_us > 0 && total_us >= max_us) ||
    129         (options.max_iters > 0 && iters >= options.max_iters)) {
    130       stats->total_us = total_us;
    131       break;
    132     }
    133   }
    134 }
    135 
    136 }  // namespace benchmark
    137 }  // namespace tfcompile
    138 }  // namespace tensorflow
    139