1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <mutex> 17 #include <unordered_map> 18 19 #include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" 20 #include "tensorflow/core/example/example.pb.h" 21 #include "tensorflow/core/example/feature.pb.h" 22 #include "tensorflow/core/framework/op.h" 23 #include "tensorflow/core/framework/tensor.h" 24 #include "tensorflow/core/framework/tensor_shape.h" 25 #include "tensorflow/core/framework/tensor_types.h" 26 #include "tensorflow/core/framework/types.pb.h" 27 #include "tensorflow/core/graph/graph.h" 28 #include "tensorflow/core/graph/node_builder.h" 29 #include "tensorflow/core/lib/core/status_test_util.h" 30 #include "tensorflow/core/lib/strings/stringprintf.h" 31 #include "tensorflow/core/platform/test.h" 32 #include "tensorflow/core/platform/test_benchmark.h" 33 #include "tensorflow/core/platform/types.h" 34 35 namespace tensorflow { 36 37 typedef std::map<std::tuple<int, int, int>, Tensor> ExampleTensorMap; 38 39 // Fillers to fill the underlying repeated array in protobuf. 40 class BytesFiller { 41 public: 42 BytesFiller() {} 43 void operator()(Feature* f, int feature_size) const { 44 for (int i = 0; i < feature_size; ++i) { 45 f->mutable_bytes_list()->add_value("abcd1234abcd1234abcd1234abcd1234!"); 46 } 47 } 48 Tensor make_dense_default(int feature_size) { 49 return Tensor(dtype, TensorShape({feature_size})); 50 } 51 DataType dtype = DT_STRING; 52 }; 53 54 class Int64Filler { 55 public: 56 Int64Filler() {} 57 void operator()(Feature* f, int feature_size) const { 58 for (int i = 0; i < feature_size; ++i) { 59 f->mutable_int64_list()->add_value(1729); 60 } 61 } 62 Tensor make_dense_default(int feature_size) { 63 return Tensor(dtype, TensorShape({feature_size})); 64 } 65 DataType dtype = DT_INT64; 66 }; 67 68 class FloatFiller { 69 public: 70 FloatFiller() {} 71 void operator()(Feature* f, int feature_size) const { 72 for (int i = 0; i < feature_size; ++i) { 73 f->mutable_float_list()->add_value(1.729); 74 } 75 } 76 Tensor make_dense_default(int feature_size) { 77 return Tensor(dtype, TensorShape({feature_size})); 78 } 79 DataType dtype = DT_FLOAT; 80 }; 81 82 template <typename T> 83 struct ExampleStore { 84 private: 85 static ExampleTensorMap serialized_example; 86 static std::once_flag flags_init; 87 88 public: 89 static ExampleTensorMap& GetSerializedExample() { 90 std::call_once(flags_init, [] { 91 AddExample(&serialized_example, 10, 1, 1); 92 AddExample(&serialized_example, 100, 1, 1); 93 AddExample(&serialized_example, 1000, 1, 1); 94 AddExample(&serialized_example, 10, 128, 1); 95 AddExample(&serialized_example, 100, 128, 1); 96 AddExample(&serialized_example, 1000, 128, 1); 97 AddExample(&serialized_example, 10, 512, 1); 98 AddExample(&serialized_example, 100, 512, 1); 99 AddExample(&serialized_example, 1000, 512, 1); 100 AddExample(&serialized_example, 1, 1, 1000000); 101 }); 102 return serialized_example; 103 } 104 typedef T Filler; 105 static void AddExample(ExampleTensorMap* examples, int num_keys, 106 int batch_size, int feature_size) { 107 Example example; 108 Filler fill; 109 Tensor record_string(DT_STRING, TensorShape({batch_size})); 110 auto string_t = record_string.vec<string>(); 111 example.Clear(); 112 for (int b = 0; b < batch_size; ++b) { 113 for (int k = 0; k < num_keys; ++k) { 114 string k_str = strings::Printf("feature_%d", k); 115 Feature f; 116 fill(&f, feature_size); 117 Features* features = example.mutable_features(); 118 (*features->mutable_feature())[k_str] = f; 119 } 120 CHECK(example.SerializeToString(&string_t(b))); 121 } 122 (*examples)[std::make_tuple(batch_size, num_keys, feature_size)] = 123 record_string; 124 } 125 }; 126 template <typename T> 127 ExampleTensorMap ExampleStore<T>::serialized_example; 128 template <typename T> 129 std::once_flag ExampleStore<T>::flags_init; 130 131 template class ExampleStore<BytesFiller>; 132 template class ExampleStore<Int64Filler>; 133 template class ExampleStore<FloatFiller>; 134 135 enum BenchmarkType { kDense, kSparse, kVarLenDense }; 136 137 template <typename S, BenchmarkType b_type> 138 struct BenchmarkOptions { 139 int benchmark_type = b_type; 140 typedef S Store; 141 typename S::Filler filler; 142 }; 143 144 template <typename Options> 145 static Graph* ParseExample(int batch_size, int num_keys, int feature_size) { 146 Graph* g = new Graph(OpRegistry::Global()); 147 Tensor& serialized = Options::Store::GetSerializedExample()[std::make_tuple( 148 batch_size, num_keys, feature_size)]; 149 Tensor names(DT_STRING, TensorShape({batch_size})); 150 151 std::vector<NodeBuilder::NodeOut> sparse_keys; 152 std::vector<NodeBuilder::NodeOut> dense_keys; 153 std::vector<NodeBuilder::NodeOut> dense_defaults; 154 std::vector<DataType> sparse_types; 155 std::vector<PartialTensorShape> dense_shapes; 156 Options opt; 157 for (int i = 0; i < num_keys; ++i) { 158 Tensor key(DT_STRING, TensorShape()); 159 key.scalar<string>()() = strings::Printf("feature_%d", i); 160 switch (opt.benchmark_type) { 161 case kDense: 162 dense_keys.emplace_back(test::graph::Constant(g, key)); 163 dense_defaults.emplace_back(test::graph::Constant( 164 g, opt.filler.make_dense_default(feature_size))); 165 dense_shapes.push_back(PartialTensorShape({feature_size})); 166 break; 167 case kVarLenDense: 168 dense_keys.emplace_back(test::graph::Constant(g, key)); 169 dense_defaults.emplace_back( 170 test::graph::Constant(g, opt.filler.make_dense_default(1))); 171 dense_shapes.push_back(PartialTensorShape({-1})); 172 break; 173 case kSparse: 174 sparse_keys.emplace_back(test::graph::Constant(g, key)); 175 sparse_types.push_back(opt.filler.dtype); 176 break; 177 } 178 } 179 180 Node* ret; 181 TF_EXPECT_OK(NodeBuilder(g->NewName("n"), "ParseExample") 182 .Input(test::graph::Constant(g, serialized)) 183 .Input(test::graph::Constant(g, names)) 184 .Input(sparse_keys) 185 .Input(dense_keys) 186 .Input(dense_defaults) 187 .Attr("sparse_types", sparse_types) 188 .Attr("dense_shapes", dense_shapes) 189 .Finalize(g, &ret)); 190 191 return g; 192 } 193 194 template <typename Options> 195 static Graph* ParseSingleExample(int num_keys, int feature_size) { 196 Graph* g = new Graph(OpRegistry::Global()); 197 Tensor& serialized_batch_1 = 198 Options::Store::GetSerializedExample()[std::make_tuple(1, num_keys, 199 feature_size)]; 200 Tensor serialized(DT_STRING, TensorShape()); 201 serialized.scalar<string>()() = serialized_batch_1.vec<string>()(0); 202 203 std::vector<string> sparse_keys; 204 std::vector<string> dense_keys; 205 std::vector<NodeBuilder::NodeOut> dense_defaults; 206 std::vector<DataType> sparse_types; 207 std::vector<PartialTensorShape> dense_shapes; 208 Options opt; 209 for (int i = 0; i < num_keys; ++i) { 210 string key = strings::Printf("feature_%d", i); 211 switch (opt.benchmark_type) { 212 case kDense: 213 dense_keys.push_back(key), 214 dense_defaults.emplace_back(test::graph::Constant( 215 g, opt.filler.make_dense_default(feature_size))); 216 dense_shapes.push_back(PartialTensorShape({feature_size})); 217 break; 218 case kVarLenDense: 219 dense_keys.push_back(key), 220 dense_defaults.emplace_back( 221 test::graph::Constant(g, opt.filler.make_dense_default(1))); 222 dense_shapes.push_back(PartialTensorShape({-1})); 223 break; 224 case kSparse: 225 sparse_keys.push_back(key), sparse_types.push_back(opt.filler.dtype); 226 break; 227 } 228 } 229 230 Node* ret; 231 TF_EXPECT_OK(NodeBuilder(g->NewName("n"), "ParseSingleExample") 232 .Input(test::graph::Constant(g, serialized)) 233 .Input(dense_defaults) 234 .Attr<int64>("num_sparse", sparse_keys.size()) 235 .Attr("sparse_keys", sparse_keys) 236 .Attr("sparse_types", sparse_types) 237 .Attr("dense_keys", dense_keys) 238 .Attr("dense_shapes", dense_shapes) 239 .Finalize(g, &ret)); 240 241 return g; 242 } 243 244 // Benchmark settings (Sparse, Dense) X (Bytes, Int64, Float) 245 typedef BenchmarkOptions<ExampleStore<BytesFiller>, kSparse> SparseString; 246 typedef BenchmarkOptions<ExampleStore<BytesFiller>, kDense> DenseString; 247 typedef BenchmarkOptions<ExampleStore<BytesFiller>, kVarLenDense> 248 VarLenDenseString; 249 typedef BenchmarkOptions<ExampleStore<Int64Filler>, kSparse> SparseInt64; 250 typedef BenchmarkOptions<ExampleStore<Int64Filler>, kDense> DenseInt64; 251 typedef BenchmarkOptions<ExampleStore<Int64Filler>, kVarLenDense> 252 VarLenDenseInt64; 253 typedef BenchmarkOptions<ExampleStore<FloatFiller>, kSparse> SparseFloat; 254 typedef BenchmarkOptions<ExampleStore<FloatFiller>, kDense> DenseFloat; 255 typedef BenchmarkOptions<ExampleStore<FloatFiller>, kVarLenDense> 256 VarLenDenseFloat; 257 258 // B == batch_size, K == num_keys. F == feature_size. 259 // K must be one of 10, 100, 1000 260 #define BM_ParseExample(TYPE, B, K, F) \ 261 static void BM_ParseExample##_##TYPE##_##B##_##K##_##F(int iters) { \ 262 int64 items_per_iter = static_cast<int64>(B) * K * F; \ 263 testing::UseRealTime(); \ 264 testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \ 265 test::Benchmark("cpu", ParseExample<TYPE>(B, K, F)).Run(iters); \ 266 } \ 267 BENCHMARK(BM_ParseExample##_##TYPE##_##B##_##K##_##F); 268 269 #define BM_AllParseExample(Type) \ 270 BM_ParseExample(Type, 1, 10, 1); \ 271 BM_ParseExample(Type, 128, 10, 1); \ 272 BM_ParseExample(Type, 512, 10, 1); \ 273 BM_ParseExample(Type, 1, 100, 1); \ 274 BM_ParseExample(Type, 128, 100, 1); \ 275 BM_ParseExample(Type, 512, 100, 1); \ 276 BM_ParseExample(Type, 1, 1000, 1); \ 277 BM_ParseExample(Type, 128, 1000, 1); \ 278 BM_ParseExample(Type, 512, 1000, 1); \ 279 BM_ParseExample(Type, 1, 1, 1000000); 280 281 BM_AllParseExample(SparseString); 282 BM_AllParseExample(DenseString); 283 BM_AllParseExample(VarLenDenseString); 284 BM_AllParseExample(SparseInt64); 285 BM_AllParseExample(DenseInt64); 286 BM_AllParseExample(VarLenDenseInt64); 287 BM_AllParseExample(SparseFloat); 288 BM_AllParseExample(DenseFloat); 289 BM_AllParseExample(VarLenDenseFloat); 290 291 // K == num_keys. F == feature_size. 292 // K must be one of 10, 100, 1000 293 #define BM_ParseSingleExample(TYPE, K, F) \ 294 static void BM_ParseSingleExample##_##TYPE##_1_##K##_##F(int iters) { \ 295 int64 items_per_iter = K * F; \ 296 testing::UseRealTime(); \ 297 testing::ItemsProcessed(static_cast<int64>(iters) * items_per_iter); \ 298 test::Benchmark("cpu", ParseSingleExample<TYPE>(K, F)).Run(iters); \ 299 } \ 300 BENCHMARK(BM_ParseSingleExample##_##TYPE##_1_##K##_##F); 301 302 #define BM_AllParseSingleExample(Type) \ 303 BM_ParseSingleExample(Type, 10, 1); \ 304 BM_ParseSingleExample(Type, 100, 1); \ 305 BM_ParseSingleExample(Type, 1000, 1); \ 306 BM_ParseSingleExample(Type, 1, 1000000); 307 308 BM_AllParseSingleExample(SparseString); 309 BM_AllParseSingleExample(DenseString); 310 BM_AllParseSingleExample(VarLenDenseString); 311 BM_AllParseSingleExample(SparseInt64); 312 BM_AllParseSingleExample(DenseInt64); 313 BM_AllParseSingleExample(VarLenDenseInt64); 314 BM_AllParseSingleExample(SparseFloat); 315 BM_AllParseSingleExample(DenseFloat); 316 BM_AllParseSingleExample(VarLenDenseFloat); 317 318 } // end namespace tensorflow 319
