1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include <memory> 17 #include <vector> 18 19 #include "tensorflow/core/framework/function_testlib.h" 20 #include "tensorflow/core/framework/op_kernel.h" 21 #include "tensorflow/core/framework/tensor_testutil.h" 22 #include "tensorflow/core/lib/strings/str_util.h" 23 #include "tensorflow/core/platform/test.h" 24 #include "tensorflow/core/public/session.h" 25 26 namespace tensorflow { 27 namespace { 28 29 namespace f = test::function; 30 using FDH = FunctionDefHelper; 31 32 std::unique_ptr<Session> NewSession() { 33 SessionOptions opts; 34 (*opts.config.mutable_device_count())["CPU"] = 1; 35 return std::unique_ptr<Session>(NewSession(opts)); 36 } 37 38 class MathGradTest : public ::testing::Test { 39 protected: 40 // Unary 41 // dst is the output dtype of op_node. 42 Status Unary(const FDH::Node& op_node, const Tensor& x, const DataType dst, 43 Tensor* y) { 44 const DataType src = x.dtype(); 45 auto adef = [](const string& name, 46 const DataType type) { // E.g., x:float, dy:double 47 return strings::StrCat(name, ":", DataTypeString(type)); 48 }; 49 // Sum(op(x)), sum all output of op(x). 50 auto test = FDH::Define("Test", {adef("x", src)}, {adef("l", dst)}, {}, 51 { 52 op_node, 53 FDH::Const("zero", 0), 54 FDH::Const("one", 1), 55 {{"r"}, "Rank", {"x"}, {{"T", src}}}, 56 {{"indices"}, "Range", {"zero", "r", "one"}}, 57 {{"l"}, "Sum", {"y", "indices"}, {{"T", dst}}}, 58 }); 59 60 // TestGrad = Test'(x) 61 auto grad = FDH::Define( 62 "TestGrad", {adef("x", src)}, {adef("dx", src)}, {}, 63 { 64 FDH::Const("one", 1), 65 {{"dy"}, "Cast", {"one"}, {{"DstT", dst}, {"SrcT", DT_INT32}}}, 66 {{"grad"}, 67 "SymbolicGradient", 68 {"x", "dy"}, 69 { 70 {"f", FDH::FunctionRef("Test")}, 71 {"Tin", DataTypeSlice{src, dst}}, 72 {"Tout", DataTypeSlice{src}}, 73 }}, 74 {{"dx"}, "Identity", {"grad"}, {{"T", src}}}, 75 }); 76 // Each test case will feed in "x:0" and expects to get "dx:0". 77 auto gdef = test::function::GDef( 78 { 79 f::NDef("x", "Placeholder", {}, {{"dtype", src}}), 80 f::NDef("dx", "TestGrad", {"x"}, {}), 81 }, 82 {test, grad}); 83 84 auto sess = NewSession(); 85 TF_CHECK_OK(sess->Create(gdef)); 86 std::vector<Tensor> outputs; 87 auto s = sess->Run({{"x:0", x}}, {"dx:0"}, {}, &outputs); 88 if (s.ok()) { 89 CHECK_EQ(outputs.size(), 1); 90 *y = outputs[0]; 91 } 92 TF_CHECK_OK(sess->Close()); 93 return s; 94 } 95 96 Status Unary(const string& op, const Tensor& x, Tensor* y) { 97 const FDH::Node op_node = {{"y"}, op, {"x"}, {{"T", x.dtype()}}}; 98 return Unary(op_node, x, x.dtype(), y); 99 } 100 101 // Unary op expecting OK. 102 Tensor SymGrad(const string& op, const Tensor& x) { 103 Tensor ret; 104 TF_CHECK_OK(Unary(op, x, &ret)); 105 return ret; 106 } 107 108 Tensor SymCastGrad(const Tensor& x, const DataType dst) { 109 Tensor ret; 110 const FDH::Node op_node = { 111 {"y"}, "Cast", {"x"}, {{"SrcT", x.dtype()}, {"DstT", dst}}}; 112 TF_CHECK_OK(Unary(op_node, x, dst, &ret)); 113 return ret; 114 } 115 116 // Binary 117 void SymGrad(const string& op, const Tensor& x, const Tensor& y, Tensor* dx, 118 Tensor* dy) { 119 const DataType T = x.dtype(); 120 auto adef = [T](const string& name) { // E.g., x:float, dy:double 121 return strings::StrCat(name, ":", DataTypeString(T)); 122 }; 123 // Sum(op(x)), sum all output of op(x). 124 auto test = FDH::Define("Test", {adef("x"), adef("y")}, {adef("l")}, {}, 125 { 126 {{"z"}, op, {"x", "y"}, {{"T", T}}}, 127 FDH::Const("zero", 0), 128 FDH::Const("one", 1), 129 {{"r"}, "Rank", {"z"}, {{"T", T}}}, 130 {{"indices"}, "Range", {"zero", "r", "one"}}, 131 {{"l"}, "Sum", {"z", "indices"}, {{"T", T}}}, 132 }); 133 134 // TestGrad = Test'(x, y) 135 auto grad = FDH::Define( 136 "TestGrad", {adef("x"), adef("y")}, {adef("dx"), adef("dy")}, {}, 137 { 138 FDH::Const("one", 1), 139 {{"dz"}, "Cast", {"one"}, {{"DstT", T}, {"SrcT", DT_INT32}}}, 140 {{"grad0", "grad1"}, 141 "SymbolicGradient", 142 {"x", "y", "dz"}, 143 { 144 {"f", FDH::FunctionRef("Test")}, 145 {"Tin", DataTypeSlice{T, T, T}}, 146 {"Tout", DataTypeSlice{T, T}}, 147 }}, 148 {{"dx"}, "Identity", {"grad0"}, {{"T", T}}}, 149 {{"dy"}, "Identity", {"grad1"}, {{"T", T}}}, 150 }); 151 // Each test case will feed in "x:0" and "y:0" and expects to get "d0" and 152 // "d:0". 153 auto gdef = test::function::GDef( 154 { 155 f::NDef("x", "Placeholder", {}, {{"dtype", T}}), 156 f::NDef("y", "Placeholder", {}, {{"dtype", T}}), 157 f::NDef("d", "TestGrad", {"x", "y"}, {}), 158 }, 159 {test, grad}); 160 161 auto sess = NewSession(); 162 TF_CHECK_OK(sess->Create(gdef)); 163 std::vector<Tensor> outputs; 164 TF_CHECK_OK( 165 sess->Run({{"x:0", x}, {"y:0", y}}, {"d:0", "d:1"}, {}, &outputs)); 166 CHECK_EQ(outputs.size(), 2); 167 TF_CHECK_OK(sess->Close()); 168 *dx = outputs[0]; 169 *dy = outputs[1]; 170 } 171 172 // Reduction grad 173 void ReductionGrad(const string& op, const Tensor& x, const Tensor& idx, 174 Tensor* dx, Tensor* di) { 175 const DataType T = x.dtype(); 176 auto adef = [T](const string& name) { // E.g., x:float, dy:double 177 return strings::StrCat(name, ":", DataTypeString(T)); 178 }; 179 // Sum(op(x, idx)), sum all output of op(x, idx). 180 auto test = FDH::Define("Test", {adef("x"), "i:int32"}, {adef("l")}, {}, 181 { 182 {{"y"}, op, {"x", "i"}, {{"T", T}}}, 183 FDH::Const("zero", 0), 184 FDH::Const("one", 1), 185 {{"r"}, "Rank", {"y"}, {{"T", T}}}, 186 {{"indices"}, "Range", {"zero", "r", "one"}}, 187 {{"l"}, "Sum", {"y", "indices"}, {{"T", T}}}, 188 }); 189 190 // TestGrad = Test'(x) 191 auto grad = FDH::Define( 192 "TestGrad", {adef("x"), "i:int32"}, {adef("dx"), "di:int32"}, {}, 193 { 194 FDH::Const("one", 1), 195 {{"dy"}, "Cast", {"one"}, {{"DstT", T}, {"SrcT", DT_INT32}}}, 196 {{"grad0", "grad1"}, 197 "SymbolicGradient", 198 {"x", "i", "dy"}, 199 { 200 {"f", FDH::FunctionRef("Test")}, 201 {"Tin", DataTypeSlice{T, DT_INT32, T}}, 202 {"Tout", DataTypeSlice{T, DT_INT32}}, 203 }}, 204 {{"dx"}, "Identity", {"grad0"}, {{"T", T}}}, 205 {{"di"}, "Identity", {"grad1"}, {{"T", DT_INT32}}}, 206 }); 207 // Each test case will feed in "x:0" and expects to get "dx:0". 208 auto gdef = test::function::GDef( 209 { 210 f::NDef("x", "Placeholder", {}, {{"dtype", T}}), 211 f::NDef("i", "Placeholder", {}, {{"dtype", DT_INT32}}), 212 f::NDef("d", "TestGrad", {"x", "i"}, {}), 213 }, 214 {test, grad}); 215 216 auto sess = NewSession(); 217 TF_CHECK_OK(sess->Create(gdef)); 218 std::vector<Tensor> outputs; 219 TF_CHECK_OK( 220 sess->Run({{"x:0", x}, {"i:0", idx}}, {"d:0", "d:1"}, {}, &outputs)); 221 CHECK_EQ(outputs.size(), 2); 222 TF_CHECK_OK(sess->Close()); 223 *dx = outputs[0]; 224 *di = outputs[1]; 225 } 226 227 Tensor MatMulCommon(const string& opname, const string& attr_adj_x, 228 const string& attr_adj_y, const Tensor& x, bool ax, 229 const Tensor& y, bool ay) { 230 auto T = x.dtype(); 231 auto gdef = test::function::GDef( 232 { 233 f::NDef("x", "Placeholder", {}, {{"dtype", T}}), 234 f::NDef("y", "Placeholder", {}, {{"dtype", T}}), 235 f::NDef("z", opname, {"x", "y"}, 236 {{"T", T}, {attr_adj_x, ax}, {attr_adj_y, ay}}), 237 }, 238 {}); 239 auto sess = NewSession(); 240 TF_CHECK_OK(sess->Create(gdef)); 241 std::vector<Tensor> outputs; 242 TF_CHECK_OK(sess->Run({{"x:0", x}, {"y:0", y}}, {"z:0"}, {}, &outputs)); 243 CHECK_EQ(outputs.size(), 1); 244 TF_CHECK_OK(sess->Close()); 245 return outputs[0]; 246 } 247 248 Tensor MatMul(const Tensor& x, bool ax, const Tensor& y, bool ay) { 249 return MatMulCommon("MatMul", "transpose_a", "transpose_b", x, ax, y, ay); 250 } 251 252 Tensor BatchMatMul(const Tensor& x, bool ax, const Tensor& y, bool ay) { 253 return MatMulCommon("BatchMatMul", "adj_x", "adj_y", x, ax, y, ay); 254 } 255 256 void MatMulGradCommon(const string& opname, const string& attr_adj_x, 257 const string& attr_adj_y, const Tensor& x, bool ax, 258 const Tensor& y, bool ay, Tensor* dx, Tensor* dy) { 259 const DataType T = x.dtype(); 260 auto adef = [T](const string& name) { // E.g., x:float, dy:double 261 return strings::StrCat(name, ":", DataTypeString(T)); 262 }; 263 // Sum(op(x)), sum all output of op(x). 264 auto test = 265 FDH::Define("Test", {adef("x"), adef("y")}, {adef("l")}, {}, 266 { 267 {{"z"}, 268 opname, 269 {"x", "y"}, 270 {{"T", T}, {attr_adj_x, ax}, {attr_adj_y, ay}}}, 271 FDH::Const("zero", 0), 272 FDH::Const("one", 1), 273 {{"r"}, "Rank", {"z"}, {{"T", T}}}, 274 {{"indices"}, "Range", {"zero", "r", "one"}}, 275 {{"l"}, "Sum", {"z", "indices"}, {{"T", T}}}, 276 }); 277 278 // TestGrad = Test'(x, y) 279 auto grad = FDH::Define( 280 "TestGrad", {adef("x"), adef("y")}, {adef("dx"), adef("dy")}, {}, 281 { 282 FDH::Const("one", 1), 283 {{"dz"}, "Cast", {"one"}, {{"DstT", T}, {"SrcT", DT_INT32}}}, 284 {{"grad0", "grad1"}, 285 "SymbolicGradient", 286 {"x", "y", "dz"}, 287 { 288 {"f", FDH::FunctionRef("Test")}, 289 {"Tin", DataTypeSlice{T, T, T}}, 290 {"Tout", DataTypeSlice{T, T}}, 291 }}, 292 {{"dx"}, "Identity", {"grad0"}, {{"T", T}}}, 293 {{"dy"}, "Identity", {"grad1"}, {{"T", T}}}, 294 }); 295 // Each test case will feed in "x:0" and "y:0" and expects to get "d0" and 296 // "d:0". 297 auto gdef = test::function::GDef( 298 { 299 f::NDef("x", "Placeholder", {}, {{"dtype", T}}), 300 f::NDef("y", "Placeholder", {}, {{"dtype", T}}), 301 f::NDef("d", "TestGrad", {"x", "y"}, {}), 302 }, 303 {test, grad}); 304 305 auto sess = NewSession(); 306 TF_CHECK_OK(sess->Create(gdef)); 307 std::vector<Tensor> outputs; 308 TF_CHECK_OK( 309 sess->Run({{"x:0", x}, {"y:0", y}}, {"d:0", "d:1"}, {}, &outputs)); 310 CHECK_EQ(outputs.size(), 2); 311 TF_CHECK_OK(sess->Close()); 312 *dx = outputs[0]; 313 *dy = outputs[1]; 314 } 315 316 void MatMulGrad(const Tensor& x, bool ax, const Tensor& y, bool ay, 317 Tensor* dx, Tensor* dy) { 318 return MatMulGradCommon("MatMul", "transpose_a", "transpose_b", x, ax, y, 319 ay, dx, dy); 320 } 321 322 void BatchMatMulGrad(const Tensor& x, bool ax, const Tensor& y, bool ay, 323 Tensor* dx, Tensor* dy) { 324 return MatMulGradCommon("BatchMatMul", "adj_x", "adj_y", x, ax, y, ay, dx, 325 dy); 326 } 327 328 void SelectGrad(const Tensor& c, const Tensor& x, const Tensor& y, Tensor* dc, 329 Tensor* dx, Tensor* dy) { 330 auto T = DT_FLOAT; 331 // Sum(Select(c, x, y)) 332 auto test = 333 FDH::Define("Test", {"c:bool", "x:float", "y:float"}, {"l:float"}, {}, 334 { 335 {{"z"}, "Select", {"c", "x", "y"}, {{"T", T}}}, 336 FDH::Const("zero", 0), 337 FDH::Const("one", 1), 338 {{"r"}, "Rank", {"z"}, {{"T", T}}}, 339 {{"indices"}, "Range", {"zero", "r", "one"}}, 340 {{"l"}, "Sum", {"z", "indices"}, {{"T", T}}}, 341 }); 342 343 // TestGrad(x, y) = Test'(c, x, y) 344 auto grad = FDH::Define("TestGrad", {"c:bool", "x:float", "y:float"}, 345 {"dc:bool", "dx:float", "dy:float"}, {}, 346 {FDH::Const("dz", 1.f), 347 {{"grad0", "grad1", "grad2"}, 348 "SymbolicGradient", 349 {"c", "x", "y", "dz"}, 350 { 351 {"f", FDH::FunctionRef("Test")}, 352 {"Tin", DataTypeSlice{DT_BOOL, T, T, T}}, 353 {"Tout", DataTypeSlice{DT_BOOL, T, T}}, 354 }}, 355 {{"dc"}, "Identity", {"grad0"}, {{"T", DT_BOOL}}}, 356 {{"dx"}, "Identity", {"grad1"}, {{"T", T}}}, 357 {{"dy"}, "Identity", {"grad2"}, {{"T", T}}}}); 358 // Each test case will feed in "x:0" and expects to get "dx:0". 359 auto gdef = test::function::GDef( 360 { 361 f::NDef("c", "Placeholder", {}, {{"dtype", DT_BOOL}}), 362 f::NDef("x", "Placeholder", {}, {{"dtype", T}}), 363 f::NDef("y", "Placeholder", {}, {{"dtype", T}}), 364 f::NDef("d", "TestGrad", {"c", "x", "y"}, {}), 365 }, 366 {test, grad}); 367 368 auto sess = NewSession(); 369 TF_CHECK_OK(sess->Create(gdef)); 370 std::vector<Tensor> outputs; 371 TF_CHECK_OK(sess->Run({{"c:0", c}, {"x:0", x}, {"y:0", y}}, 372 {"d:0", "d:1", "d:2"}, {}, &outputs)); 373 CHECK_EQ(outputs.size(), 3); 374 TF_CHECK_OK(sess->Close()); 375 *dc = outputs[0]; 376 *dx = outputs[1]; 377 *dy = outputs[2]; 378 } 379 }; 380 381 void HasError(const Status& s, const string& substr) { 382 EXPECT_TRUE(str_util::StrContains(s.ToString(), substr)) 383 << s << ", expected substring " << substr; 384 } 385 386 REGISTER_OP("TestOpWithNoGrad") 387 .Input("x: T") 388 .Output("y: T") 389 .Attr("T: {float, double}") 390 .Doc(R"doc( 391 Test op with no grad registered. 392 393 x: input 394 y: output 395 )doc"); 396 397 class TestOp : public OpKernel { 398 public: 399 explicit TestOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} 400 void Compute(OpKernelContext* ctx) override { ctx->set_output(0, Tensor()); } 401 }; 402 REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_CPU), TestOp); 403 #ifdef TENSORFLOW_USE_SYCL 404 REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_SYCL), TestOp); 405 #endif // TENSORFLOW_USE_SYCL 406 407 TEST_F(MathGradTest, Error_Reporting) { 408 auto x = test::AsTensor<float>({-3.f}); 409 auto dx = test::AsTensor<float>({3.f}); 410 Tensor donotcare; 411 HasError(Unary("TestOpWithNoGrad", x, &donotcare), 412 "No gradient defined for op: TestOpWithNoGrad"); 413 } 414 415 TEST_F(MathGradTest, Abs) { 416 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 417 TensorShape({2, 3})); 418 auto g = [](float x) { return x < 0 ? -1.f : 1.f; }; 419 auto dx = test::AsTensor<float>( 420 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 421 auto ans = SymGrad("Abs", x); 422 test::ExpectClose(ans, dx); 423 } 424 425 TEST_F(MathGradTest, Neg) { 426 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 427 TensorShape({2, 3})); 428 auto g = [](float x) { return -1.f; }; 429 auto dx = test::AsTensor<float>( 430 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 431 auto ans = SymGrad("Neg", x); 432 test::ExpectClose(ans, dx); 433 } 434 435 TEST_F(MathGradTest, Reciprocal) { 436 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 437 TensorShape({2, 3})); 438 auto g = [](float x) { return -1.f / (x * x); }; 439 auto dx = test::AsTensor<float>( 440 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 441 auto ans = SymGrad("Reciprocal", x); 442 test::ExpectClose(ans, dx); 443 } 444 445 TEST_F(MathGradTest, Square) { 446 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 447 TensorShape({2, 3})); 448 auto g = [](float x) { return 2 * x; }; 449 auto dx = test::AsTensor<float>( 450 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 451 auto ans = SymGrad("Square", x); 452 test::ExpectClose(ans, dx); 453 } 454 455 TEST_F(MathGradTest, Sqrt) { 456 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 457 TensorShape({2, 3})); 458 auto g = [](float x) { return 0.5f / std::sqrt(x); }; 459 auto dx = test::AsTensor<float>( 460 {g(1.f), g(2.f), g(3.f), g(4.f), g(5.f), g(6.f)}, TensorShape({2, 3})); 461 auto ans = SymGrad("Sqrt", x); 462 test::ExpectClose(ans, dx); 463 } 464 465 TEST_F(MathGradTest, Rsqrt) { 466 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 467 TensorShape({2, 3})); 468 auto g = [](float x) { return -0.5f / (x * std::sqrt(x)); }; 469 auto dx = test::AsTensor<float>( 470 {g(1.f), g(2.f), g(3.f), g(4.f), g(5.f), g(6.f)}, TensorShape({2, 3})); 471 auto ans = SymGrad("Rsqrt", x); 472 test::ExpectClose(ans, dx); 473 } 474 475 TEST_F(MathGradTest, Exp) { 476 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 477 TensorShape({2, 3})); 478 auto g = [](float x) { return std::exp(x); }; 479 auto dx = test::AsTensor<float>( 480 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 481 auto ans = SymGrad("Exp", x); 482 test::ExpectClose(ans, dx); 483 } 484 485 TEST_F(MathGradTest, Expm1) { 486 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 487 TensorShape({2, 3})); 488 auto g = [](float x) { return std::exp(x); }; 489 auto dx = test::AsTensor<float>( 490 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 491 auto ans = SymGrad("Expm1", x); 492 test::ExpectClose(ans, dx); 493 } 494 495 TEST_F(MathGradTest, Log) { 496 auto x = test::AsTensor<float>({0.1f, 1.f, 2.f, 3.f, 4.f, 10.f}, 497 TensorShape({2, 3})); 498 auto g = [](float x) { return 1 / x; }; 499 auto dx = test::AsTensor<float>( 500 {g(.1f), g(1.f), g(2.f), g(3.f), g(4.f), g(10.f)}, TensorShape({2, 3})); 501 auto ans = SymGrad("Log", x); 502 test::ExpectClose(ans, dx); 503 } 504 505 TEST_F(MathGradTest, Log1p) { 506 auto x = test::AsTensor<float>({0.1f, 1.f, 2.f, 3.f, 4.f, 10.f}, 507 TensorShape({2, 3})); 508 auto g = [](float x) { return 1 / (1 + x); }; 509 auto dx = test::AsTensor<float>( 510 {g(.1f), g(1.f), g(2.f), g(3.f), g(4.f), g(10.f)}, TensorShape({2, 3})); 511 auto ans = SymGrad("Log1p", x); 512 test::ExpectClose(ans, dx); 513 } 514 515 TEST_F(MathGradTest, Sinh) { 516 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 517 TensorShape({2, 3})); 518 auto g = [](float x) { return std::cosh(x); }; 519 auto dx = test::AsTensor<float>( 520 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 521 auto ans = SymGrad("Sinh", x); 522 test::ExpectClose(ans, dx); 523 } 524 525 TEST_F(MathGradTest, Cosh) { 526 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 527 TensorShape({2, 3})); 528 auto g = [](float x) { return std::sinh(x); }; 529 auto dx = test::AsTensor<float>( 530 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 531 auto ans = SymGrad("Cosh", x); 532 test::ExpectClose(ans, dx); 533 } 534 535 TEST_F(MathGradTest, Tanh) { 536 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 537 TensorShape({2, 3})); 538 auto g = [](float x) { 539 auto y = std::tanh(x); 540 return 1 - y * y; 541 }; 542 auto dx = test::AsTensor<float>( 543 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 544 auto ans = SymGrad("Tanh", x); 545 test::ExpectClose(ans, dx); 546 } 547 548 TEST_F(MathGradTest, Asinh) { 549 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 550 TensorShape({2, 3})); 551 auto g = [](float x) { 552 auto y = std::asinh(x); 553 return std::cosh(y); 554 }; 555 auto dx = test::AsTensor<float>( 556 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 557 auto ans = SymGrad("Asinh", x); 558 test::ExpectClose(ans, dx); 559 } 560 561 TEST_F(MathGradTest, Acosh) { 562 auto x = test::AsTensor<float>({6.f, 5.f, 4.f, 1.f, 2.f, 3.f}, 563 TensorShape({2, 3})); 564 auto g = [](float x) { 565 auto y = std::acosh(x); 566 return std::sinh(y); 567 }; 568 auto dx = test::AsTensor<float>( 569 {g(6.f), g(5.f), g(4.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 570 auto ans = SymGrad("Acosh", x); 571 test::ExpectClose(ans, dx); 572 } 573 574 TEST_F(MathGradTest, Atanh) { 575 auto x = test::AsTensor<float>({-0.3f, -0.2f, -0.1f, 0.1f, 0.2f, 0.3f}, 576 TensorShape({2, 3})); 577 auto g = [](float x) { return 1.f / (1.f - x * x); }; 578 auto dx = test::AsTensor<float>( 579 {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, 580 TensorShape({2, 3})); 581 auto ans = SymGrad("Atanh", x); 582 test::ExpectClose(ans, dx); 583 } 584 585 TEST_F(MathGradTest, Sigmoid) { 586 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 587 TensorShape({2, 3})); 588 auto g = [](float x) { 589 auto y = 1.f / (1.f + std::exp(-x)); 590 return y * (1 - y); 591 }; 592 auto dx = test::AsTensor<float>( 593 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 594 auto ans = SymGrad("Sigmoid", x); 595 test::ExpectClose(ans, dx); 596 } 597 598 TEST_F(MathGradTest, Sign) { 599 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 600 TensorShape({2, 3})); 601 auto g = [](float x) { return 0.f; }; 602 auto dx = test::AsTensor<float>( 603 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 604 auto ans = SymGrad("Sign", x); 605 test::ExpectClose(ans, dx); 606 } 607 608 TEST_F(MathGradTest, Sin) { 609 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 610 TensorShape({2, 3})); 611 auto g = [](float x) { return std::cos(x); }; 612 auto dx = test::AsTensor<float>( 613 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 614 auto ans = SymGrad("Sin", x); 615 test::ExpectClose(ans, dx); 616 } 617 618 TEST_F(MathGradTest, Cos) { 619 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 620 TensorShape({2, 3})); 621 auto g = [](float x) { return -std::sin(x); }; 622 auto dx = test::AsTensor<float>( 623 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 624 auto ans = SymGrad("Cos", x); 625 test::ExpectClose(ans, dx); 626 } 627 628 TEST_F(MathGradTest, Cast) { 629 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 630 TensorShape({2, 3})); 631 auto g = [](float x) { return 1.f; }; 632 auto dx = test::AsTensor<float>( 633 {g(-3.f), g(-2.f), g(-1.f), g(1.f), g(2.f), g(3.f)}, TensorShape({2, 3})); 634 Tensor ans = SymCastGrad(x, DT_INT32); 635 test::ExpectClose(ans, dx); 636 } 637 638 // TODO(zhifengc) 639 // TEST_F(MathGradSComplexTest, Real) {} 640 // TEST_F(MathGradSComplexTest, Imag) {} 641 // TEST_F(MathGradSComplexTest, Angle) {} 642 // TEST_F(MathGradSComplexTest, Conj) {} 643 // TEST_F(MathGradTernary, Select) {} 644 645 TEST_F(MathGradTest, Add) { 646 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 647 TensorShape({2, 3})); 648 auto y = test::AsTensor<float>({-10.f, 10.f}, TensorShape({2, 1})); 649 auto ans_dx = test::AsTensor<float>({1.f, 1.f, 1.f, 1.f, 1.f, 1.f}, 650 TensorShape({2, 3})); 651 auto ans_dy = test::AsTensor<float>({3.f, 3.f}, TensorShape({2, 1})); 652 Tensor dx; 653 Tensor dy; 654 { 655 SymGrad("Add", x, y, &dx, &dy); 656 test::ExpectClose(ans_dx, dx); 657 test::ExpectClose(ans_dy, dy); 658 } 659 { // Swap x and y 660 SymGrad("Add", y, x, &dy, &dx); 661 test::ExpectClose(ans_dx, dx); 662 test::ExpectClose(ans_dy, dy); 663 } 664 } 665 666 TEST_F(MathGradTest, Sub) { 667 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 668 TensorShape({2, 3})); 669 auto y = test::AsTensor<float>({-10.f, 10.f}, TensorShape({2, 1})); 670 Tensor dx; 671 Tensor dy; 672 { 673 SymGrad("Sub", x, y, &dx, &dy); 674 auto ans_dx = test::AsTensor<float>({1.f, 1.f, 1.f, 1.f, 1.f, 1.f}, 675 TensorShape({2, 3})); 676 auto ans_dy = test::AsTensor<float>({-3.f, -3.f}, TensorShape({2, 1})); 677 test::ExpectClose(ans_dx, dx); 678 test::ExpectClose(ans_dy, dy); 679 } 680 { // Swap x and y 681 SymGrad("Sub", y, x, &dy, &dx); 682 auto ans_dx = test::AsTensor<float>({-1.f, -1.f, -1.f, -1.f, -1.f, -1.f}, 683 TensorShape({2, 3})); 684 auto ans_dy = test::AsTensor<float>({3.f, 3.f}, TensorShape({2, 1})); 685 test::ExpectClose(ans_dx, dx); 686 test::ExpectClose(ans_dy, dy); 687 } 688 } 689 690 TEST_F(MathGradTest, Mul) { 691 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 692 TensorShape({2, 3})); 693 auto y = test::AsTensor<float>({-10.f, 10.f}, TensorShape({2, 1})); 694 auto ans_dx = test::AsTensor<float>({-10.f, -10.f, -10.f, 10.f, 10.f, 10.f}, 695 TensorShape({2, 3})); 696 auto ans_dy = test::AsTensor<float>({-3.f + (-2.f) + (-1.f), 1.f + 2.f + 3.f}, 697 TensorShape({2, 1})); 698 Tensor dx; 699 Tensor dy; 700 { 701 SymGrad("Mul", x, y, &dx, &dy); 702 test::ExpectClose(ans_dx, dx); 703 test::ExpectClose(ans_dy, dy); 704 } 705 { // Swap x and y 706 SymGrad("Mul", y, x, &dy, &dx); 707 test::ExpectClose(ans_dx, dx); 708 test::ExpectClose(ans_dy, dy); 709 } 710 } 711 712 TEST_F(MathGradTest, Div) { 713 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 714 TensorShape({2, 3})); 715 auto y = test::AsTensor<float>({-10.f, 10.f}, TensorShape({2, 1})); 716 Tensor dx; 717 Tensor dy; 718 { 719 SymGrad("Div", x, y, &dx, &dy); 720 { 721 auto g = [](float x, float y) { return 1.f / y; }; 722 test::ExpectClose(dx, test::AsTensor<float>( 723 {g(-3.f, -10.f), g(-2.f, -10.f), g(-1.f, -10.f), 724 g(1.f, 10.f), g(2.f, 10.f), g(3.f, 10.f)}, 725 TensorShape({2, 3}))); 726 } 727 { 728 auto g = [](float x, float y) { return -x / (y * y); }; 729 test::ExpectClose(dy, 730 test::AsTensor<float>( 731 {g(-3.f, -10.f) + g(-2.f, -10.f) + g(-1.f, -10.f), 732 g(1.f, 10.f) + g(2.f, 10.f) + g(3.f, 10.f)}, 733 TensorShape({2, 1}))); 734 } 735 } 736 { // Swap x and y 737 SymGrad("Div", y, x, &dy, &dx); 738 { 739 auto g = [](float x, float y) { return 1.f / y; }; 740 test::ExpectClose(dy, 741 test::AsTensor<float>( 742 {g(-10.f, -3.f) + g(-10.f, -2.f) + g(-10.f, -1.f), 743 g(10.f, 1.f) + g(10.f, 2.f) + g(10.f, 3.f)}, 744 TensorShape({2, 1}))); 745 } 746 { 747 auto g = [](float x, float y) { return -x / (y * y); }; 748 test::ExpectClose(dx, test::AsTensor<float>( 749 {g(-10.f, -3.f), g(-10.f, -2.f), g(-10.f, -1.f), 750 g(10.f, 1.f), g(10.f, 2.f), g(10.f, 3.f)}, 751 TensorShape({2, 3}))); 752 } 753 } 754 } 755 756 TEST_F(MathGradTest, DivNoNan) { 757 auto x = test::AsTensor<float>( 758 {0.f, -3.f, -2.f, -1.f, 0.f, 1.f, 2.f, 3.f, 0.f}, TensorShape({3, 3})); 759 auto y = test::AsTensor<float>({-10.f, 0.f, 10.f}, TensorShape({3, 1})); 760 Tensor dx; 761 Tensor dy; 762 { 763 SymGrad("DivNoNan", x, y, &dx, &dy); 764 { 765 auto g = [](float x, float y) { 766 if (y == 0.f) { 767 return 0.f; 768 } else { 769 return 1.f / y; 770 } 771 }; 772 test::ExpectClose(dx, test::AsTensor<float>( 773 {g(0.f, -10.f), g(-3.f, -10.f), g(-2.f, -10.f), 774 g(-1.f, 0.f), g(0.f, 0.f), g(1.f, 0.f), 775 g(2.f, 10.f), g(3.f, 10.f), g(0.f, 10.f)}, 776 TensorShape({3, 3}))); 777 } 778 { 779 auto g = [](float x, float y) { 780 if (y == 0.f) { 781 return 0.f; 782 } else { 783 return -x / (y * y); 784 } 785 }; 786 test::ExpectClose(dy, 787 test::AsTensor<float>( 788 {g(0.f, -10.f) + g(-3.f, -10.f) + g(-2.f, -10.f), 789 g(-1.f, 0.f) + g(0.f, 0.f) + g(1.f, 0.f), 790 g(2.f, 10.f) + g(3.f, 10.f) + g(0.f, 10.f)}, 791 TensorShape({3, 1}))); 792 } 793 } 794 { // Swap x and y. 795 SymGrad("DivNoNan", y, x, &dy, &dx); 796 { 797 auto g = [](float x, float y) { 798 if (y == 0.f) { 799 return 0.f; 800 } else { 801 return 1.f / y; 802 } 803 }; 804 test::ExpectClose(dy, 805 test::AsTensor<float>( 806 {g(-10.f, 0.f) + g(-10.f, -3.f) + g(-10.f, -2.f), 807 g(0.f, -1.f) + g(0.f, 0.f) + g(0.f, 1.f), 808 g(10.f, 2.f) + g(10.f, 3.f) + g(10.f, 0.f)}, 809 TensorShape({3, 1}))); 810 } 811 { 812 auto g = [](float x, float y) { 813 if (y == 0.f) { 814 return 0.f; 815 } else { 816 return -x / (y * y); 817 } 818 }; 819 test::ExpectClose(dx, test::AsTensor<float>( 820 {g(-10.f, 0.f), g(-10.f, -3.f), g(-10.f, -2.f), 821 g(0.f, -1.f), g(0.f, 0.f), g(0.f, 1.f), 822 g(10.f, 2.f), g(10.f, 3.f), g(10.f, 0.f)}, 823 TensorShape({3, 3}))); 824 } 825 } 826 } 827 828 TEST_F(MathGradTest, Pow) { 829 auto x = test::AsTensor<float>({0.f, 1.f, 2.f, 3.f, 4.f, 5.f}, 830 TensorShape({2, 3})); 831 auto y = test::AsTensor<float>({.5f, 2.f}, TensorShape({2, 1})); 832 Tensor dx; 833 Tensor dy; 834 auto g = [](float x, float y) { return y * std::pow(x, y - 1); }; 835 auto h = [](float x, float y) { 836 return std::pow(x, y) * (x ? std::log(x) : 0); 837 }; 838 { 839 SymGrad("Pow", x, y, &dx, &dy); 840 test::ExpectClose( 841 dx, test::AsTensor<float>({g(0.f, .5f), g(1.f, .5f), g(2.f, .5f), 842 g(3.f, 2.f), g(4.f, 2.f), g(5.f, 2.f)}, 843 TensorShape({2, 3}))); 844 test::ExpectClose( 845 dy, test::AsTensor<float>({h(0.f, .5f) + h(1.f, .5f) + h(2.f, .5f), 846 h(3.f, 2.f) + h(4.f, 2.f) + h(5.f, 2.f)}, 847 TensorShape({2, 1}))); 848 } 849 { // Swap x and y 850 SymGrad("Pow", y, x, &dy, &dx); 851 test::ExpectClose( 852 dy, test::AsTensor<float>({g(.5f, 0.f) + g(.5f, 1.f) + g(.5f, 2.f), 853 g(2.f, 3.f) + g(2.f, 4.f) + g(2.f, 5.f)}, 854 TensorShape({2, 1}))); 855 test::ExpectClose( 856 dx, test::AsTensor<float>({h(.5f, 0.f), h(.5f, 1.f), h(.5f, 2.f), 857 h(2.f, 3.f), h(2.f, 4.f), h(2.f, 5.f)}, 858 TensorShape({2, 3}))); 859 } 860 } 861 862 // TODO{lukeiwanski}: Implement Complex Pow for SYCL 863 #ifndef TENSORFLOW_USE_SYCL 864 TEST_F(MathGradTest, ComplexPow) { 865 auto x = test::AsTensor<complex64>({0.f, 2.f, -2.f}, TensorShape({3})); 866 auto y = test::AsTensor<complex64>({2.f, 2.f, 2.f}, TensorShape({3})); 867 Tensor dx; 868 Tensor dy; 869 auto g = [](complex64 x, complex64 y) { return y * std::pow(x, y - 1.f); }; 870 auto h = [](complex64 x, complex64 y) { 871 return std::pow(x, y) * (x != complex64(0) ? std::log(x) : 0); 872 }; 873 SymGrad("Pow", x, y, &dx, &dy); 874 875 // This case failed on Kokoro MacOS: 876 // dx[2] = (-4,6.0398321011234657e-07), 877 // test::AsTensor[2] = (-4,-3.4969110629390343e-07). 878 // dx[2] on linux is close to test::AsTensor[2]. 879 // This error hasn't shown up before because 880 // ExpectClose used to check just the magnitude of a complex number, i.e., 881 // std::abs(complex) = sqrt(real^2 + imag^2). 882 // Now ExpectClose checks the value of each component separately. 883 // Workaround: I set a big tolerance to make the case pass for now. 884 // TODO(penporn): Fix this or file a bug. This is not a precision issue. 885 // Even the most significant digit (or the sign) doesn't match. 886 test::ExpectClose( 887 dx, 888 test::AsTensor<complex64>({g(0.f, 2.f), g(2.f, 2.f), g(-2.f, 2.f)}, 889 TensorShape({3})), 890 1e-6f); 891 892 // This case failed on Kokoro MacOS: 893 // dx[2] = (2.7725925445556641,12.56636905670166), 894 // test::AsTensor[2] = (2.7725865840911865,12.566371917724609) 895 // dx[2] on linux is close to test::AsTensor[2]. 896 // Default atol = rtol = 5.96046e-07. 897 // Real: diff = 5.96046e-06 > threshold = 2.248633e-06 <- failed 898 // Complex: diff = 2.86102e-06 <= threshold = 8.08618e-06 <- passed 899 // Again, this error hasn't shown up before because ExpectClose used to 900 // check just the magnitude of the complex number. Now it checks each 901 // component separately. 902 // Workaround: Set a larger tolerance for now. 903 // TODO(penporn): See if this is a precision issue or a bug. 904 test::ExpectClose( 905 dy, 906 test::AsTensor<complex64>({h(0.f, 2.f), h(2.f, 2.f), h(-2.f, 2.f)}, 907 TensorShape({3})), 908 4.5e-6f); 909 } 910 #endif // TENSORFLOW_USE_SYCL 911 912 TEST_F(MathGradTest, Xlogy) { 913 auto x = test::AsTensor<float>({0.f, 0.f, 2.f, 3.f, 4.f, 5.f}, 914 TensorShape({2, 3})); 915 auto y = test::AsTensor<float>({.5f, 2.f}, TensorShape({2, 1})); 916 Tensor dx; 917 Tensor dy; 918 auto g = [](float x, float y) -> float { return x == 0. ? 0. : std::log(y); }; 919 auto h = [](float x, float y) -> float { return x == 0. ? 0. : x / y; }; 920 SymGrad("Xlogy", x, y, &dx, &dy); 921 test::ExpectClose( 922 dx, test::AsTensor<float>({g(0.f, .5f), g(0.f, 0.f), g(2.f, .5f), 923 g(3.f, 2.f), g(4.f, 2.f), g(5.f, 2.f)}, 924 TensorShape({2, 3}))); 925 test::ExpectClose( 926 dy, test::AsTensor<float>({h(0.f, .5f) + h(0.f, 0.f) + h(2.f, .5f), 927 h(3.f, 2.f) + h(4.f, 2.f) + h(5.f, 2.f)}, 928 TensorShape({2, 1}))); 929 } 930 931 TEST_F(MathGradTest, Xdivy) { 932 auto x = test::AsTensor<float>({0.f, 0.f, 2.f, 3.f, 4.f, 5.f}, 933 TensorShape({2, 3})); 934 auto y = test::AsTensor<float>({.5f, 2.f}, TensorShape({2, 1})); 935 Tensor dx; 936 Tensor dy; 937 auto g = [](float x, float y) -> float { return x == 0. ? 0. : 1 / y; }; 938 auto h = [](float x, float y) -> float { 939 return x == 0. ? 0. : -x / (y * y); 940 }; 941 SymGrad("Xdivy", x, y, &dx, &dy); 942 test::ExpectClose( 943 dx, test::AsTensor<float>({g(0.f, .5f), g(0.f, 0.f), g(2.f, .5f), 944 g(3.f, 2.f), g(4.f, 2.f), g(5.f, 2.f)}, 945 TensorShape({2, 3}))); 946 test::ExpectClose( 947 dy, test::AsTensor<float>({h(0.f, .5f) + h(0.f, 0.f) + h(2.f, .5f), 948 h(3.f, 2.f) + h(4.f, 2.f) + h(5.f, 2.f)}, 949 TensorShape({2, 1}))); 950 } 951 952 TEST_F(MathGradTest, SquaredDifference) { 953 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 954 TensorShape({2, 3})); 955 auto y = test::AsTensor<float>({.5f, 2.f}, TensorShape({2, 1})); 956 Tensor dx; 957 Tensor dy; 958 auto g = [](float x, float y) -> float { return 2. * (x - y); }; 959 auto h = [](float x, float y) -> float { return 2. * (y - x); }; 960 SymGrad("SquaredDifference", x, y, &dx, &dy); 961 test::ExpectClose( 962 dx, test::AsTensor<float>({g(-3.f, .5f), g(-2.f, .5f), g(-1.f, .5f), 963 g(1.f, 2.f), g(2.f, 2.f), g(3.f, 2.f)}, 964 TensorShape({2, 3}))); 965 test::ExpectClose( 966 dy, test::AsTensor<float>({h(-3.f, .5f) + h(-2.f, .5f) + h(-1.f, .5f), 967 h(1.f, 2.f) + h(2.f, 2.f) + h(3.f, 2.f)}, 968 TensorShape({2, 1}))); 969 } 970 971 TEST_F(MathGradTest, Maximum) { 972 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 973 TensorShape({2, 3})); 974 auto y = test::AsTensor<float>({-1.5f, 1.5f}, TensorShape({2, 1})); 975 Tensor dx; 976 Tensor dy; 977 { 978 SymGrad("Maximum", x, y, &dx, &dy); 979 { 980 auto g = [](float x, float y) { return x >= y ? 1.f : 0.f; }; 981 test::ExpectClose(dx, test::AsTensor<float>( 982 {g(-3.f, -1.5f), g(-2.f, -1.5f), g(-1.f, -1.5f), 983 g(1.f, 1.5f), g(2.f, 1.5f), g(3.f, 1.5f)}, 984 TensorShape({2, 3}))); 985 } 986 { 987 auto g = [](float x, float y) { return x < y ? 1.f : 0.f; }; 988 test::ExpectClose(dy, 989 test::AsTensor<float>( 990 {g(-3.f, -1.5f) + g(-2.f, -1.5f) + g(-1.f, -1.5f), 991 g(1.f, 1.5f) + g(2.f, 1.5f) + g(3.f, 1.5f)}, 992 TensorShape({2, 1}))); 993 } 994 } 995 { // Swap x and y 996 SymGrad("Maximum", y, x, &dy, &dx); 997 { 998 auto g = [](float x, float y) { return x >= y ? 1.f : 0.f; }; 999 test::ExpectClose(dy, 1000 test::AsTensor<float>( 1001 {g(-1.5f, -3.f) + g(-1.5f, -2.f) + g(-1.5f, -1.f), 1002 g(1.5f, 1.f) + g(1.5f, 2.f) + g(1.5f, 3.f)}, 1003 TensorShape({2, 1}))); 1004 } 1005 { 1006 auto g = [](float x, float y) { return x < y ? 1.f : 0.f; }; 1007 test::ExpectClose(dx, test::AsTensor<float>( 1008 {g(-1.5f, -3.f), g(-1.5f, -2.f), g(-1.5f, -1.f), 1009 g(1.5f, 1.f), g(1.5f, 2.f), g(1.5f, 3.f)}, 1010 TensorShape({2, 3}))); 1011 } 1012 } 1013 } 1014 1015 TEST_F(MathGradTest, Minimum) { 1016 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1017 TensorShape({2, 3})); 1018 auto y = test::AsTensor<float>({-1.5f, 1.5f}, TensorShape({2, 1})); 1019 Tensor dx; 1020 Tensor dy; 1021 { 1022 SymGrad("Minimum", x, y, &dx, &dy); 1023 { 1024 auto g = [](float x, float y) { return x <= y ? 1.f : 0.f; }; 1025 test::ExpectClose(dx, test::AsTensor<float>( 1026 {g(-3.f, -1.5f), g(-2.f, -1.5f), g(-1.f, -1.5f), 1027 g(1.f, 1.5f), g(2.f, 1.5f), g(3.f, 1.5f)}, 1028 TensorShape({2, 3}))); 1029 } 1030 { 1031 auto g = [](float x, float y) { return x > y ? 1.f : 0.f; }; 1032 test::ExpectClose(dy, 1033 test::AsTensor<float>( 1034 {g(-3.f, -1.5f) + g(-2.f, -1.5f) + g(-1.f, -1.5f), 1035 g(1.f, 1.5f) + g(2.f, 1.5f) + g(3.f, 1.5f)}, 1036 TensorShape({2, 1}))); 1037 } 1038 } 1039 { // Swap x and y 1040 SymGrad("Minimum", y, x, &dy, &dx); 1041 { 1042 auto g = [](float x, float y) { return x <= y ? 1.f : 0.f; }; 1043 test::ExpectClose(dy, 1044 test::AsTensor<float>( 1045 {g(-1.5f, -3.f) + g(-1.5f, -2.f) + g(-1.5f, -1.f), 1046 g(1.5f, 1.f) + g(1.5f, 2.f) + g(1.5f, 3.f)}, 1047 TensorShape({2, 1}))); 1048 } 1049 { 1050 auto g = [](float x, float y) { return x > y ? 1.f : 0.f; }; 1051 test::ExpectClose(dx, test::AsTensor<float>( 1052 {g(-1.5f, -3.f), g(-1.5f, -2.f), g(-1.5f, -1.f), 1053 g(1.5f, 1.f), g(1.5f, 2.f), g(1.5f, 3.f)}, 1054 TensorShape({2, 3}))); 1055 } 1056 } 1057 } 1058 1059 TEST_F(MathGradTest, Select) { 1060 auto c = test::AsTensor<bool>({true, false, false, true, true, false}, 1061 TensorShape({2, 3})); 1062 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1063 TensorShape({2, 3})); 1064 auto y = test::AsTensor<float>({3.f, 2.f, 1.f, 1.f, 2.f, 3.f}, 1065 TensorShape({2, 3})); 1066 Tensor dc; 1067 Tensor dx; 1068 Tensor dy; 1069 { 1070 SelectGrad(c, x, y, &dc, &dx, &dy); 1071 test::ExpectTensorEqual<bool>( 1072 dc, test::AsTensor<bool>({false, false, false, false, false, false}, 1073 TensorShape({2, 3}))); 1074 test::ExpectTensorEqual<float>( 1075 dx, test::AsTensor<float>({1.f, 0.f, 0.f, 1.f, 1.f, 0.f}, 1076 TensorShape({2, 3}))); 1077 test::ExpectTensorEqual<float>( 1078 dy, test::AsTensor<float>({0.f, 1.f, 1.f, 0.f, 0.f, 1.f}, 1079 TensorShape({2, 3}))); 1080 } 1081 } 1082 1083 TEST_F(MathGradTest, MatMul_00) { 1084 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1085 TensorShape({2, 3})); 1086 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({3, 1})); 1087 Tensor dx; 1088 Tensor dy; 1089 MatMulGrad(x, false, y, false, &dx, &dy); 1090 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({2, 1})); 1091 test::ExpectClose(dx, MatMul(dz, false, y, true)); 1092 test::ExpectClose(dy, MatMul(x, true, dz, false)); 1093 } 1094 1095 TEST_F(MathGradTest, MatMul_01) { 1096 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1097 TensorShape({2, 3})); 1098 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 3})); 1099 Tensor dx; 1100 Tensor dy; 1101 MatMulGrad(x, false, y, true, &dx, &dy); 1102 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({2, 1})); 1103 test::ExpectClose(dx, MatMul(dz, false, y, false)); 1104 test::ExpectClose(dy, MatMul(dz, true, x, false)); 1105 } 1106 1107 TEST_F(MathGradTest, MatMul_10) { 1108 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1109 TensorShape({3, 2})); 1110 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({3, 1})); 1111 Tensor dx; 1112 Tensor dy; 1113 MatMulGrad(x, true, y, false, &dx, &dy); 1114 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({2, 1})); 1115 test::ExpectClose(dx, MatMul(y, false, dz, true)); 1116 test::ExpectClose(dy, MatMul(x, false, dz, false)); 1117 } 1118 1119 TEST_F(MathGradTest, MatMul_11) { 1120 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1121 TensorShape({3, 2})); 1122 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 3})); 1123 Tensor dx; 1124 Tensor dy; 1125 MatMulGrad(x, true, y, true, &dx, &dy); 1126 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({2, 1})); 1127 test::ExpectClose(dx, MatMul(y, true, dz, true)); 1128 test::ExpectClose(dy, MatMul(dz, true, x, true)); 1129 } 1130 1131 // TODO{lukeiwanski}: Implement BatchMatMul for SYCL 1132 #ifndef TENSORFLOW_USE_SYCL 1133 TEST_F(MathGradTest, BatchMatMul_00) { 1134 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1135 TensorShape({1, 2, 3})); 1136 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 3, 1})); 1137 Tensor dx; 1138 Tensor dy; 1139 BatchMatMulGrad(x, false, y, false, &dx, &dy); 1140 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({1, 2, 1})); 1141 test::ExpectClose(dx, BatchMatMul(dz, false, y, true)); 1142 test::ExpectClose(dy, BatchMatMul(x, true, dz, false)); 1143 } 1144 1145 TEST_F(MathGradTest, BatchMatMul_01) { 1146 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1147 TensorShape({1, 2, 3})); 1148 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 1, 3})); 1149 Tensor dx; 1150 Tensor dy; 1151 BatchMatMulGrad(x, false, y, true, &dx, &dy); 1152 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({1, 2, 1})); 1153 test::ExpectClose(dx, BatchMatMul(dz, false, y, false)); 1154 test::ExpectClose(dy, BatchMatMul(dz, true, x, false)); 1155 } 1156 1157 TEST_F(MathGradTest, BatchMatMul_10) { 1158 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1159 TensorShape({1, 3, 2})); 1160 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 3, 1})); 1161 Tensor dx; 1162 Tensor dy; 1163 BatchMatMulGrad(x, true, y, false, &dx, &dy); 1164 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({1, 2, 1})); 1165 test::ExpectClose(dx, BatchMatMul(y, false, dz, true)); 1166 test::ExpectClose(dy, BatchMatMul(x, false, dz, false)); 1167 } 1168 1169 TEST_F(MathGradTest, BatchMatMul_11) { 1170 auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, 1171 TensorShape({1, 3, 2})); 1172 auto y = test::AsTensor<float>({-1.f, .5f, 2.f}, TensorShape({1, 1, 3})); 1173 Tensor dx; 1174 Tensor dy; 1175 BatchMatMulGrad(x, true, y, true, &dx, &dy); 1176 auto dz = test::AsTensor<float>({1.f, 1.f}, TensorShape({1, 2, 1})); 1177 test::ExpectClose(dx, BatchMatMul(y, true, dz, true)); 1178 test::ExpectClose(dy, BatchMatMul(dz, true, x, true)); 1179 } 1180 #endif // TENSORFLOW_USE_SYCL 1181 1182 TEST_F(MathGradTest, Sum_dim0) { 1183 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1184 TensorShape({2, 3})); 1185 auto i = test::AsTensor<int32>({0}, TensorShape({})); 1186 Tensor dx; 1187 Tensor di; 1188 ReductionGrad("Sum", x, i, &dx, &di); 1189 test::ExpectTensorEqual<float>( 1190 dx, test::AsTensor<float>({1.f, 1.f, 1.f, 1.f, 1.f, 1.f}, 1191 TensorShape({2, 3}))); 1192 test::ExpectTensorEqual<int32>(di, 1193 test::AsTensor<int32>({0}, TensorShape({}))); 1194 } 1195 1196 TEST_F(MathGradTest, Sum_dim1) { 1197 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1198 TensorShape({2, 3})); 1199 auto i = test::AsTensor<int32>({1}, TensorShape({})); 1200 Tensor dx; 1201 Tensor di; 1202 ReductionGrad("Sum", x, i, &dx, &di); 1203 test::ExpectTensorEqual<float>( 1204 dx, test::AsTensor<float>({1.f, 1.f, 1.f, 1.f, 1.f, 1.f}, 1205 TensorShape({2, 3}))); 1206 test::ExpectTensorEqual<int32>(di, 1207 test::AsTensor<int32>({0}, TensorShape({}))); 1208 } 1209 1210 TEST_F(MathGradTest, Mean_dim0) { 1211 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1212 TensorShape({2, 3})); 1213 auto i = test::AsTensor<int32>({0}, TensorShape({})); 1214 Tensor dx; 1215 Tensor di; 1216 ReductionGrad("Mean", x, i, &dx, &di); 1217 test::ExpectTensorEqual<float>( 1218 dx, test::AsTensor<float>( 1219 {1.f / 2, 1.f / 2, 1.f / 2, 1.f / 2, 1.f / 2, 1.f / 2}, 1220 TensorShape({2, 3}))); 1221 test::ExpectTensorEqual<int32>(di, 1222 test::AsTensor<int32>({0}, TensorShape({}))); 1223 } 1224 1225 TEST_F(MathGradTest, Mean_dim1) { 1226 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1227 TensorShape({2, 3})); 1228 auto i = test::AsTensor<int32>({1}, TensorShape({})); 1229 Tensor dx; 1230 Tensor di; 1231 ReductionGrad("Mean", x, i, &dx, &di); 1232 test::ExpectTensorEqual<float>( 1233 dx, test::AsTensor<float>( 1234 {1.f / 3, 1.f / 3, 1.f / 3, 1.f / 3, 1.f / 3, 1.f / 3}, 1235 TensorShape({2, 3}))); 1236 test::ExpectTensorEqual<int32>(di, 1237 test::AsTensor<int32>({0}, TensorShape({}))); 1238 } 1239 1240 TEST_F(MathGradTest, Mean_dim0_dim1) { 1241 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1242 TensorShape({2, 3})); 1243 auto i = test::AsTensor<int32>({0, 1}, TensorShape({2})); 1244 Tensor dx; 1245 Tensor di; 1246 ReductionGrad("Mean", x, i, &dx, &di); 1247 test::ExpectTensorEqual<float>( 1248 dx, test::AsTensor<float>( 1249 {1.f / 6, 1.f / 6, 1.f / 6, 1.f / 6, 1.f / 6, 1.f / 6}, 1250 TensorShape({2, 3}))); 1251 test::ExpectTensorEqual<int32>( 1252 di, test::AsTensor<int32>({0, 0}, TensorShape({2}))); 1253 } 1254 1255 TEST_F(MathGradTest, Min_dim0) { 1256 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1257 TensorShape({2, 3})); 1258 auto i = test::AsTensor<int32>({0}, TensorShape({})); 1259 Tensor dx; 1260 Tensor di; 1261 ReductionGrad("Min", x, i, &dx, &di); 1262 test::ExpectTensorEqual<float>( 1263 dx, test::AsTensor<float>({1.f, 1.f, 1.f, 0.f, 0.f, 0.f}, 1264 TensorShape({2, 3}))); 1265 test::ExpectTensorEqual<int32>(di, 1266 test::AsTensor<int32>({0}, TensorShape({}))); 1267 } 1268 1269 TEST_F(MathGradTest, Min_dim1) { 1270 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1271 TensorShape({2, 3})); 1272 auto i = test::AsTensor<int32>({1}, TensorShape({})); 1273 Tensor dx; 1274 Tensor di; 1275 ReductionGrad("Min", x, i, &dx, &di); 1276 test::ExpectTensorEqual<float>( 1277 dx, test::AsTensor<float>({1.f, 0.f, 0.f, 1.f, 0.f, 0.f}, 1278 TensorShape({2, 3}))); 1279 test::ExpectTensorEqual<int32>(di, 1280 test::AsTensor<int32>({0}, TensorShape({}))); 1281 } 1282 1283 TEST_F(MathGradTest, Min_dim0_dim1) { 1284 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1285 TensorShape({2, 3})); 1286 auto i = test::AsTensor<int32>({0, 1}, TensorShape({2})); 1287 Tensor dx; 1288 Tensor di; 1289 ReductionGrad("Min", x, i, &dx, &di); 1290 test::ExpectTensorEqual<float>( 1291 dx, test::AsTensor<float>({1.f, 0.f, 0.f, 0.f, 0.f, 0.f}, 1292 TensorShape({2, 3}))); 1293 test::ExpectTensorEqual<int32>( 1294 di, test::AsTensor<int32>({0, 0}, TensorShape({2}))); 1295 } 1296 1297 TEST_F(MathGradTest, Min_dim0_dim1_Dups) { 1298 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, -3.f}, 1299 TensorShape({2, 3})); 1300 auto i = test::AsTensor<int32>({0, 1}, TensorShape({2})); 1301 Tensor dx; 1302 Tensor di; 1303 ReductionGrad("Min", x, i, &dx, &di); 1304 test::ExpectTensorEqual<float>( 1305 dx, test::AsTensor<float>({.5f, 0.f, 0.f, 0.f, 0.f, .5f}, 1306 TensorShape({2, 3}))); 1307 test::ExpectTensorEqual<int32>( 1308 di, test::AsTensor<int32>({0, 0}, TensorShape({2}))); 1309 } 1310 1311 TEST_F(MathGradTest, Max_dim0) { 1312 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1313 TensorShape({2, 3})); 1314 auto i = test::AsTensor<int32>({0}, TensorShape({})); 1315 Tensor dx; 1316 Tensor di; 1317 ReductionGrad("Max", x, i, &dx, &di); 1318 LOG(INFO) << dx.SummarizeValue(6); 1319 test::ExpectTensorEqual<float>( 1320 dx, test::AsTensor<float>({0.f, 0.f, 0.f, 1.f, 1.f, 1.f}, 1321 TensorShape({2, 3}))); 1322 test::ExpectTensorEqual<int32>(di, 1323 test::AsTensor<int32>({0}, TensorShape({}))); 1324 } 1325 1326 TEST_F(MathGradTest, Max_dim1) { 1327 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1328 TensorShape({2, 3})); 1329 auto i = test::AsTensor<int32>({1}, TensorShape({})); 1330 Tensor dx; 1331 Tensor di; 1332 ReductionGrad("Max", x, i, &dx, &di); 1333 test::ExpectTensorEqual<float>( 1334 dx, test::AsTensor<float>({0.f, 0.f, 1.f, 0.f, 0.f, 1.f}, 1335 TensorShape({2, 3}))); 1336 test::ExpectTensorEqual<int32>(di, 1337 test::AsTensor<int32>({0}, TensorShape({}))); 1338 } 1339 1340 TEST_F(MathGradTest, Max_dim0_dim1) { 1341 auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1342 TensorShape({2, 3})); 1343 auto i = test::AsTensor<int32>({0, 1}, TensorShape({2})); 1344 Tensor dx; 1345 Tensor di; 1346 ReductionGrad("Max", x, i, &dx, &di); 1347 test::ExpectTensorEqual<float>( 1348 dx, test::AsTensor<float>({0.f, 0.f, 0.f, 0.f, 0.f, 1.f}, 1349 TensorShape({2, 3}))); 1350 test::ExpectTensorEqual<int32>( 1351 di, test::AsTensor<int32>({0, 0}, TensorShape({2}))); 1352 } 1353 1354 TEST_F(MathGradTest, Max_dim0_dim1_Dups) { 1355 auto x = test::AsTensor<float>({3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, 1356 TensorShape({2, 3})); 1357 auto i = test::AsTensor<int32>({0, 1}, TensorShape({2})); 1358 Tensor dx; 1359 Tensor di; 1360 ReductionGrad("Max", x, i, &dx, &di); 1361 test::ExpectTensorEqual<float>( 1362 dx, test::AsTensor<float>({.5f, 0.f, 0.f, 0.f, 0.f, .5f}, 1363 TensorShape({2, 3}))); 1364 test::ExpectTensorEqual<int32>( 1365 di, test::AsTensor<int32>({0, 0}, TensorShape({2}))); 1366 } 1367 1368 } // namespace 1369 } // namespace tensorflow 1370
