1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #include "tensorflow/core/debug/debug_graph_utils.h" 17 #include "tensorflow/core/debug/debug_grpc_testlib.h" 18 #include "tensorflow/core/debug/debug_io_utils.h" 19 #include "tensorflow/core/lib/core/notification.h" 20 #include "tensorflow/core/lib/core/status_test_util.h" 21 #include "tensorflow/core/lib/core/threadpool.h" 22 #include "tensorflow/core/lib/io/path.h" 23 #include "tensorflow/core/lib/strings/str_util.h" 24 #include "tensorflow/core/lib/strings/stringprintf.h" 25 #include "tensorflow/core/platform/env.h" 26 #include "tensorflow/core/platform/tracing.h" 27 28 namespace tensorflow { 29 30 class GrpcDebugTest : public ::testing::Test { 31 protected: 32 struct ServerData { 33 int port; 34 string url; 35 std::unique_ptr<test::TestEventListenerImpl> server; 36 std::unique_ptr<thread::ThreadPool> thread_pool; 37 }; 38 39 void SetUp() override { 40 ClearEnabledWatchKeys(); 41 SetUpInProcessServer(&server_data_, 0); 42 } 43 44 void TearDown() override { TearDownInProcessServer(&server_data_); } 45 46 void SetUpInProcessServer(ServerData* server_data, 47 int64 server_start_delay_micros) { 48 server_data->port = testing::PickUnusedPortOrDie(); 49 server_data->url = strings::StrCat("grpc://localhost:", server_data->port); 50 server_data->server.reset(new test::TestEventListenerImpl()); 51 52 server_data->thread_pool.reset( 53 new thread::ThreadPool(Env::Default(), "test_server", 1)); 54 server_data->thread_pool->Schedule( 55 [server_data, server_start_delay_micros]() { 56 Env::Default()->SleepForMicroseconds(server_start_delay_micros); 57 server_data->server->RunServer(server_data->port); 58 }); 59 } 60 61 void TearDownInProcessServer(ServerData* server_data) { 62 server_data->server->StopServer(); 63 server_data->thread_pool.reset(); 64 } 65 66 void ClearEnabledWatchKeys() { DebugGrpcIO::ClearEnabledWatchKeys(); } 67 68 const int64 GetChannelConnectionTimeoutMicros() { 69 return DebugGrpcIO::channel_connection_timeout_micros; 70 } 71 72 void SetChannelConnectionTimeoutMicros(const int64 timeout) { 73 DebugGrpcIO::channel_connection_timeout_micros = timeout; 74 } 75 76 ServerData server_data_; 77 }; 78 79 TEST_F(GrpcDebugTest, ConnectionTimeoutWorks) { 80 // Use a short timeout so the test won't take too long. 81 const int64 kOriginalTimeoutMicros = GetChannelConnectionTimeoutMicros(); 82 const int64 kShortTimeoutMicros = 500 * 1000; 83 SetChannelConnectionTimeoutMicros(kShortTimeoutMicros); 84 ASSERT_EQ(kShortTimeoutMicros, GetChannelConnectionTimeoutMicros()); 85 86 const string& kInvalidGrpcUrl = 87 strings::StrCat("grpc://localhost:", testing::PickUnusedPortOrDie()); 88 Tensor tensor(DT_FLOAT, TensorShape({1, 1})); 89 tensor.flat<float>()(0) = 42.0; 90 Status publish_status = DebugIO::PublishDebugTensor( 91 DebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", "foo_tensor", 0, 92 "DebugIdentity"), 93 tensor, Env::Default()->NowMicros(), {kInvalidGrpcUrl}); 94 SetChannelConnectionTimeoutMicros(kOriginalTimeoutMicros); 95 TF_ASSERT_OK(DebugIO::CloseDebugURL(kInvalidGrpcUrl)); 96 97 ASSERT_FALSE(publish_status.ok()); 98 const string expected_error_msg = strings::StrCat( 99 "Failed to connect to gRPC channel at ", kInvalidGrpcUrl.substr(7), 100 " within a timeout of ", kShortTimeoutMicros / 1e6, " s"); 101 ASSERT_NE(string::npos, 102 publish_status.error_message().find(expected_error_msg)); 103 } 104 105 TEST_F(GrpcDebugTest, ConnectionToDelayedStartingServerWorks) { 106 ServerData server_data; 107 // Server start will be delayed for 1 second. 108 SetUpInProcessServer(&server_data, 1 * 1000 * 1000); 109 110 Tensor tensor(DT_FLOAT, TensorShape({1, 1})); 111 tensor.flat<float>()(0) = 42.0; 112 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 113 "foo_tensor", 0, "DebugIdentity"); 114 Status publish_status = DebugIO::PublishDebugTensor( 115 kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data.url}); 116 ASSERT_TRUE(publish_status.ok()); 117 TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data.url)); 118 119 ASSERT_EQ(1, server_data.server->node_names.size()); 120 ASSERT_EQ(1, server_data.server->output_slots.size()); 121 ASSERT_EQ(1, server_data.server->debug_ops.size()); 122 EXPECT_EQ(kDebugNodeKey.device_name, server_data.server->device_names[0]); 123 EXPECT_EQ(kDebugNodeKey.node_name, server_data.server->node_names[0]); 124 EXPECT_EQ(kDebugNodeKey.output_slot, server_data.server->output_slots[0]); 125 EXPECT_EQ(kDebugNodeKey.debug_op, server_data.server->debug_ops[0]); 126 TearDownInProcessServer(&server_data); 127 } 128 129 TEST_F(GrpcDebugTest, SendSingleDebugTensorViaGrpcTest) { 130 Tensor tensor(DT_FLOAT, TensorShape({1, 1})); 131 tensor.flat<float>()(0) = 42.0; 132 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 133 "foo_tensor", 0, "DebugIdentity"); 134 TF_ASSERT_OK(DebugIO::PublishDebugTensor( 135 kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url})); 136 TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url)); 137 138 // Verify that the expected debug tensor sending happened. 139 ASSERT_EQ(1, server_data_.server->node_names.size()); 140 ASSERT_EQ(1, server_data_.server->output_slots.size()); 141 ASSERT_EQ(1, server_data_.server->debug_ops.size()); 142 EXPECT_EQ(kDebugNodeKey.device_name, server_data_.server->device_names[0]); 143 EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]); 144 EXPECT_EQ(kDebugNodeKey.output_slot, server_data_.server->output_slots[0]); 145 EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]); 146 } 147 148 TEST_F(GrpcDebugTest, SendDebugTensorWithLargeStringAtIndex0ViaGrpcTest) { 149 Tensor tensor(DT_STRING, TensorShape({1, 1})); 150 tensor.flat<string>()(0) = string(5000 * 1024, 'A'); 151 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 152 "foo_tensor", 0, "DebugIdentity"); 153 const Status status = DebugIO::PublishDebugTensor( 154 kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url}); 155 ASSERT_FALSE(status.ok()); 156 ASSERT_NE(status.error_message().find("string value at index 0 from debug " 157 "node foo_tensor:0:DebugIdentity does " 158 "not fit gRPC message size limit"), 159 string::npos); 160 TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url)); 161 } 162 163 TEST_F(GrpcDebugTest, SendDebugTensorWithLargeStringAtIndex1ViaGrpcTest) { 164 Tensor tensor(DT_STRING, TensorShape({1, 2})); 165 tensor.flat<string>()(0) = "A"; 166 tensor.flat<string>()(1) = string(5000 * 1024, 'A'); 167 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 168 "foo_tensor", 0, "DebugIdentity"); 169 const Status status = DebugIO::PublishDebugTensor( 170 kDebugNodeKey, tensor, Env::Default()->NowMicros(), {server_data_.url}); 171 ASSERT_FALSE(status.ok()); 172 ASSERT_NE(status.error_message().find("string value at index 1 from debug " 173 "node foo_tensor:0:DebugIdentity does " 174 "not fit gRPC message size limit"), 175 string::npos); 176 TF_ASSERT_OK(DebugIO::CloseDebugURL(server_data_.url)); 177 } 178 179 TEST_F(GrpcDebugTest, SendMultipleDebugTensorsSynchronizedViaGrpcTest) { 180 const int32 kSends = 4; 181 182 // Prepare the tensors to sent. 183 std::vector<Tensor> tensors; 184 for (int i = 0; i < kSends; ++i) { 185 Tensor tensor(DT_INT32, TensorShape({1, 1})); 186 tensor.flat<int>()(0) = i * i; 187 tensors.push_back(tensor); 188 } 189 190 thread::ThreadPool* tp = 191 new thread::ThreadPool(Env::Default(), "grpc_debug_test", kSends); 192 193 mutex mu; 194 Notification all_done; 195 int tensor_count GUARDED_BY(mu) = 0; 196 std::vector<Status> statuses GUARDED_BY(mu); 197 198 const std::vector<string> urls({server_data_.url}); 199 200 // Set up the concurrent tasks of sending Tensors via an Event stream to the 201 // server. 202 auto fn = [this, &mu, &tensor_count, &tensors, &statuses, &all_done, 203 &urls]() { 204 int this_count; 205 { 206 mutex_lock l(mu); 207 this_count = tensor_count++; 208 } 209 210 // Different concurrent tasks will send different tensors. 211 const uint64 wall_time = Env::Default()->NowMicros(); 212 Status publish_status = DebugIO::PublishDebugTensor( 213 DebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 214 strings::StrCat("synchronized_node_", this_count), 0, 215 "DebugIdentity"), 216 tensors[this_count], wall_time, urls); 217 218 { 219 mutex_lock l(mu); 220 statuses.push_back(publish_status); 221 if (this_count == kSends - 1 && !all_done.HasBeenNotified()) { 222 all_done.Notify(); 223 } 224 } 225 }; 226 227 // Schedule the concurrent tasks. 228 for (int i = 0; i < kSends; ++i) { 229 tp->Schedule(fn); 230 } 231 232 // Wait for all client tasks to finish. 233 all_done.WaitForNotification(); 234 delete tp; 235 236 // Close the debug gRPC stream. 237 Status close_status = DebugIO::CloseDebugURL(server_data_.url); 238 ASSERT_TRUE(close_status.ok()); 239 240 // Check all statuses from the PublishDebugTensor calls(). 241 for (const Status& status : statuses) { 242 TF_ASSERT_OK(status); 243 } 244 245 // One prep tensor plus kSends concurrent tensors are expected. 246 ASSERT_EQ(kSends, server_data_.server->node_names.size()); 247 for (size_t i = 0; i < server_data_.server->node_names.size(); ++i) { 248 std::vector<string> items = 249 str_util::Split(server_data_.server->node_names[i], '_'); 250 int tensor_index; 251 strings::safe_strto32(items[2], &tensor_index); 252 253 ASSERT_EQ(TensorShape({1, 1}), 254 server_data_.server->debug_tensors[i].shape()); 255 ASSERT_EQ(tensor_index * tensor_index, 256 server_data_.server->debug_tensors[i].flat<int>()(0)); 257 } 258 } 259 260 TEST_F(GrpcDebugTest, SendDebugTensorsThroughMultipleRoundsUsingGrpcGating) { 261 // Prepare the tensor to send. 262 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 263 "test_namescope/test_node", 0, 264 "DebugIdentity"); 265 Tensor tensor(DT_INT32, TensorShape({1, 1})); 266 tensor.flat<int>()(0) = 42; 267 268 const std::vector<string> urls({server_data_.url}); 269 for (int i = 0; i < 3; ++i) { 270 server_data_.server->ClearReceivedDebugData(); 271 const uint64 wall_time = Env::Default()->NowMicros(); 272 273 // On the 1st send (i == 0), gating is disabled, so data should be sent. 274 // On the 2nd send (i == 1), gating is enabled, and the server has enabled 275 // the watch key in the previous send, so data should be sent. 276 // On the 3rd send (i == 2), gating is enabled, but the server has disabled 277 // the watch key in the previous send, so data should not be sent. 278 const bool enable_gated_grpc = (i != 0); 279 TF_ASSERT_OK(DebugIO::PublishDebugTensor(kDebugNodeKey, tensor, wall_time, 280 urls, enable_gated_grpc)); 281 282 server_data_.server->RequestDebugOpStateChangeAtNextStream( 283 i == 0 ? EventReply::DebugOpStateChange::READ_ONLY 284 : EventReply::DebugOpStateChange::DISABLED, 285 kDebugNodeKey); 286 287 // Close the debug gRPC stream. 288 Status close_status = DebugIO::CloseDebugURL(server_data_.url); 289 ASSERT_TRUE(close_status.ok()); 290 291 // Check dumped files according to the expected gating results. 292 if (i < 2) { 293 ASSERT_EQ(1, server_data_.server->node_names.size()); 294 ASSERT_EQ(1, server_data_.server->output_slots.size()); 295 ASSERT_EQ(1, server_data_.server->debug_ops.size()); 296 EXPECT_EQ(kDebugNodeKey.device_name, 297 server_data_.server->device_names[0]); 298 EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]); 299 EXPECT_EQ(kDebugNodeKey.output_slot, 300 server_data_.server->output_slots[0]); 301 EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]); 302 } else { 303 ASSERT_EQ(0, server_data_.server->node_names.size()); 304 } 305 } 306 } 307 308 TEST_F(GrpcDebugTest, SendDebugTensorsThroughMultipleRoundsUnderReadWriteMode) { 309 // Prepare the tensor to send. 310 const DebugNodeKey kDebugNodeKey("/job:localhost/replica:0/task:0/cpu:0", 311 "test_namescope/test_node", 0, 312 "DebugIdentity"); 313 Tensor tensor(DT_INT32, TensorShape({1, 1})); 314 tensor.flat<int>()(0) = 42; 315 316 const std::vector<string> urls({server_data_.url}); 317 for (int i = 0; i < 3; ++i) { 318 server_data_.server->ClearReceivedDebugData(); 319 const uint64 wall_time = Env::Default()->NowMicros(); 320 321 // On the 1st send (i == 0), gating is disabled, so data should be sent. 322 // On the 2nd send (i == 1), gating is enabled, and the server has enabled 323 // the watch key in the previous send (READ_WRITE), so data should be 324 // sent. In this iteration, the server response with a EventReply proto to 325 // unblock the debug node. 326 // On the 3rd send (i == 2), gating is enabled, but the server has disabled 327 // the watch key in the previous send, so data should not be sent. 328 const bool enable_gated_grpc = (i != 0); 329 TF_ASSERT_OK(DebugIO::PublishDebugTensor(kDebugNodeKey, tensor, wall_time, 330 urls, enable_gated_grpc)); 331 332 server_data_.server->RequestDebugOpStateChangeAtNextStream( 333 i == 0 ? EventReply::DebugOpStateChange::READ_WRITE 334 : EventReply::DebugOpStateChange::DISABLED, 335 kDebugNodeKey); 336 337 // Close the debug gRPC stream. 338 Status close_status = DebugIO::CloseDebugURL(server_data_.url); 339 ASSERT_TRUE(close_status.ok()); 340 341 // Check dumped files according to the expected gating results. 342 if (i < 2) { 343 ASSERT_EQ(1, server_data_.server->node_names.size()); 344 ASSERT_EQ(1, server_data_.server->output_slots.size()); 345 ASSERT_EQ(1, server_data_.server->debug_ops.size()); 346 EXPECT_EQ(kDebugNodeKey.device_name, 347 server_data_.server->device_names[0]); 348 EXPECT_EQ(kDebugNodeKey.node_name, server_data_.server->node_names[0]); 349 EXPECT_EQ(kDebugNodeKey.output_slot, 350 server_data_.server->output_slots[0]); 351 EXPECT_EQ(kDebugNodeKey.debug_op, server_data_.server->debug_ops[0]); 352 } else { 353 ASSERT_EQ(0, server_data_.server->node_names.size()); 354 } 355 } 356 } 357 358 TEST_F(GrpcDebugTest, TestGateDebugNodeOnEmptyEnabledSet) { 359 ASSERT_FALSE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", 360 {"grpc://localhost:3333"})); 361 362 // file:// debug URLs are not subject to grpc gating. 363 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen( 364 "foo:0:DebugIdentity", {"grpc://localhost:3333", "file:///tmp/tfdbg_1"})); 365 } 366 367 TEST_F(GrpcDebugTest, TestGateDebugNodeOnNonEmptyEnabledSet) { 368 const string kGrpcUrl1 = "grpc://localhost:3333"; 369 const string kGrpcUrl2 = "grpc://localhost:3334"; 370 371 DebugGrpcIO::SetDebugNodeKeyGrpcState( 372 kGrpcUrl1, "foo:0:DebugIdentity", 373 EventReply::DebugOpStateChange::READ_ONLY); 374 DebugGrpcIO::SetDebugNodeKeyGrpcState( 375 kGrpcUrl1, "bar:0:DebugIdentity", 376 EventReply::DebugOpStateChange::READ_ONLY); 377 378 ASSERT_FALSE( 379 DebugIO::IsDebugNodeGateOpen("foo:1:DebugIdentity", {kGrpcUrl1})); 380 ASSERT_FALSE( 381 DebugIO::IsDebugNodeGateOpen("foo:1:DebugNumericSummary", {kGrpcUrl1})); 382 ASSERT_FALSE( 383 DebugIO::IsDebugNodeGateOpen("qux:0:DebugIdentity", {kGrpcUrl1})); 384 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl1})); 385 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl1})); 386 387 // Wrong grpc:// debug URLs. 388 ASSERT_FALSE( 389 DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl2})); 390 ASSERT_FALSE( 391 DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl2})); 392 393 // file:// debug URLs are not subject to grpc gating. 394 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("qux:0:DebugIdentity", 395 {"file:///tmp/tfdbg_1", kGrpcUrl1})); 396 } 397 398 TEST_F(GrpcDebugTest, TestGateDebugNodeOnMultipleEmptyEnabledSets) { 399 const string kGrpcUrl1 = "grpc://localhost:3333"; 400 const string kGrpcUrl2 = "grpc://localhost:3334"; 401 const string kGrpcUrl3 = "grpc://localhost:3335"; 402 403 DebugGrpcIO::SetDebugNodeKeyGrpcState( 404 kGrpcUrl1, "foo:0:DebugIdentity", 405 EventReply::DebugOpStateChange::READ_ONLY); 406 DebugGrpcIO::SetDebugNodeKeyGrpcState( 407 kGrpcUrl2, "bar:0:DebugIdentity", 408 EventReply::DebugOpStateChange::READ_ONLY); 409 410 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl1})); 411 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl2})); 412 ASSERT_FALSE( 413 DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl2})); 414 ASSERT_FALSE( 415 DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl1})); 416 ASSERT_FALSE( 417 DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", {kGrpcUrl3})); 418 ASSERT_FALSE( 419 DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", {kGrpcUrl3})); 420 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", 421 {kGrpcUrl1, kGrpcUrl2})); 422 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", 423 {kGrpcUrl1, kGrpcUrl2})); 424 ASSERT_TRUE(DebugIO::IsDebugNodeGateOpen("foo:0:DebugIdentity", 425 {kGrpcUrl1, kGrpcUrl3})); 426 ASSERT_FALSE(DebugIO::IsDebugNodeGateOpen("bar:0:DebugIdentity", 427 {kGrpcUrl1, kGrpcUrl3})); 428 } 429 430 TEST_F(GrpcDebugTest, TestGateDebugNodeOnNonEmptyEnabledSetAndEmptyURLs) { 431 DebugGrpcIO::SetDebugNodeKeyGrpcState( 432 "grpc://localhost:3333", "foo:0:DebugIdentity", 433 EventReply::DebugOpStateChange::READ_ONLY); 434 435 std::vector<string> debug_urls_1; 436 ASSERT_FALSE( 437 DebugIO::IsDebugNodeGateOpen("foo:1:DebugIdentity", debug_urls_1)); 438 } 439 440 TEST_F(GrpcDebugTest, TestGateCopyNodeOnEmptyEnabledSet) { 441 const string kGrpcUrl1 = "grpc://localhost:3333"; 442 const string kWatch1 = "foo:0:DebugIdentity"; 443 444 ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen( 445 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true)})); 446 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 447 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, false)})); 448 449 // file:// debug URLs are not subject to grpc gating. 450 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 451 {DebugWatchAndURLSpec("foo:0:DebugIdentity", kGrpcUrl1, true), 452 DebugWatchAndURLSpec("foo:0:DebugIdentity", "file:///tmp/tfdbg_1", 453 false)})); 454 } 455 456 TEST_F(GrpcDebugTest, TestGateCopyNodeOnNonEmptyEnabledSet) { 457 const string kGrpcUrl1 = "grpc://localhost:3333"; 458 const string kGrpcUrl2 = "grpc://localhost:3334"; 459 const string kWatch1 = "foo:0:DebugIdentity"; 460 const string kWatch2 = "foo:1:DebugIdentity"; 461 DebugGrpcIO::SetDebugNodeKeyGrpcState( 462 kGrpcUrl1, kWatch1, EventReply::DebugOpStateChange::READ_ONLY); 463 464 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 465 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true)})); 466 467 ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen( 468 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, true)})); 469 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 470 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, false)})); 471 472 ASSERT_FALSE(DebugIO::IsCopyNodeGateOpen( 473 {DebugWatchAndURLSpec(kWatch2, kGrpcUrl1, true)})); 474 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 475 {DebugWatchAndURLSpec(kWatch2, kGrpcUrl1, false)})); 476 477 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 478 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true), 479 DebugWatchAndURLSpec(kWatch1, kGrpcUrl2, true)})); 480 ASSERT_TRUE(DebugIO::IsCopyNodeGateOpen( 481 {DebugWatchAndURLSpec(kWatch1, kGrpcUrl1, true), 482 DebugWatchAndURLSpec(kWatch2, kGrpcUrl2, true)})); 483 } 484 485 } // namespace tensorflow 486
