1 # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Unit tests for debug_gradients module.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 import shutil 22 import tempfile 23 24 from tensorflow.core.protobuf import config_pb2 25 from tensorflow.core.protobuf import rewriter_config_pb2 26 from tensorflow.python.client import session 27 from tensorflow.python.debug.lib import debug_data 28 from tensorflow.python.debug.lib import debug_gradients 29 from tensorflow.python.debug.lib import debug_utils 30 from tensorflow.python.framework import ops 31 from tensorflow.python.framework import test_util 32 from tensorflow.python.ops import gradients_impl 33 from tensorflow.python.ops import math_ops 34 from tensorflow.python.ops import variables 35 from tensorflow.python.platform import googletest 36 from tensorflow.python.training import gradient_descent 37 38 39 class IdentifyGradientTest(test_util.TensorFlowTestCase): 40 41 def setUp(self): 42 rewriter_config = rewriter_config_pb2.RewriterConfig( 43 disable_model_pruning=True, 44 dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF) 45 graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config) 46 config = config_pb2.ConfigProto(graph_options=graph_options) 47 self.sess = session.Session(config=config) 48 with self.sess.as_default(): 49 self.u = variables.Variable(2.0, name="u") 50 self.v = variables.Variable(3.0, name="v") 51 self.w = math_ops.multiply(self.u.value(), self.v.value(), name="w") 52 53 def tearDown(self): 54 ops.reset_default_graph() 55 debug_gradients.clear_gradient_debuggers() 56 57 def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self): 58 grad_debugger = debug_gradients.GradientsDebugger() 59 id_grad_w = grad_debugger.identify_gradient(self.w) 60 y = math_ops.add(id_grad_w, -1.0, name="y") 61 62 grads = gradients_impl.gradients(y, [self.u, self.v]) 63 self.assertEqual(2, len(grads)) 64 u_grad = grads[0] 65 v_grad = grads[1] 66 67 self.sess.run(variables.global_variables_initializer()) 68 self.assertAllClose(5.0, self.sess.run(y)) 69 self.assertAllClose(3.0, self.sess.run(u_grad)) 70 self.assertAllClose(2.0, self.sess.run(v_grad)) 71 72 # Fetch the gradient tensor with the x-tensor object. 73 w_grad = grad_debugger.gradient_tensor(self.w) 74 self.assertIsInstance(w_grad, ops.Tensor) 75 self.assertAllClose(1.0, self.sess.run(w_grad)) 76 77 # Fetch the gradient tensor with the x-tensor's name. 78 w_grad = grad_debugger.gradient_tensor(self.w.name) 79 self.assertIsInstance(w_grad, ops.Tensor) 80 self.assertAllClose(1.0, self.sess.run(w_grad)) 81 82 # Fetch the gradient tensor with the x-tensor name. 83 w_grad = grad_debugger.gradient_tensor(self.w.name) 84 self.assertIsInstance(w_grad, ops.Tensor) 85 self.assertAllClose(1.0, self.sess.run(w_grad)) 86 87 def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self): 88 grad_debugger = debug_gradients.GradientsDebugger() 89 id_grad_w = grad_debugger.identify_gradient(self.w) 90 y = math_ops.add(id_grad_w, -1.0, name="y") 91 92 with grad_debugger: 93 grads = gradients_impl.gradients(y, [self.u, self.v]) 94 self.assertEqual(2, len(grads)) 95 u_grad = grads[0] 96 v_grad = grads[1] 97 98 self.sess.run(variables.global_variables_initializer()) 99 self.assertAllClose(5.0, self.sess.run(y)) 100 self.assertAllClose(3.0, self.sess.run(u_grad)) 101 self.assertAllClose(2.0, self.sess.run(v_grad)) 102 103 # Fetch the gradient tensor with the x-tensor object. 104 w_grad = grad_debugger.gradient_tensor(self.w) 105 self.assertIsInstance(w_grad, ops.Tensor) 106 self.assertAllClose(1.0, self.sess.run(w_grad)) 107 108 # Fetch the gradient tensor with the x-tensor's name. 109 w_grad = grad_debugger.gradient_tensor(self.w.name) 110 self.assertIsInstance(w_grad, ops.Tensor) 111 self.assertAllClose(1.0, self.sess.run(w_grad)) 112 113 # Fetch the gradient tensor with the x-tensor name. 114 w_grad = grad_debugger.gradient_tensor(self.w.name) 115 self.assertIsInstance(w_grad, ops.Tensor) 116 self.assertAllClose(1.0, self.sess.run(w_grad)) 117 118 def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self): 119 grad_debugger = debug_gradients.GradientsDebugger() 120 grad_debugger.identify_gradient(self.w) 121 with self.assertRaisesRegexp(ValueError, 122 "The graph already contains an op named .*"): 123 grad_debugger.identify_gradient(self.w) 124 125 def testIdentifyGradientWorksOnMultipleLosses(self): 126 grad_debugger_1 = debug_gradients.GradientsDebugger() 127 grad_debugger_2 = debug_gradients.GradientsDebugger() 128 129 y = math_ops.add(self.w, -1.0, name="y") 130 debug_y = grad_debugger_1.identify_gradient(y) 131 z1 = math_ops.square(debug_y, name="z1") 132 133 debug_y = grad_debugger_2.identify_gradient(y) 134 z2 = math_ops.sqrt(debug_y, name="z2") 135 136 with grad_debugger_1: 137 gradient_descent.GradientDescentOptimizer(0.1).minimize(z1) 138 with grad_debugger_2: 139 gradient_descent.GradientDescentOptimizer(0.1).minimize(z2) 140 141 dz1_dy = grad_debugger_1.gradient_tensor(y) 142 dz2_dy = grad_debugger_2.gradient_tensor(y) 143 self.assertIsInstance(dz1_dy, ops.Tensor) 144 self.assertIsInstance(dz2_dy, ops.Tensor) 145 self.assertIsNot(dz1_dy, dz2_dy) 146 147 self.sess.run(variables.global_variables_initializer()) 148 self.assertAllClose(5.0**2, self.sess.run(z1)) 149 self.assertAllClose(5.0**0.5, self.sess.run(z2)) 150 self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) 151 self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) 152 153 def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self): 154 grad_debugger_1 = debug_gradients.GradientsDebugger() 155 grad_debugger_2 = debug_gradients.GradientsDebugger() 156 id_grad_w = grad_debugger_1.identify_gradient(self.w) 157 y = math_ops.add(id_grad_w, -1.0, name="y") 158 159 # There are >1 gradient debuggers registered, and grad_debugger is not used 160 # as a context manager here, so the gradient w.r.t. self.w will not be 161 # registered. 162 gradients_impl.gradients(y, [self.u, self.v]) 163 164 with self.assertRaisesRegexp( 165 LookupError, 166 r"This GradientsDebugger has not received any gradient tensor for "): 167 grad_debugger_1.gradient_tensor(self.w) 168 with self.assertRaisesRegexp( 169 LookupError, 170 r"This GradientsDebugger has not received any gradient tensor for "): 171 grad_debugger_2.gradient_tensor(self.w) 172 173 def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self): 174 grad_debugger = debug_gradients.GradientsDebugger() 175 with self.assertRaisesRegexp( 176 TypeError, 177 r"x_tensor must be a str or tf\.Tensor or tf\.Variable, but instead " 178 r"has type .*Operation.*"): 179 grad_debugger.gradient_tensor(variables.global_variables_initializer()) 180 181 def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self): 182 grad_debugger = debug_gradients.GradientsDebugger() 183 id_grad_w = grad_debugger.identify_gradient(self.w) 184 y = math_ops.add(id_grad_w, -1.0, name="y") 185 186 with grad_debugger: 187 gradient_descent.GradientDescentOptimizer(0.1).minimize(y) 188 189 self.sess.run(variables.global_variables_initializer()) 190 191 # Fetch the gradient tensor with the x-tensor object. 192 w_grad = grad_debugger.gradient_tensor(self.w) 193 self.assertIsInstance(w_grad, ops.Tensor) 194 self.assertAllClose(1.0, self.sess.run(w_grad)) 195 196 def testWatchGradientsByXTensorNamesWorks(self): 197 y = math_ops.add(self.w, -1.0, name="y") 198 199 # The constructrion of the forward graph has completed. 200 # But we can still get the gradient tensors by using 201 # watch_gradients_by_tensor_names(). 202 grad_debugger = debug_gradients.GradientsDebugger() 203 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$"): 204 grads = gradients_impl.gradients(y, [self.u, self.v]) 205 self.assertEqual(2, len(grads)) 206 u_grad = grads[0] 207 v_grad = grads[1] 208 209 self.sess.run(variables.global_variables_initializer()) 210 self.assertAllClose(5.0, self.sess.run(y)) 211 self.assertAllClose(3.0, self.sess.run(u_grad)) 212 self.assertAllClose(2.0, self.sess.run(v_grad)) 213 214 w_grad = grad_debugger.gradient_tensor(self.w) 215 self.assertIsInstance(w_grad, ops.Tensor) 216 self.assertAllClose(1.0, self.sess.run(w_grad)) 217 218 w_grad = grad_debugger.gradient_tensor("w:0") 219 self.assertIsInstance(w_grad, ops.Tensor) 220 self.assertAllClose(1.0, self.sess.run(w_grad)) 221 222 def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self): 223 y = math_ops.add(self.w, -1.0, name="y") 224 225 # The constructrion of the forward graph has completed. 226 # But we can still get the gradient tensors by using 227 # watch_gradients_by_tensor_names(). 228 grad_debugger = debug_gradients.GradientsDebugger() 229 grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$") 230 grads = gradients_impl.gradients(y, [self.u, self.v]) 231 self.assertEqual(2, len(grads)) 232 u_grad = grads[0] 233 v_grad = grads[1] 234 235 self.sess.run(variables.global_variables_initializer()) 236 self.assertAllClose(5.0, self.sess.run(y)) 237 self.assertAllClose(3.0, self.sess.run(u_grad)) 238 self.assertAllClose(2.0, self.sess.run(v_grad)) 239 240 w_grad = grad_debugger.gradient_tensor(self.w) 241 self.assertIsInstance(w_grad, ops.Tensor) 242 self.assertAllClose(1.0, self.sess.run(w_grad)) 243 244 w_grad = grad_debugger.gradient_tensor("w:0") 245 self.assertIsInstance(w_grad, ops.Tensor) 246 self.assertAllClose(1.0, self.sess.run(w_grad)) 247 248 def testWatchGradientsWorksOnRefTensor(self): 249 y = math_ops.add(self.w, -1.0, name="y") 250 251 grad_debugger = debug_gradients.GradientsDebugger() 252 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "u:0$"): 253 grads = gradients_impl.gradients(y, [self.u, self.v]) 254 self.assertEqual(2, len(grads)) 255 u_grad = grads[0] 256 v_grad = grads[1] 257 258 self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0")) 259 260 self.sess.run(variables.global_variables_initializer()) 261 self.assertAllClose(3.0, self.sess.run(u_grad)) 262 self.assertAllClose(2.0, self.sess.run(v_grad)) 263 self.assertAllClose(3.0, self.sess.run( 264 grad_debugger.gradient_tensor("u:0"))) 265 266 def testWatchGradientsWorksOnMultipleTensors(self): 267 y = math_ops.add(self.w, -1.0, name="y") 268 269 grad_debugger = debug_gradients.GradientsDebugger() 270 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, 271 "(u|w):0$"): 272 grads = gradients_impl.gradients(y, [self.u, self.v]) 273 self.assertEqual(2, len(grads)) 274 u_grad = grads[0] 275 276 self.assertEqual(2, len(grad_debugger.gradient_tensors())) 277 self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0")) 278 self.assertIsInstance(grad_debugger.gradient_tensor("w:0"), ops.Tensor) 279 280 self.sess.run(variables.global_variables_initializer()) 281 self.assertAllClose(1.0, self.sess.run( 282 grad_debugger.gradient_tensor("w:0"))) 283 self.assertAllClose(3.0, self.sess.run( 284 grad_debugger.gradient_tensor("u:0"))) 285 286 def testWatchGradientsByXTensorsWorks(self): 287 y = math_ops.add(self.w, -1.0, name="foo/y") 288 z = math_ops.square(y, name="foo/z") 289 290 # The constructrion of the forward graph has completed. 291 # But we can still get the gradient tensors by using 292 # watch_gradients_by_x_tensors(). 293 grad_debugger = debug_gradients.GradientsDebugger() 294 with grad_debugger.watch_gradients_by_tensors(self.sess.graph, 295 [self.w, self.u, y]): 296 gradient_descent.GradientDescentOptimizer(0.1).minimize(z) 297 298 self.assertEqual(3, len(grad_debugger.gradient_tensors())) 299 u_grad = grad_debugger.gradient_tensor(self.u) 300 w_grad = grad_debugger.gradient_tensor(self.w) 301 y_grad = grad_debugger.gradient_tensor(y) 302 303 self.sess.run(variables.global_variables_initializer()) 304 self.assertAllClose(10.0, self.sess.run(y_grad)) 305 self.assertAllClose(10.0, self.sess.run(w_grad)) 306 self.assertAllClose(30.0, self.sess.run(u_grad)) 307 308 def testWatchGradientsByTensorCanWorkOnMultipleLosses(self): 309 y = math_ops.add(self.w, -1.0, name="y") 310 z1 = math_ops.square(y, name="z1") 311 z2 = math_ops.sqrt(y, name="z2") 312 313 grad_debugger_1 = debug_gradients.GradientsDebugger() 314 with grad_debugger_1.watch_gradients_by_tensors(self.sess.graph, y): 315 gradient_descent.GradientDescentOptimizer(0.1).minimize(z1) 316 317 grad_debugger_2 = debug_gradients.GradientsDebugger() 318 with grad_debugger_2.watch_gradients_by_tensors(self.sess.graph, y): 319 gradient_descent.GradientDescentOptimizer(0.1).minimize(z2) 320 321 dz1_dy = grad_debugger_1.gradient_tensor(y) 322 dz2_dy = grad_debugger_2.gradient_tensor(y) 323 self.assertIsInstance(dz1_dy, ops.Tensor) 324 self.assertIsInstance(dz2_dy, ops.Tensor) 325 self.assertIsNot(dz1_dy, dz2_dy) 326 327 self.sess.run(variables.global_variables_initializer()) 328 self.assertAllClose(5.0**2, self.sess.run(z1)) 329 self.assertAllClose(5.0**0.5, self.sess.run(z2)) 330 self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) 331 self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) 332 333 def testGradientsValuesFromDumpWorks(self): 334 y = math_ops.add(self.w, -1.0, name="y") 335 z = math_ops.square(y, name="z") 336 337 grad_debugger = debug_gradients.GradientsDebugger() 338 with grad_debugger.watch_gradients_by_tensors(self.sess.graph, 339 [self.w, self.u, y]): 340 train_op = gradient_descent.GradientDescentOptimizer(0.1).minimize(z) 341 342 self.sess.run(variables.global_variables_initializer()) 343 344 run_options = config_pb2.RunOptions(output_partition_graphs=True) 345 dump_dir = tempfile.mkdtemp() 346 debug_url = "file://" + dump_dir 347 debug_utils.watch_graph(run_options, self.sess.graph, debug_urls=debug_url) 348 run_metadata = config_pb2.RunMetadata() 349 self.assertAllClose(2.0, self.sess.run(self.u)) 350 self.sess.run(train_op, options=run_options, run_metadata=run_metadata) 351 self.assertAllClose(-1.0, self.sess.run(self.u)) 352 353 dump = debug_data.DebugDumpDir( 354 dump_dir, partition_graphs=run_metadata.partition_graphs) 355 dump.set_python_graph(self.sess.graph) 356 357 y_grad_values = debug_gradients.gradient_values_from_dump( 358 grad_debugger, y, dump) 359 self.assertEqual(1, len(y_grad_values)) 360 self.assertAllClose(10.0, y_grad_values[0]) 361 362 w_grad_values = debug_gradients.gradient_values_from_dump( 363 grad_debugger, self.w, dump) 364 self.assertEqual(1, len(w_grad_values)) 365 self.assertAllClose(10.0, w_grad_values[0]) 366 367 u_grad_values = debug_gradients.gradient_values_from_dump( 368 grad_debugger, self.u, dump) 369 self.assertEqual(1, len(u_grad_values)) 370 self.assertAllClose(30.0, u_grad_values[0]) 371 372 with self.assertRaisesRegexp( 373 LookupError, 374 r"This GradientsDebugger has not received any gradient tensor for " 375 r"x-tensor v:0"): 376 debug_gradients.gradient_values_from_dump(grad_debugger, self.v, dump) 377 378 # Cleanup. 379 shutil.rmtree(dump_dir) 380 381 382 if __name__ == "__main__": 383 googletest.main() 384
