1 # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Functional test for optimizer.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 from tensorflow.python.framework import constant_op 22 from tensorflow.python.framework import dtypes 23 from tensorflow.python.framework import ops 24 from tensorflow.python.framework import test_util 25 from tensorflow.python.ops import array_ops 26 from tensorflow.python.ops import clip_ops 27 from tensorflow.python.ops import gradients_impl 28 from tensorflow.python.ops import resource_variable_ops 29 from tensorflow.python.ops import state_ops 30 from tensorflow.python.ops import variables 31 from tensorflow.python.platform import test 32 from tensorflow.python.training import gradient_descent 33 34 35 class OptimizerTest(test.TestCase): 36 37 @test_util.run_in_graph_and_eager_modes() 38 def testBasic(self): 39 for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): 40 # Note that we name the variables uniquely here since the variables don't 41 # seem to be getting deleted at the end of the loop. 42 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, 43 name='a_%d' % i) 44 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, 45 name='b_%d' % i) 46 def loss(): 47 return 5 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop 48 # Note that for eager execution, minimize expects a function instead of a 49 # Tensor. 50 global_step = resource_variable_ops.ResourceVariable( 51 array_ops.zeros([], dtypes.int64), name='global_step_%d' % i) 52 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 53 54 self.evaluate(variables.global_variables_initializer()) 55 # Fetch params to validate initial values 56 self.assertAllClose([1.0, 2.0], self.evaluate(var0)) 57 self.assertAllClose([3.0, 4.0], self.evaluate(var1)) 58 # Run 1 step of sgd through optimizer 59 opt_op = sgd_op.minimize(loss, global_step, [var0, var1]) 60 self.evaluate(opt_op) 61 # Validate updated params 62 self.assertAllClose([-14., -13.], self.evaluate(var0)) 63 self.assertAllClose([-6., -5.], self.evaluate(var1)) 64 65 def testAggregationMethod(self): 66 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 67 with self.test_session(): 68 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 69 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 70 cost = 5 * var0 + 3 * var1 71 global_step = variables.Variable( 72 array_ops.zeros([], dtypes.int64), name='global_step') 73 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 74 opt_op = sgd_op.minimize( 75 cost, 76 global_step, [var0, var1], 77 aggregation_method=gradients_impl.AggregationMethod. 78 EXPERIMENTAL_ACCUMULATE_N) 79 80 variables.global_variables_initializer().run() 81 # Fetch params to validate initial values 82 self.assertAllClose([1.0, 2.0], var0.eval()) 83 self.assertAllClose([3.0, 4.0], var1.eval()) 84 # Run 1 step of sgd through optimizer 85 opt_op.run() 86 # Validate updated params 87 self.assertAllClose([-14., -13.], var0.eval()) 88 self.assertAllClose([-6., -5.], var1.eval()) 89 90 def testPrecomputedGradient(self): 91 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 92 with self.test_session(): 93 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 94 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 95 cost = 5 * var0 + 3 * var1 96 grad_loss = constant_op.constant([42, -42], dtype=dtype) 97 global_step = variables.Variable( 98 array_ops.zeros([], dtypes.int64), name='global_step') 99 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 100 opt_op = sgd_op.minimize( 101 cost, global_step, [var0, var1], grad_loss=grad_loss) 102 103 variables.global_variables_initializer().run() 104 # Fetch params to validate initial values 105 self.assertAllClose([1.0, 2.0], var0.eval()) 106 self.assertAllClose([3.0, 4.0], var1.eval()) 107 # Run 1 step of sgd through optimizer 108 opt_op.run() 109 # Validate updated params 110 self.assertAllClose([1.0 - 3 * 5 * 42.0, 2.0 - 3 * 5 * (-42.0)], 111 var0.eval()) 112 self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)], 113 var1.eval()) 114 115 @test_util.run_in_graph_and_eager_modes() 116 def testNoVariables(self): 117 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 118 # pylint: disable=cell-var-from-loop 119 def loss(): 120 var0 = resource_variable_ops.ResourceVariable( 121 [1.0, 2.0], dtype=dtype, trainable=False, name='a') 122 var1 = resource_variable_ops.ResourceVariable( 123 [3.0, 4.0], dtype=dtype, trainable=False, name='b') 124 return 5 * var0 + var1 125 # pylint: enable=cell-var-from-loop 126 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 127 with self.assertRaisesRegexp(ValueError, 'No.*variables'): 128 sgd_op.minimize(loss) 129 130 @test_util.run_in_graph_and_eager_modes() 131 def testNoGradients(self): 132 for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): 133 # Note that we name the variables uniquely here since the variables don't 134 # seem to be getting deleted at the end of the loop. 135 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, 136 name='a%d' % i) 137 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, 138 name='b%d' % i) 139 # pylint: disable=cell-var-from-loop 140 def loss(): 141 return 5 * var0 142 # pylint: enable=cell-var-from-loop 143 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 144 with self.assertRaisesRegexp(ValueError, 'No gradients'): 145 # var1 has no gradient 146 sgd_op.minimize(loss, var_list=[var1]) 147 148 @test_util.run_in_graph_and_eager_modes() 149 def testNoGradientsForAnyVariables_Minimize(self): 150 for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): 151 # Note that we name the variables uniquely here since the variables don't 152 # seem to be getting deleted at the end of the loop. 153 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, 154 name='a_%d' % i) 155 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, 156 name='b_%d' % i) 157 def loss(): 158 return constant_op.constant(5.0) 159 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 160 with self.assertRaisesRegexp(ValueError, 161 'No gradients provided for any variable'): 162 sgd_op.minimize(loss, var_list=[var0, var1]) 163 164 @test_util.run_in_graph_and_eager_modes() 165 def testNoGradientsForAnyVariables_ApplyGradients(self): 166 for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): 167 # Note that we name the variables uniquely here since the variables don't 168 # seem to be getting deleted at the end of the loop. 169 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, 170 name='a_%d' % i) 171 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, 172 name='b_%d' % i) 173 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 174 with self.assertRaisesRegexp(ValueError, 175 'No gradients provided for any variable'): 176 sgd_op.apply_gradients([(None, var0), (None, var1)]) 177 178 @test_util.run_in_graph_and_eager_modes() 179 def testGradientsAsVariables(self): 180 for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): 181 # Note that we name the variables uniquely here since the variables don't 182 # seem to be getting deleted at the end of the loop. 183 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, 184 name='a%d' % i) 185 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, 186 name='b%d' % i) 187 def loss(): 188 return 5 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop 189 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 190 grads_and_vars = sgd_op.compute_gradients(loss, [var0, var1]) 191 # Convert gradients to tf.Variables 192 converted_grads = [ 193 resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype), 194 name='c_%d_%d' % (i, j)) 195 for j, gv in enumerate(grads_and_vars) 196 ] 197 convert_ops = [ 198 state_ops.assign(converted_grads[j], gv[0]) 199 for j, gv in enumerate(grads_and_vars) 200 ] 201 202 self.evaluate(variables.global_variables_initializer()) 203 # Run convert_ops to achieve the gradietns converting 204 self.evaluate(convert_ops) 205 # Fetch params to validate initial values 206 self.assertAllClose([1.0, 2.0], self.evaluate(var0)) 207 self.assertAllClose([3.0, 4.0], self.evaluate(var1)) 208 209 # Run 1 step of sgd through optimizer 210 converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) 211 opt_op = sgd_op.apply_gradients(converted_grads_and_vars) 212 self.evaluate(opt_op) 213 214 # Validate updated params 215 self.assertAllClose([-14., -13.], self.evaluate(var0)) 216 self.assertAllClose([-6., -5.], self.evaluate(var1)) 217 218 @test_util.run_in_graph_and_eager_modes() 219 def testComputeGradientsWithTensors(self): 220 x = ops.convert_to_tensor(1.0) 221 def f(): 222 return x * x 223 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 224 grads_and_vars = sgd_op.compute_gradients(f, [x]) 225 self.assertEqual(1, len(grads_and_vars)) 226 grad, x_as_var = grads_and_vars[0] 227 self.assertIs(x, x_as_var) 228 self.assertEqual(2.0, self.evaluate(grad)) 229 230 with self.assertRaises(NotImplementedError): 231 sgd_op.apply_gradients(grads_and_vars) 232 233 def testTrainOp(self): 234 with self.test_session(): 235 var0 = variables.Variable([1.0, 2.0]) 236 var1 = variables.Variable([3.0, 4.0]) 237 cost = 5 * var0 + 3 * var1 238 global_step = variables.Variable( 239 array_ops.zeros([], dtypes.int64), name='global_step') 240 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 241 opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) 242 self.assertTrue(opt_op in ops.get_collection(ops.GraphKeys.TRAIN_OP)) 243 244 def testConstraint(self): 245 constraint_01 = lambda x: clip_ops.clip_by_value(x, -0.1, 0.) 246 constraint_0 = lambda x: clip_ops.clip_by_value(x, 0., 1.) 247 with self.test_session(): 248 var0 = variables.Variable([1.0, 2.0], 249 constraint=constraint_01) 250 var1 = variables.Variable([3.0, 4.0], 251 constraint=constraint_0) 252 cost = 5 * var0 + 3 * var1 253 global_step = variables.Variable( 254 array_ops.zeros([], dtypes.int64), name='global_step') 255 sgd_op = gradient_descent.GradientDescentOptimizer(3.0) 256 opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) 257 258 variables.global_variables_initializer().run() 259 # Fetch params to validate initial values 260 self.assertAllClose([1.0, 2.0], var0.eval()) 261 self.assertAllClose([3.0, 4.0], var1.eval()) 262 # Run 1 step of sgd through optimizer 263 opt_op.run() 264 # Validate updated params 265 self.assertAllClose([-0.1, -0.1], var0.eval()) 266 self.assertAllClose([0., 0.], var1.eval()) 267 268 269 if __name__ == '__main__': 270 test.main() 271
