1 # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Functional tests for aggregate operations.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 import numpy as np 22 23 from tensorflow.python.framework import constant_op 24 from tensorflow.python.framework import dtypes 25 from tensorflow.python.framework import ops 26 from tensorflow.python.ops import embedding_ops 27 from tensorflow.python.ops import math_ops 28 from tensorflow.python.ops import resource_variable_ops 29 from tensorflow.python.ops import variable_scope 30 from tensorflow.python.ops import variables 31 from tensorflow.python.platform import test 32 from tensorflow.python.training import adagrad 33 34 35 class AdagradOptimizerTest(test.TestCase): 36 37 def doTestBasic(self, use_locking=False, use_resource=False): 38 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 39 with self.test_session(): 40 if use_resource: 41 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 42 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 43 else: 44 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 45 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 46 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 47 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 48 ada_opt = adagrad.AdagradOptimizer( 49 3.0, initial_accumulator_value=0.1, use_locking=use_locking) 50 ada_update = ada_opt.apply_gradients( 51 zip([grads0, grads1], [var0, var1])) 52 variables.global_variables_initializer().run() 53 # Fetch params to validate initial values 54 self.assertAllClose([1.0, 2.0], var0.eval()) 55 self.assertAllClose([3.0, 4.0], var1.eval()) 56 # Run 3 steps of adagrad 57 for _ in range(3): 58 ada_update.run() 59 # Validate updated params 60 self.assertAllCloseAccordingToType( 61 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) 62 self.assertAllCloseAccordingToType( 63 np.array([2.715679168701172, 3.715679168701172]), var1.eval()) 64 65 def testBasic(self): 66 self.doTestBasic(use_locking=False) 67 68 def testBasicResource(self): 69 self.doTestBasic(use_locking=False, use_resource=True) 70 71 def testBasicLocked(self): 72 self.doTestBasic(use_locking=True) 73 74 def testMinimizeSparseResourceVariable(self): 75 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 76 with self.test_session(): 77 var0 = resource_variable_ops.ResourceVariable( 78 [[1.0, 2.0], [3.0, 4.0]], dtype=dtype) 79 x = constant_op.constant([[4.0], [5.0]], dtype=dtype) 80 pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) 81 loss = pred * pred 82 sgd_op = adagrad.AdagradOptimizer(1.0).minimize(loss) 83 variables.global_variables_initializer().run() 84 # Fetch params to validate initial values 85 self.assertAllCloseAccordingToType( 86 [[1.0, 2.0], [3.0, 4.0]], var0.eval()) 87 # Run 1 step of sgd 88 sgd_op.run() 89 # Validate updated params 90 self.assertAllCloseAccordingToType( 91 [[0, 1], [3, 4]], var0.eval(), atol=0.01) 92 93 def testTensorLearningRate(self): 94 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 95 with self.test_session(): 96 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 97 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 98 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 99 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 100 ada_opt = adagrad.AdagradOptimizer( 101 constant_op.constant(3.0), initial_accumulator_value=0.1) 102 ada_update = ada_opt.apply_gradients( 103 zip([grads0, grads1], [var0, var1])) 104 variables.global_variables_initializer().run() 105 # Fetch params to validate initial values 106 self.assertAllClose([1.0, 2.0], var0.eval()) 107 self.assertAllClose([3.0, 4.0], var1.eval()) 108 # Run 3 steps of adagrad 109 for _ in range(3): 110 ada_update.run() 111 # Validate updated params 112 self.assertAllCloseAccordingToType( 113 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) 114 self.assertAllCloseAccordingToType( 115 np.array([2.715679168701172, 3.715679168701172]), var1.eval()) 116 117 def testSparseBasic(self): 118 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 119 with self.test_session(): 120 var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) 121 var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) 122 grads0 = ops.IndexedSlices( 123 constant_op.constant( 124 [0.1], shape=[1, 1], dtype=dtype), 125 constant_op.constant([0]), 126 constant_op.constant([2, 1])) 127 grads1 = ops.IndexedSlices( 128 constant_op.constant( 129 [0.01], shape=[1, 1], dtype=dtype), 130 constant_op.constant([1]), 131 constant_op.constant([2, 1])) 132 ada_opt = adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) 133 ada_update = ada_opt.apply_gradients( 134 zip([grads0, grads1], [var0, var1])) 135 variables.global_variables_initializer().run() 136 # Fetch params to validate initial values 137 self.assertAllClose([[1.0], [2.0]], var0.eval()) 138 self.assertAllClose([[3.0], [4.0]], var1.eval()) 139 # Run 3 step of sgd 140 for _ in range(3): 141 ada_update.run() 142 # Validate updated params 143 self.assertAllCloseAccordingToType( 144 np.array([[-1.6026098728179932], [2.0]]), var0.eval()) 145 self.assertAllCloseAccordingToType( 146 np.array([[3.0], [3.715679168701172]]), var1.eval()) 147 148 def testSparseRepeatedIndices(self): 149 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 150 with self.test_session(): 151 repeated_index_update_var = variables.Variable( 152 [[1.0], [2.0]], dtype=dtype) 153 aggregated_update_var = variables.Variable( 154 [[1.0], [2.0]], dtype=dtype) 155 grad_repeated_index = ops.IndexedSlices( 156 constant_op.constant( 157 [0.1, 0.1], shape=[2, 1], dtype=dtype), 158 constant_op.constant([1, 1]), 159 constant_op.constant([2, 1])) 160 grad_aggregated = ops.IndexedSlices( 161 constant_op.constant( 162 [0.2], shape=[1, 1], dtype=dtype), 163 constant_op.constant([1]), 164 constant_op.constant([2, 1])) 165 repeated_update = adagrad.AdagradOptimizer(3.0).apply_gradients( 166 [(grad_repeated_index, repeated_index_update_var)]) 167 aggregated_update = adagrad.AdagradOptimizer(3.0).apply_gradients( 168 [(grad_aggregated, aggregated_update_var)]) 169 variables.global_variables_initializer().run() 170 self.assertAllClose(aggregated_update_var.eval(), 171 repeated_index_update_var.eval()) 172 for _ in range(3): 173 repeated_update.run() 174 aggregated_update.run() 175 self.assertAllClose(aggregated_update_var.eval(), 176 repeated_index_update_var.eval()) 177 178 def testSparseRepeatedIndicesResourceVariable(self): 179 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 180 with self.test_session(): 181 var_repeated = resource_variable_ops.ResourceVariable( 182 [1.0, 2.0], dtype=dtype) 183 loss_repeated = math_ops.reduce_sum( 184 embedding_ops.embedding_lookup(var_repeated, [0, 0])) 185 var_aggregated = resource_variable_ops.ResourceVariable( 186 [1.0, 2.0], dtype=dtype) 187 loss_aggregated = 2 * math_ops.reduce_sum( 188 embedding_ops.embedding_lookup(var_aggregated, [0])) 189 update_op_repeated = adagrad.AdagradOptimizer( 190 2.0).minimize(loss_repeated) 191 update_op_aggregated = adagrad.AdagradOptimizer( 192 2.0).minimize(loss_aggregated) 193 variables.global_variables_initializer().run() 194 self.assertAllCloseAccordingToType( 195 var_repeated.eval(), var_aggregated.eval()) 196 for _ in range(3): 197 update_op_repeated.run() 198 update_op_aggregated.run() 199 self.assertAllCloseAccordingToType( 200 var_repeated.eval(), var_aggregated.eval()) 201 202 def testSparseStability(self): 203 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 204 with self.test_session(): 205 shape = [1, 6] 206 var0 = variables.Variable( 207 [[ 208 0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257, 209 -0.0105945 210 ]], 211 dtype=dtype) 212 grads0 = ops.IndexedSlices( 213 constant_op.constant( 214 [[ 215 -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05, 216 -8.4877e-05, -9.48906e-05 217 ]], 218 shape=shape, 219 dtype=dtype), 220 constant_op.constant([0]), 221 constant_op.constant(shape)) 222 ada_opt = adagrad.AdagradOptimizer(1.0, initial_accumulator_value=0.1) 223 ada_update = ada_opt.apply_gradients(zip([grads0], [var0])) 224 self.assertEqual(["accumulator"], ada_opt.get_slot_names()) 225 slot0 = ada_opt.get_slot(var0, "accumulator") 226 init = variables.global_variables_initializer() 227 for _ in range(100): 228 init.run() 229 ada_update.run() 230 self.assertAllCloseAccordingToType( 231 np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval()) 232 self.assertAllCloseAccordingToType( 233 np.array([[ 234 0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573, 235 -0.01029443 236 ]]), var0.eval()) 237 238 def testSharing(self): 239 for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: 240 with self.test_session(): 241 var0 = variables.Variable([1.0, 2.0], dtype=dtype) 242 var1 = variables.Variable([3.0, 4.0], dtype=dtype) 243 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 244 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 245 ada_opt = adagrad.AdagradOptimizer(3.0) 246 # Apply the optimizer twice. Both applications will use 247 # the same accums. 248 ada_update1 = ada_opt.apply_gradients( 249 zip([grads0, grads1], [var0, var1])) 250 ada_update2 = ada_opt.apply_gradients( 251 zip([grads0, grads1], [var0, var1])) 252 self.assertEqual(["accumulator"], ada_opt.get_slot_names()) 253 slot0 = ada_opt.get_slot(var0, "accumulator") 254 self.assertEquals(slot0.get_shape(), var0.get_shape()) 255 slot1 = ada_opt.get_slot(var1, "accumulator") 256 self.assertEquals(slot1.get_shape(), var1.get_shape()) 257 variables.global_variables_initializer().run() 258 259 # Fetch params to validate initial values. 260 self.assertAllClose([1.0, 2.0], var0.eval()) 261 self.assertAllClose([3.0, 4.0], var1.eval()) 262 # Mix the first and the second adagrad for 3 steps. 263 ada_update1.run() 264 ada_update2.run() 265 ada_update1.run() 266 # Validate updated params (the same as with only 1 Adagrad). 267 self.assertAllCloseAccordingToType( 268 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval()) 269 self.assertAllCloseAccordingToType( 270 np.array([2.715679168701172, 3.715679168701172]), var1.eval()) 271 272 def testDynamicShapeVariable_Ok(self): 273 with self.test_session(): 274 v = variable_scope.get_variable("v", initializer=constant_op.constant(1.), 275 validate_shape=False) 276 self.assertFalse(v.shape.is_fully_defined()) 277 # Creating optimizer should cause no exception. 278 adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) 279 280 281 if __name__ == "__main__": 282 test.main() 283
