1 # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Functional test for moving_averages.py.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 from tensorflow.python.framework import constant_op 22 from tensorflow.python.framework import dtypes 23 from tensorflow.python.framework import ops 24 from tensorflow.python.ops import array_ops 25 from tensorflow.python.ops import gen_state_ops 26 from tensorflow.python.ops import variable_scope 27 from tensorflow.python.ops import variables 28 from tensorflow.python.platform import test 29 from tensorflow.python.training import moving_averages 30 from tensorflow.python.training import saver as saver_lib 31 32 33 class MovingAveragesTest(test.TestCase): 34 35 def testAssignMovingAverageWithoutZeroDebias(self): 36 with self.test_session(): 37 var = variables.Variable([10.0, 11.0]) 38 val = constant_op.constant([1.0, 2.0], dtypes.float32) 39 decay = 0.25 40 assign = moving_averages.assign_moving_average( 41 var, val, decay, zero_debias=False) 42 variables.global_variables_initializer().run() 43 self.assertAllClose([10.0, 11.0], var.eval()) 44 assign.op.run() 45 self.assertAllClose( 46 [10.0 * 0.25 + 1.0 * (1.0 - 0.25), 11.0 * 0.25 + 2.0 * (1.0 - 0.25)], 47 var.eval()) 48 49 def testAssignMovingAverage(self): 50 with self.test_session(): 51 var = variables.Variable([0.0, 0.0]) 52 val = constant_op.constant([1.0, 2.0], dtypes.float32) 53 decay = 0.25 54 assign = moving_averages.assign_moving_average(var, val, decay) 55 variables.global_variables_initializer().run() 56 self.assertAllClose([0.0, 0.0], var.eval()) 57 assign.op.run() 58 self.assertAllClose([ 59 1.0 * (1.0 - 0.25) / (1 - 0.25), 2.0 * (1.0 - 0.25) / (1 - 0.25) 60 ], var.eval()) 61 62 def testAssignMovingAverageNewNamingMultipleCalls(self): 63 with variable_scope.variable_scope("scope1") as vs1: 64 with variable_scope.variable_scope("scope2"): 65 var = variables.Variable(1.0, name="Var") 66 moving_averages.assign_moving_average(var, 0.0, 0.99) 67 moving_averages.assign_moving_average(var, 0.0, 0.99) 68 expected_names = ["scope1/scope2/Var:0", 69 "scope1/scope2/scope1/scope2/Var/biased:0", 70 "scope1/scope2/scope1/scope2/Var/local_step:0", 71 "scope1/scope2/scope1/scope2/Var/biased_1:0", 72 "scope1/scope2/scope1/scope2/Var/local_step_1:0"] 73 actual_names = [v.name for v in vs1.global_variables()] 74 self.assertSetEqual(set(expected_names), set(actual_names)) 75 76 def testAssignMovingAverageNewNamingMultipleCallsWithReuse(self): 77 with variable_scope.variable_scope("scope1") as vs1: 78 var = variable_scope.get_variable("Var", shape=[]) 79 moving_averages.assign_moving_average(var, 0.0, 0.99) 80 moving_averages.assign_moving_average(var, 0.0, 0.99) 81 with variable_scope.variable_scope(vs1, reuse=True): 82 var = variable_scope.get_variable("Var", shape=[]) 83 moving_averages.assign_moving_average(var, 0.0, 0.99) 84 moving_averages.assign_moving_average(var, 0.0, 0.99) 85 86 def testWeightedMovingAverage(self): 87 with self.test_session() as sess: 88 decay = 0.5 89 weight = array_ops.placeholder(dtypes.float32, []) 90 val = array_ops.placeholder(dtypes.float32, []) 91 92 wma = moving_averages.weighted_moving_average(val, decay, weight) 93 variables.global_variables_initializer().run() 94 95 # Get the first weighted moving average. 96 val_1 = 3.0 97 weight_1 = 4.0 98 wma_array = sess.run(wma, feed_dict={val: val_1, weight: weight_1}) 99 numerator_1 = val_1 * weight_1 * (1.0 - decay) 100 denominator_1 = weight_1 * (1.0 - decay) 101 self.assertAllClose(numerator_1 / denominator_1, wma_array) 102 103 # Get the second weighted moving average. 104 val_2 = 11.0 105 weight_2 = 22.0 106 wma_array = sess.run(wma, feed_dict={val: val_2, weight: weight_2}) 107 numerator_2 = numerator_1 * decay + val_2 * weight_2 * (1.0 - decay) 108 denominator_2 = denominator_1 * decay + weight_2 * (1.0 - decay) 109 self.assertAllClose(numerator_2 / denominator_2, wma_array) 110 111 112 def _Repeat(value, dim): 113 if dim == 1: 114 return value 115 return [value] * dim 116 117 118 class ExponentialMovingAverageTest(test.TestCase): 119 120 def _CheckDecay(self, ema, actual_decay, dim): 121 122 def _Scale(dk, steps): 123 if ema._zero_debias: 124 return 1 - dk**steps 125 else: 126 return 1 127 128 tens = _Repeat(10.0, dim) 129 thirties = _Repeat(30.0, dim) 130 var0 = variables.Variable(tens, name="v0") 131 var1 = variables.Variable(thirties, name="v1") 132 variables.global_variables_initializer().run() 133 # Note that tensor2 is not a Variable but just a plain Tensor resulting 134 # from the sum operation. 135 tensor2 = var0 + var1 136 update = ema.apply([var0, var1, tensor2]) 137 avg0 = ema.average(var0) 138 avg1 = ema.average(var1) 139 avg2 = ema.average(tensor2) 140 141 self.assertItemsEqual([var0, var1], variables.moving_average_variables()) 142 143 self.assertFalse(avg0 in variables.trainable_variables()) 144 self.assertFalse(avg1 in variables.trainable_variables()) 145 self.assertFalse(avg2 in variables.trainable_variables()) 146 variables.global_variables_initializer().run() 147 148 self.assertEqual("v0/ExponentialMovingAverage:0", avg0.name) 149 self.assertEqual("v1/ExponentialMovingAverage:0", avg1.name) 150 self.assertEqual("add/ExponentialMovingAverage:0", avg2.name) 151 152 # Check initial values. 153 self.assertAllClose(tens, var0.eval()) 154 self.assertAllClose(thirties, var1.eval()) 155 self.assertAllClose(_Repeat(10.0 + 30.0, dim), tensor2.eval()) 156 157 # Check that averages are initialized correctly. 158 self.assertAllClose(tens, avg0.eval()) 159 self.assertAllClose(thirties, avg1.eval()) 160 # Note that averages of Tensor's initialize to zeros_like since no value 161 # of the Tensor is known because the Op has not been run (yet). 162 self.assertAllClose(_Repeat(0.0, dim), avg2.eval()) 163 164 # Update the averages and check. 165 update.run() 166 dk = actual_decay 167 168 expected = _Repeat(10.0 * dk + 10.0 * (1 - dk), dim) 169 self.assertAllClose(expected, avg0.eval()) 170 expected = _Repeat(30.0 * dk + 30.0 * (1 - dk), dim) 171 self.assertAllClose(expected, avg1.eval()) 172 expected = _Repeat(0.0 * dk + (10.0 + 30.0) * (1 - dk) / _Scale(dk, 1), dim) 173 self.assertAllClose(expected, avg2.eval()) 174 175 # Again, update the averages and check. 176 update.run() 177 expected = _Repeat((10.0 * dk + 10.0 * (1 - dk)) * dk + 10.0 * (1 - dk), 178 dim) 179 self.assertAllClose(expected, avg0.eval()) 180 expected = _Repeat((30.0 * dk + 30.0 * (1 - dk)) * dk + 30.0 * (1 - dk), 181 dim) 182 self.assertAllClose(expected, avg1.eval()) 183 expected = _Repeat(((0.0 * dk + (10.0 + 30.0) * (1 - dk)) * dk + 184 (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim) 185 self.assertAllClose(expected, avg2.eval()) 186 187 def testAverageVariablesNoNumUpdates_Scalar(self): 188 with self.test_session(): 189 ema = moving_averages.ExponentialMovingAverage(0.25) 190 self._CheckDecay(ema, actual_decay=0.25, dim=1) 191 192 def testAverageVariablesNoNumUpdates_Scalar_Debias(self): 193 with self.test_session(): 194 ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True) 195 self._CheckDecay(ema, actual_decay=0.25, dim=1) 196 197 def testAverageVariablesNoNumUpdates_Vector(self): 198 with self.test_session(): 199 ema = moving_averages.ExponentialMovingAverage(0.25) 200 self._CheckDecay(ema, actual_decay=0.25, dim=5) 201 202 def testAverageVariablesNoNumUpdates_Vector_Debias(self): 203 with self.test_session(): 204 ema = moving_averages.ExponentialMovingAverage(0.25, zero_debias=True) 205 self._CheckDecay(ema, actual_decay=0.25, dim=5) 206 207 def testAverageVariablesNumUpdates_Scalar(self): 208 with self.test_session(): 209 # With num_updates 1, the decay applied is 0.1818 210 ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1) 211 self._CheckDecay(ema, actual_decay=0.181818, dim=1) 212 213 def testAverageVariablesNumUpdates_Scalar_Debias(self): 214 with self.test_session(): 215 # With num_updates 1, the decay applied is 0.1818 216 ema = moving_averages.ExponentialMovingAverage( 217 0.25, num_updates=1, zero_debias=True) 218 self._CheckDecay(ema, actual_decay=0.181818, dim=1) 219 220 def testAverageVariablesNumUpdates_Vector(self): 221 with self.test_session(): 222 # With num_updates 1, the decay applied is 0.1818 223 ema = moving_averages.ExponentialMovingAverage(0.25, num_updates=1) 224 self._CheckDecay(ema, actual_decay=0.181818, dim=5) 225 226 def testAverageVariablesNumUpdates_Vector_Debias(self): 227 with self.test_session(): 228 # With num_updates 1, the decay applied is 0.1818 229 ema = moving_averages.ExponentialMovingAverage( 230 0.25, num_updates=1, zero_debias=True) 231 self._CheckDecay(ema, actual_decay=0.181818, dim=5) 232 233 def testAverageVariablesWithControlDeps(self): 234 with self.test_session() as sess: 235 v0 = variables.Variable(0, name="v0") 236 add_to_v0 = v0.assign_add(1) 237 v1 = variables.Variable([10.0], name="v1") 238 assign_to_v1 = v1.assign([20.0]) 239 ema = moving_averages.ExponentialMovingAverage(0.25) 240 with ops.control_dependencies([add_to_v0]): 241 ema_op = ema.apply([v1]) 242 # the moving average of v1 should not have any control inputs 243 v1_avg = ema.average(v1) 244 self.assertEqual([], v1_avg.initializer.control_inputs) 245 self.assertEqual([], v1_avg.value().op.control_inputs) 246 self.assertEqual([], v1_avg.value().op.control_inputs) 247 # We should be able to initialize v1_avg before v0. 248 sess.run(v1_avg.initializer) 249 sess.run(v0.initializer) 250 self.assertEqual([10.0], sess.run(v1_avg)) 251 # running ema_op should add to v0 (in addition to updating v1_avg) 252 sess.run(assign_to_v1) 253 sess.run(ema_op) 254 self.assertEqual(1, sess.run(v0)) 255 self.assertEqual([17.5], sess.run(v1_avg)) 256 257 def averageVariablesNamesHelper(self, zero_debias): 258 with self.test_session(): 259 v0 = variables.Variable(10.0, name="v0") 260 v1 = variables.Variable(30.0, name="v1") 261 # Add a non-trainable variable. 262 v2 = variables.Variable(20.0, name="v2", trainable=False) 263 tensor2 = v0 + v1 264 ema = moving_averages.ExponentialMovingAverage( 265 0.25, zero_debias=zero_debias, name="foo") 266 self.assertEqual("v0/foo", ema.average_name(v0)) 267 self.assertEqual("v1/foo", ema.average_name(v1)) 268 self.assertEqual("add/foo", ema.average_name(tensor2)) 269 ema.apply([v0, v1, tensor2]) 270 vars_to_restore = ema.variables_to_restore() 271 # vars_to_restore should contain the following: 272 # {v0/foo : v0, 273 # v1/foo : v1, 274 # add/foo : add/foo, 275 # v2 : v2} 276 expected_names = [ 277 ema.average_name(v0), ema.average_name(v1), ema.average_name(tensor2), 278 v2.op.name 279 ] 280 if zero_debias: 281 # vars_to_restore should also contain the following: 282 # {add/foo/biased: add/foo/biased, 283 # add/foo/local_step: add/foo/local_step} 284 expected_names += [ 285 ema.average_name(tensor2) + "/biased", 286 ema.average_name(tensor2) + "/local_step" 287 ] 288 self.assertEqual(sorted(expected_names), sorted(vars_to_restore.keys())) 289 self.assertEqual(ema.average(v0).op.name, ema.average_name(v0)) 290 self.assertEqual(ema.average(v1).op.name, ema.average_name(v1)) 291 self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2)) 292 293 def testAverageVariablesNames(self): 294 self.averageVariablesNamesHelper(zero_debias=True) 295 296 def testAverageVariablesNamesNoDebias(self): 297 self.averageVariablesNamesHelper(zero_debias=False) 298 299 def averageVariablesNamesRespectScopeHelper(self, zero_debias): 300 # See discussion on #2740. 301 with self.test_session(): 302 with variable_scope.variable_scope("scope1"): 303 v0 = variables.Variable(10.0, name="v0") 304 v1 = variables.Variable(30.0, name="v1") 305 # Add a non-trainable variable. 306 v2 = variables.Variable(20.0, name="v2", trainable=False) 307 tensor2 = v0 + v1 308 with variable_scope.variable_scope("scope2"): 309 ema = moving_averages.ExponentialMovingAverage( 310 0.25, zero_debias=zero_debias, name="foo") 311 self.assertEqual("scope2/scope1/v0/foo", ema.average_name(v0)) 312 self.assertEqual("scope2/scope1/v1/foo", ema.average_name(v1)) 313 self.assertEqual("scope2/scope1/add/foo", ema.average_name(tensor2)) 314 ema.apply([v0, v1, tensor2]) 315 vars_to_restore = ema.variables_to_restore() 316 # `vars_to_restore` should contain the following: 317 # {scope2/scope1/v0/foo : v0, 318 # scope2/scope1/v1/foo : v1, 319 # scope2/scope1/add/foo : add/foo, 320 # scope1/v2 : v2} 321 expected_names = [ 322 ema.average_name(v0), ema.average_name(v1), 323 ema.average_name(tensor2), v2.op.name 324 ] 325 if zero_debias: 326 # `vars_to_restore` should also contain the following: 327 # {scope2/scope2/scope1/add/foo/biased: add/foo/biased, 328 # scope2/scope2/scope1/add/foo/local_step: add/foo/local_step} 329 sc = "scope2/" 330 expected_names += [ 331 sc + ema.average_name(tensor2) + "/biased", 332 sc + ema.average_name(tensor2) + "/local_step" 333 ] 334 335 self.assertEqual(sorted(expected_names), sorted(vars_to_restore.keys())) 336 self.assertEqual(ema.average(v0).op.name, ema.average_name(v0)) 337 self.assertEqual(ema.average(v1).op.name, ema.average_name(v1)) 338 self.assertEqual( 339 ema.average(tensor2).op.name, ema.average_name(tensor2)) 340 341 def testAverageVariablesNamesRespectScope(self): 342 self.averageVariablesNamesRespectScopeHelper(zero_debias=True) 343 344 def testAverageVariablesNamesRespectScopeNoDebias(self): 345 self.averageVariablesNamesRespectScopeHelper(zero_debias=False) 346 347 def testSubsetAverageVariablesNames(self): 348 with self.test_session(): 349 v0 = variables.Variable(10.0, name="v0") 350 v1 = variables.Variable(30.0, name="v1") 351 # Add a non-trainable variable. 352 v2 = variables.Variable(20.0, name="v2", trainable=False) 353 tensor2 = v0 + v1 354 ema = moving_averages.ExponentialMovingAverage(0.25, name="foo_avg") 355 self.assertEqual("v0/foo_avg", ema.average_name(v0)) 356 self.assertEqual("v1/foo_avg", ema.average_name(v1)) 357 self.assertEqual("add/foo_avg", ema.average_name(tensor2)) 358 vars_to_restore = ema.variables_to_restore([v0, tensor2]) 359 # vars_to_restore should contain the following: 360 # {v0/foo_avg : v0, 361 # add/foo_avg : add 362 # v1 : v1, 363 # v2 : v2} 364 self.assertEqual( 365 sorted(vars_to_restore.keys()), 366 sorted([ 367 ema.average_name(v0), ema.average_name(tensor2), v1.op.name, 368 v2.op.name 369 ])) 370 ema.apply([v0, v1, tensor2]) 371 self.assertEqual(ema.average(v0).op.name, ema.average_name(v0)) 372 self.assertEqual(ema.average(v1).op.name, ema.average_name(v1)) 373 self.assertEqual(ema.average(tensor2).op.name, ema.average_name(tensor2)) 374 375 def testAverageVariablesDeviceAssignment(self): 376 with ops.device("/job:dev_v0"): 377 v0 = variables.Variable(10.0, name="v0") 378 with ops.device("/job:dev_v1"): 379 v1 = gen_state_ops._variable( 380 shape=[1], 381 dtype=dtypes.float32, 382 name="v1", 383 container="", 384 shared_name="") 385 v1.set_shape([1]) 386 tensor2 = v0 + v1 387 ema = moving_averages.ExponentialMovingAverage(0.25, name="foo_avg") 388 with ops.device("/job:default"): 389 ema.apply([v0, v1, tensor2]) 390 self.assertDeviceEqual("/job:dev_v0", ema.average(v0).device) 391 self.assertDeviceEqual("/job:dev_v1", ema.average(v1).device) 392 # However, the colocation property is maintained. 393 self.assertEqual([b"loc:@v1"], ema.average(v1).op.colocation_groups()) 394 self.assertDeviceEqual("/job:default", ema.average(tensor2).device) 395 396 def _ExportAndImportGraph(self, graph): 397 """Export and import graph into a new graph.""" 398 meta_graph = saver_lib.export_meta_graph( 399 graph=graph, collection_list=graph.get_all_collection_keys()) 400 graph_copy = ops.Graph() 401 with graph_copy.as_default(): 402 _ = saver_lib.import_meta_graph(meta_graph) 403 return graph_copy 404 405 def testImportedGraphVariablesToRestore(self): 406 g = ops.Graph() 407 with g.as_default(): 408 variables.Variable(10.0, name="v") 409 # Export and import the graph into a new graph. 410 g_copy = self._ExportAndImportGraph(g) 411 with g_copy.as_default(): 412 ema = moving_averages.ExponentialMovingAverage(0.25, name="foo_avg") 413 vars_to_restore = ema.variables_to_restore() 414 # There should only be one variable in vars_to_restore. This is important 415 # to check because when importing from a GraphDef, TF makes duplicate 416 # python Variable objects referring to the same underlying variable. We 417 # need to be sure that two variables referring to the same variable don't 418 # both get added to vars_to_restore. 419 self.assertEqual(len(vars_to_restore), 1) 420 self.assertTrue("v/foo_avg" in vars_to_restore) 421 422 423 if __name__ == "__main__": 424 test.main() 425