1 # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Tests for tensorflow.learning.training_ops.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 import itertools 22 23 import numpy as np 24 25 from tensorflow.python.framework import constant_op 26 from tensorflow.python.framework import dtypes 27 from tensorflow.python.framework.test_util import TensorFlowTestCase 28 # Import resource_variable_ops for the variables-to-tensor implicit conversion. 29 from tensorflow.python.ops import resource_variable_ops # pylint: disable=unused-import 30 from tensorflow.python.ops import variables 31 from tensorflow.python.platform import googletest 32 from tensorflow.python.training import training_ops 33 34 35 class TrainingOpsTest(TensorFlowTestCase): 36 37 def _toType(self, dtype): 38 if dtype == np.float16: 39 return dtypes.float16 40 elif dtype == np.float32: 41 return dtypes.float32 42 elif dtype == np.float64: 43 return dtypes.float64 44 elif dtype == np.int32: 45 return dtypes.int32 46 elif dtype == np.int64: 47 return dtypes.int64 48 else: 49 assert False, (dtype) 50 51 def _testTypes(self, x, alpha, delta, use_gpu=None): 52 self.setUp() 53 with self.test_session(use_gpu=use_gpu): 54 var = variables.Variable(x) 55 variables.global_variables_initializer().run() 56 self.assertAllCloseAccordingToType(x, var.eval()) 57 apply_sgd = training_ops.apply_gradient_descent(var, alpha, delta) 58 out = apply_sgd.eval() 59 self.assertShapeEqual(out, apply_sgd) 60 self.assertAllCloseAccordingToType(x - alpha * delta, out) 61 62 def testApplyGradientDescent(self): 63 for (dtype, use_gpu) in itertools.product( 64 [np.float16, np.float32, np.float64], [False, True]): 65 x = np.arange(100).astype(dtype) 66 alpha = np.array(2.0).astype(dtype) 67 delta = np.arange(100).astype(dtype) 68 self._testTypes(x, alpha, delta, use_gpu) 69 70 def _testTypesForAdagrad(self, x, y, lr, grad, use_gpu=None): 71 self.setUp() 72 with self.test_session(use_gpu=use_gpu): 73 var = variables.Variable(x) 74 accum = variables.Variable(y) 75 variables.global_variables_initializer().run() 76 77 self.assertAllCloseAccordingToType(x, var.eval()) 78 apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad) 79 out = apply_adagrad.eval() 80 self.assertShapeEqual(out, apply_adagrad) 81 self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)** 82 (-0.5), out) 83 self.assertAllCloseAccordingToType(y + grad * grad, accum.eval()) 84 85 def _testTypesForFtrl(self, 86 x, 87 y, 88 z, 89 lr, 90 grad, 91 use_gpu=None, 92 l1=0.0, 93 l2=0.0, 94 lr_power=-0.5): 95 self.setUp() 96 with self.test_session(use_gpu=use_gpu): 97 var = variables.Variable(x) 98 accum = variables.Variable(y) 99 linear = variables.Variable(z) 100 variables.global_variables_initializer().run() 101 102 self.assertAllCloseAccordingToType(x, var.eval()) 103 apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2, 104 lr_power) 105 out = apply_ftrl.eval() 106 self.assertShapeEqual(out, apply_ftrl) 107 accum_update = y + grad * grad 108 linear_update = z + grad - (accum_update**(-lr_power) - y** 109 (-lr_power)) / lr * x 110 quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2 111 expected_out = np.array([( 112 np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if 113 np.abs(linear_update[i]) > l1 else 0.0 114 for i in range(linear_update.size)]) 115 self.assertAllCloseAccordingToType(accum_update, accum.eval()) 116 if x.dtype == np.float16: 117 # The calculations here really are not very precise in float16. 118 self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2) 119 self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2) 120 elif x.dtype == np.float32: 121 # The calculations here not sufficiently precise in float32. 122 self.assertAllClose(linear_update, linear.eval(), rtol=1e-5, atol=1e-5) 123 self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5) 124 else: 125 self.assertAllClose(linear_update, linear.eval()) 126 self.assertAllClose(expected_out, out) 127 128 def testApplyAdagrad(self): 129 for (dtype, use_gpu) in itertools.product( 130 [np.float16, np.float32, np.float64], [False, True]): 131 x = np.arange(100).astype(dtype) 132 y = np.arange(1, 101).astype(dtype) 133 lr = np.array(2.0).astype(dtype) 134 grad = np.arange(100).astype(dtype) 135 self._testTypesForAdagrad(x, y, lr, grad, use_gpu) 136 137 def testApplyFtrl(self): 138 for dtype in [np.float16, np.float32, np.float64]: 139 x = np.arange(100).astype(dtype) 140 y = np.arange(1, 101).astype(dtype) 141 z = np.arange(102, 202).astype(dtype) 142 lr = np.array(2.0).astype(dtype) 143 l1 = np.array(3.0).astype(dtype) 144 l2 = np.array(4.0).astype(dtype) 145 grad = np.arange(100).astype(dtype) 146 self._testTypesForFtrl(x, y, z, lr, grad, use_gpu=False, l1=l1, l2=l2) 147 148 def _testTypesForSparseAdagrad(self, x, y, lr, grad, indices): 149 self.setUp() 150 with self.test_session(use_gpu=False): 151 var = variables.Variable(x) 152 accum = variables.Variable(y) 153 variables.global_variables_initializer().run() 154 155 self.assertAllCloseAccordingToType(x, var.eval()) 156 sparse_apply_adagrad = training_ops.sparse_apply_adagrad( 157 var, accum, lr, grad, 158 constant_op.constant(indices, self._toType(indices.dtype))) 159 out = sparse_apply_adagrad.eval() 160 self.assertShapeEqual(out, sparse_apply_adagrad) 161 162 for (i, index) in enumerate(indices): 163 self.assertAllCloseAccordingToType( 164 x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])**(-0.5), 165 var.eval()[index]) 166 self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i], 167 accum.eval()[index]) 168 169 def _testTypesForSparseFtrl(self, 170 x, 171 y, 172 z, 173 lr, 174 grad, 175 indices, 176 l1=0.0, 177 l2=0.0, 178 lr_power=-0.5): 179 self.setUp() 180 with self.test_session(use_gpu=False): 181 var = variables.Variable(x) 182 accum = variables.Variable(y) 183 linear = variables.Variable(z) 184 variables.global_variables_initializer().run() 185 186 self.assertAllCloseAccordingToType(x, var.eval()) 187 sparse_apply_ftrl = training_ops.sparse_apply_ftrl( 188 var, 189 accum, 190 linear, 191 grad, 192 constant_op.constant(indices, self._toType(indices.dtype)), 193 lr, 194 l1, 195 l2, 196 lr_power=lr_power) 197 out = sparse_apply_ftrl.eval() 198 self.assertShapeEqual(out, sparse_apply_ftrl) 199 200 for (i, index) in enumerate(indices): 201 self.assertAllCloseAccordingToType(x[index] - lr * grad[i] * 202 (y[index] + grad[i] * grad[i])** 203 (lr_power), var.eval()[index]) 204 self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i], 205 accum.eval()[index]) 206 207 def testSparseApplyAdagrad(self): 208 for (dtype, index_type) in itertools.product( 209 [np.float16, np.float32, np.float64], [np.int32, np.int64]): 210 x_val = [np.arange(10), np.arange(10, 20), np.arange(20, 30)] 211 y_val = [np.arange(1, 11), np.arange(11, 21), np.arange(21, 31)] 212 x = np.array(x_val).astype(dtype) 213 y = np.array(y_val).astype(dtype) 214 lr = np.array(2.0).astype(dtype) 215 grad_val = [np.arange(10), np.arange(10)] 216 grad = np.array(grad_val).astype(dtype) 217 indices = np.array([0, 2]).astype(index_type) 218 self._testTypesForSparseAdagrad(x, y, lr, grad, indices) 219 220 def testSparseApplyAdagradDim1(self): 221 for (dtype, index_type) in itertools.product( 222 [np.float16, np.float32, np.float64], [np.int32, np.int64]): 223 x_val = [[1.0], [2.0], [3.0]] 224 y_val = [[4.0], [5.0], [6.0]] 225 x = np.array(x_val).astype(dtype) 226 y = np.array(y_val).astype(dtype) 227 lr = np.array(2.0).astype(dtype) 228 grad_val = [[1.5], [2.5]] 229 grad = np.array(grad_val).astype(dtype) 230 indices = np.array([0, 2]).astype(index_type) 231 self._testTypesForSparseAdagrad(x, y, lr, grad, indices) 232 233 def testSparseApplyFtrlDim1(self): 234 for (dtype, index_type) in itertools.product( 235 [np.float16, np.float32, np.float64], [np.int32, np.int64]): 236 x_val = [[0.0], [0.0], [0.0]] 237 y_val = [[4.0], [5.0], [6.0]] 238 z_val = [[0.0], [0.0], [0.0]] 239 x = np.array(x_val).astype(dtype) 240 y = np.array(y_val).astype(dtype) 241 z = np.array(z_val).astype(dtype) 242 lr = np.array(2.0).astype(dtype) 243 grad_val = [[1.5], [2.5]] 244 grad = np.array(grad_val).astype(dtype) 245 indices = np.array([0, 2]).astype(index_type) 246 self._testTypesForSparseFtrl(x, y, z, lr, grad, indices) 247 248 def testApplyAdam(self): 249 for dtype, use_gpu in itertools.product( 250 [np.float16, np.float32, np.float64], [False, True]): 251 var = np.arange(100).astype(dtype) 252 m = np.arange(1, 101).astype(dtype) 253 v = np.arange(101, 201).astype(dtype) 254 grad = np.arange(100).astype(dtype) 255 self._testTypesForAdam(var, m, v, grad, use_gpu) 256 257 def _testTypesForAdam(self, var, m, v, grad, use_gpu): 258 self.setUp() 259 with self.test_session(use_gpu=use_gpu): 260 var_t = variables.Variable(var) 261 m_t = variables.Variable(m) 262 v_t = variables.Variable(v) 263 264 t = 1 265 beta1 = np.array(0.9, dtype=var.dtype) 266 beta2 = np.array(0.999, dtype=var.dtype) 267 beta1_power = beta1**t 268 beta2_power = beta2**t 269 lr = np.array(0.001, dtype=var.dtype) 270 epsilon = np.array(1e-8, dtype=var.dtype) 271 beta1_t = constant_op.constant(beta1, self._toType(var.dtype), []) 272 beta2_t = constant_op.constant(beta2, self._toType(var.dtype), []) 273 beta1_power_t = variables.Variable(beta1_power) 274 beta2_power_t = variables.Variable(beta2_power) 275 lr_t = constant_op.constant(lr, self._toType(var.dtype), []) 276 epsilon_t = constant_op.constant(epsilon, self._toType(var.dtype), []) 277 variables.global_variables_initializer().run() 278 279 self.assertAllCloseAccordingToType(var, var_t.eval()) 280 new_var, _, _ = self._adamUpdateNumpy(var, grad, t, m, v, lr, beta1, 281 beta2, epsilon) 282 apply_adam = training_ops.apply_adam(var_t, m_t, v_t, beta1_power_t, 283 beta2_power_t, lr_t, beta1_t, 284 beta2_t, epsilon_t, grad) 285 out = apply_adam.eval() 286 self.assertShapeEqual(out, apply_adam) 287 self.assertAllCloseAccordingToType(new_var, out) 288 289 def _adamUpdateNumpy(self, param, g_t, t, m, v, alpha, beta1, beta2, epsilon): 290 alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t) 291 292 m_t = beta1 * m + (1 - beta1) * g_t 293 v_t = beta2 * v + (1 - beta2) * g_t * g_t 294 295 param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon) 296 return param_t, m_t, v_t 297 298 299 if __name__ == '__main__': 300 googletest.main() 301
