1 # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 # ============================================================================== 15 """Forward-mode derivatives.""" 16 17 from __future__ import absolute_import 18 from __future__ import division 19 from __future__ import print_function 20 21 from tensorflow.python.framework import ops 22 from tensorflow.python.ops import array_ops 23 from tensorflow.python.ops import control_flow_ops 24 from tensorflow.python.ops.gradients_impl import gradients 25 26 27 def fwd_gradients(ys, xs, grad_xs=None, assert_unused=False): 28 """Computes forward-mode derivatives. 29 30 This is accomplished in pure-python using tensorflow's existing (reverse-mode) 31 gradients. There is additional overhead on graph construction, but runtime 32 performance should be equal to a manual implementation [citation needed]. 33 34 See https://j-towns.github.io/2017/06/12/A-new-trick.html and 35 https://github.com/HIPS/autograd/pull/175 for the original discussion of this 36 method, and https://github.com/renmengye/tensorflow-forward-ad for a "direct" 37 implementation. 38 39 Args: 40 ys: A list of tensors. 41 xs: A list of tensors. 42 grad_xs: An optional list of tensors. If provided, must have the same length 43 and shapes compatible with xs. 44 assert_unused: Add assertions that intermediate values are not computed. 45 Returns: 46 A list of tensors of the same shapes as ys. The directional derivatives of 47 ys with respect to xs in the direction grad_xs. Leaving grad_xs unspecified 48 is equivalent to passing in 1s for each x in xs. 49 """ 50 # This version of forward-mode autodiff is based on code by Tim Cooijmans 51 # and handles list arguments and certain special cases such as when the 52 # ys doesn't depend on one or more of the xs, and when tf.IndexedSlices are 53 # generated by the first tf.gradients call. 54 55 us = [array_ops.zeros_like(y) + float('nan') for y in ys] 56 57 dydxs = gradients(ys, xs, grad_ys=us) 58 59 # deal with strange types that tf.gradients returns but can't deal with 60 dydxs = [ops.convert_to_tensor(dydx) if isinstance(dydx, ops.IndexedSlices) 61 else dydx for dydx in dydxs] 62 63 if assert_unused: 64 with ops.control_dependencies(dydxs): 65 assert_unused = control_flow_ops.Assert(False, [1], name='fwd_gradients') 66 with ops.control_dependencies([assert_unused]): 67 dydxs = array_ops.identity_n(dydxs) 68 69 dydxs = [array_ops.zeros_like(x) if dydx is None else dydx 70 for x, dydx in zip(xs, dydxs)] 71 for x, dydx in zip(xs, dydxs): 72 dydx.set_shape(x.shape) 73 74 dysdx = gradients(dydxs, us, grad_ys=grad_xs) 75 76 return dysdx 77
