1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_ 17 #define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_ 18 19 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" 20 21 namespace Eigen { 22 23 /** SpatialConvolutionBackwardInput 24 * \ingroup CXX11_NeuralNetworks_Module 25 * 26 * \brief Computes the backprop for the input of a 2D convolution. 27 * 28 * The output_backward parameter is expected to be a tensor with a rank of 3 or 29 * more (channels, height, width, and optionally others) 30 * The kernel parameter is expected to be a 4D tensor (filters, channels, 31 * kernel_height, kernel_width) 32 * The output_backward and the kernel must both be in col-major layout. The 33 * result will also be in col-major layout. 34 * 35 * If row_in_stride, col_in_stride > 1, then applies convolution with holes 36 * (aka atrous convolution), sampling every row_in_stride, col_in_stride input 37 * pixels. 38 * 39 * The result can be assigned to a tensor of rank equal to the rank of the 40 * output_backward. The dimensions of the result will be filters, height, width 41 * (and others if applicable). 42 * 43 * It is possible to swap the order of the width and height dimensions provided 44 * that the same order is used in the input, the kernel, and the output. 45 * 46 */ 47 #ifdef EIGEN_HAS_INDEX_LIST 48 typedef IndexList<type2index<0>, type2index<0>, type2index<1>, type2index<1> > 49 ReverseColMajor; 50 typedef IndexList<type2index<1>, type2index<1>, type2index<0>, type2index<0> > 51 ReverseRowMajor; 52 #else 53 typedef array<bool, 4> ReverseColMajor; 54 typedef array<bool, 4> ReverseRowMajor; 55 #endif 56 57 template <typename OutputBackward, typename Kernel> 58 EIGEN_ALWAYS_INLINE static const typename internal::conditional< 59 internal::traits<OutputBackward>::Layout == ColMajor, 60 TensorReshapingOp< 61 const DSizes<typename internal::traits<OutputBackward>::Index, 62 internal::traits<OutputBackward>::NumDimensions>, 63 const TensorContractionOp< 64 const array< 65 IndexPair<typename internal::traits<OutputBackward>::Index>, 1>, 66 const Eigen::TensorForcedEvalOp<const TensorReshapingOp< 67 const DSizes<typename internal::traits<OutputBackward>::Index, 68 2>, 69 const TensorShufflingOp< 70 const array< 71 typename internal::traits<OutputBackward>::Index, 4>, 72 const TensorReverseOp<const ReverseColMajor, 73 const Kernel> > > >, 74 const TensorReshapingOp< 75 const DSizes<typename internal::traits<OutputBackward>::Index, 76 2>, 77 const TensorImagePatchOp<Dynamic, Dynamic, 78 const OutputBackward> > > >, 79 TensorReshapingOp< 80 const DSizes<typename internal::traits<OutputBackward>::Index, 81 internal::traits<OutputBackward>::NumDimensions>, 82 const TensorContractionOp< 83 const array< 84 IndexPair<typename internal::traits<OutputBackward>::Index>, 1>, 85 const TensorReshapingOp< 86 const DSizes<typename internal::traits<OutputBackward>::Index, 87 2>, 88 const TensorImagePatchOp<Dynamic, Dynamic, 89 const OutputBackward> >, 90 const Eigen::TensorForcedEvalOp<const TensorReshapingOp< 91 const DSizes<typename internal::traits<OutputBackward>::Index, 92 2>, 93 const TensorShufflingOp< 94 const array< 95 typename internal::traits<OutputBackward>::Index, 4>, 96 const TensorReverseOp<const ReverseRowMajor, 97 const Kernel> > > > > > >::type 98 SpatialConvolutionBackwardInput( 99 const Kernel& kernel, const OutputBackward& output_backward, 100 typename internal::traits<OutputBackward>::Index inputRows, 101 typename internal::traits<OutputBackward>::Index inputCols, 102 const DenseIndex row_stride = 1, const DenseIndex col_stride = 1, 103 const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) { 104 typedef typename internal::traits<OutputBackward>::Index TensorIndex; 105 typedef typename internal::traits<OutputBackward>::Scalar OutScalar; 106 TensorRef<Tensor<typename internal::traits<Kernel>::Scalar, 107 internal::traits<Kernel>::NumDimensions, 108 internal::traits<Kernel>::Layout, TensorIndex> > 109 kern(kernel); 110 TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions, 111 internal::traits<OutputBackward>::Layout, TensorIndex> > 112 out(output_backward); 113 114 EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout == 115 internal::traits<OutputBackward>::Layout, 116 YOU_MADE_A_PROGRAMMING_MISTAKE); 117 118 static const bool isColMajor = 119 (internal::traits<OutputBackward>::Layout == ColMajor); 120 121 static const int NumDims = internal::traits<OutputBackward>::NumDimensions; 122 123 // Number of filters to apply. This is the same as the output depth of the 124 // result 125 const TensorIndex kernelFilters = 126 isColMajor ? kern.dimensions()[0] : kern.dimensions()[3]; 127 // Number of channels. This is the same as the input depth. 128 const TensorIndex kernelChannels = 129 isColMajor ? kern.dimensions()[1] : kern.dimensions()[2]; 130 const TensorIndex kernelRows = 131 isColMajor ? kern.dimensions()[2] : kern.dimensions()[1]; 132 const TensorIndex kernelCols = 133 isColMajor ? kern.dimensions()[3] : kern.dimensions()[0]; 134 135 // This is the effective kernel size, taking into account the (*_in_stride - 136 // 1) zero-values 137 // inserted between consecutive kernel elements in atrous convolution 138 const TensorIndex kernelRowsEff = 139 kernelRows + (kernelRows - 1) * (row_in_stride - 1); 140 const TensorIndex kernelColsEff = 141 kernelCols + (kernelCols - 1) * (col_in_stride - 1); 142 143 const TensorIndex outputRows = isColMajor 144 ? output_backward.dimension(1) 145 : output_backward.dimension(NumDims - 2); 146 const TensorIndex outputCols = isColMajor 147 ? output_backward.dimension(2) 148 : output_backward.dimension(NumDims - 3); 149 150 // Computing the forward padding 151 const TensorIndex forward_pad_top = numext::maxi<Index>( 152 0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2); 153 const TensorIndex forward_pad_left = numext::maxi<Index>( 154 0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2); 155 const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top; 156 const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left; 157 158 const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride - 159 2 - padding_top + kernelRowsEff; 160 const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride - 161 2 - padding_left + kernelColsEff; 162 163 eigen_assert(padding_top >= 0); 164 eigen_assert(padding_left >= 0); 165 eigen_assert(padding_bottom >= 0); 166 eigen_assert(padding_right >= 0); 167 168 // The kernel has dimensions filters X channels X patch_rows X patch_cols 169 // We need to reverse the kernel along dimensions corresponding to rows and 170 // cols. 171 // TODO(yangke): we can make things slightly faster by collapsing the 172 // dimensions 173 // where we don't reverse. Try that once we have a faster compiler. 174 typedef typename internal::conditional<isColMajor, ReverseColMajor, 175 ReverseRowMajor>::type Reverse; 176 Reverse kernel_reverse; 177 178 #ifndef EIGEN_HAS_INDEX_LIST 179 if (isColMajor) { 180 kernel_reverse[0] = false; 181 kernel_reverse[1] = false; 182 kernel_reverse[2] = true; 183 kernel_reverse[3] = true; 184 } else { 185 kernel_reverse[0] = true; 186 kernel_reverse[1] = true; 187 kernel_reverse[2] = false; 188 kernel_reverse[3] = false; 189 } 190 #endif 191 192 // Reorder the dimensions to filters X patch_rows X patch_cols X channels 193 array<TensorIndex, 4> kernel_shuffle; 194 if (isColMajor) { 195 kernel_shuffle[0] = 0; 196 kernel_shuffle[1] = 2; 197 kernel_shuffle[2] = 3; 198 kernel_shuffle[3] = 1; 199 } else { 200 kernel_shuffle[0] = 2; 201 kernel_shuffle[1] = 0; 202 kernel_shuffle[2] = 1; 203 kernel_shuffle[3] = 3; 204 } 205 206 // Collapse the dims 207 DSizes<TensorIndex, 2> kernel_dims; 208 if (isColMajor) { 209 kernel_dims[0] = kernelFilters * kernelRows * kernelCols; 210 kernel_dims[1] = kernelChannels; 211 } else { 212 kernel_dims[1] = kernelFilters * kernelRows * kernelCols; 213 kernel_dims[0] = kernelChannels; 214 } 215 216 // The output_backward has dimensions out_depth X out_rows X out_cols X OTHERS 217 // When we extract the image patches from output_backward, it will have 218 // dimensions 219 // out_depth X (patch_rows * patch_cols) X (input_rows * input_cols * 220 // OTHERS) 221 DSizes<TensorIndex, 2> pre_contract_dims; 222 if (isColMajor) { 223 pre_contract_dims[0] = kernelFilters * kernelRows * kernelCols; 224 pre_contract_dims[1] = inputRows * inputCols; 225 for (int i = 3; i < NumDims; ++i) { 226 pre_contract_dims[1] *= out.dimension(i); 227 } 228 } else { 229 pre_contract_dims[1] = kernelFilters * kernelRows * kernelCols; 230 pre_contract_dims[0] = inputRows * inputCols; 231 for (int i = 0; i < NumDims - 3; ++i) { 232 pre_contract_dims[0] *= out.dimension(i); 233 } 234 } 235 236 // We will contract along the fused dimension that contains the kernelFilters, 237 // the kernelRows and the kernelCols. 238 array<IndexPair<TensorIndex>, 1> contract_dims; 239 if (isColMajor) { 240 // col-major: kernel.contract(output.patches) 241 contract_dims[0] = IndexPair<TensorIndex>(0, 0); 242 } else { 243 // row-major: output.patches.contract(kernel) 244 contract_dims[0] = IndexPair<TensorIndex>(1, 1); 245 } 246 247 // Post contraction, the dimensions of the input_backprop is 248 // channels X input_rows X input_cols X OTHERS 249 DSizes<TensorIndex, NumDims> post_contract_dims; 250 if (isColMajor) { 251 post_contract_dims[0] = kernelChannels; 252 post_contract_dims[1] = inputRows; 253 post_contract_dims[2] = inputCols; 254 for (int i = 3; i < NumDims; ++i) { 255 post_contract_dims[i] = out.dimension(i); 256 } 257 } else { 258 post_contract_dims[NumDims - 1] = kernelChannels; 259 post_contract_dims[NumDims - 2] = inputRows; 260 post_contract_dims[NumDims - 3] = inputCols; 261 for (int i = 0; i < NumDims - 3; ++i) { 262 post_contract_dims[i] = out.dimension(i); 263 } 264 } 265 266 return choose( 267 Cond<internal::traits<OutputBackward>::Layout == ColMajor>(), 268 kernel.reverse(kernel_reverse) 269 .shuffle(kernel_shuffle) 270 .reshape(kernel_dims) 271 .eval() 272 .contract( 273 output_backward 274 .extract_image_patches( 275 kernelRows, kernelCols, 1, 1, row_in_stride, 276 col_in_stride, row_stride, col_stride, padding_top, 277 padding_bottom, padding_left, padding_right, OutScalar(0)) 278 .reshape(pre_contract_dims), 279 contract_dims) 280 .reshape(post_contract_dims), 281 output_backward 282 .extract_image_patches(kernelRows, kernelCols, 1, 1, row_in_stride, 283 col_in_stride, row_stride, col_stride, 284 padding_top, padding_bottom, padding_left, 285 padding_right, OutScalar(0)) 286 .reshape(pre_contract_dims) 287 .contract(kernel.reverse(kernel_reverse) 288 .shuffle(kernel_shuffle) 289 .reshape(kernel_dims) 290 .eval(), 291 contract_dims) 292 .reshape(post_contract_dims)); 293 } 294 295 /** SpatialConvolutionBackwardKernel 296 * \ingroup CXX11_NeuralNetworks_Module 297 * 298 * \brief Computes the backprop for the filter of a 2D convolution. 299 * 300 * The output_backward parameter is expected to be a tensor with a rank of 3 or 301 * more (channels, height, width, and optionally others) 302 * The kernel parameter is expected to be a 4D tensor (filters, channels, 303 * kernel_height, kernel_width) 304 * The output_backward and the kernel must both be in col-major layout. The 305 * result will also be in col-major layout. 306 * 307 * If row_in_stride, col_stride > 1, then applies convolution with holes (aka 308 * atrous convolution), sampling every row_in_stride, col_in_stride input 309 * pixels. 310 * 311 * The result can be assigned to a tensor of rank equal to the rank of the 312 * output_backward. The dimensions of the result will be filters, height, width 313 * (and others if applicable). 314 * 315 * It is possible to swap the order of the width and height dimensions provided 316 * that the same order is used in the input, the kernel, and the output. 317 * 318 */ 319 320 template <typename OutputBackward, typename Input> 321 EIGEN_ALWAYS_INLINE static const typename internal::conditional< 322 internal::traits<OutputBackward>::Layout == ColMajor, 323 TensorReshapingOp< 324 const DSizes<typename internal::traits<Input>::Index, 4>, 325 const TensorContractionOp< 326 const array<IndexPair<typename internal::traits<Input>::Index>, 1>, 327 const TensorReshapingOp< 328 const DSizes<typename internal::traits<Input>::Index, 2>, 329 const OutputBackward>, 330 const TensorShufflingOp< 331 const array<typename internal::traits<OutputBackward>::Index, 332 2>, 333 const TensorReshapingOp< 334 const DSizes<typename internal::traits<Input>::Index, 2>, 335 const TensorImagePatchOp<Dynamic, Dynamic, 336 const Input> > > > >, 337 TensorReshapingOp< 338 const DSizes<typename internal::traits<Input>::Index, 4>, 339 const TensorContractionOp< 340 const array<IndexPair<typename internal::traits<Input>::Index>, 1>, 341 const TensorShufflingOp< 342 const array<typename internal::traits<OutputBackward>::Index, 343 2>, 344 const TensorReshapingOp< 345 const DSizes<typename internal::traits<Input>::Index, 2>, 346 const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >, 347 const TensorReshapingOp< 348 const DSizes<typename internal::traits<Input>::Index, 2>, 349 const OutputBackward> > > >::type 350 SpatialConvolutionBackwardKernel( 351 const Input& input, const OutputBackward& output_backward, 352 typename internal::traits<Input>::Index kernelRows, 353 typename internal::traits<Input>::Index kernelCols, 354 const DenseIndex row_stride = 1, const DenseIndex col_stride = 1, 355 const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) { 356 typedef typename internal::traits<Input>::Index TensorIndex; 357 typedef typename internal::traits<OutputBackward>::Scalar OutScalar; 358 TensorRef<Tensor<typename internal::traits<Input>::Scalar, 359 internal::traits<Input>::NumDimensions, 360 internal::traits<Input>::Layout, TensorIndex> > 361 in(input); 362 TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions, 363 internal::traits<OutputBackward>::Layout, TensorIndex> > 364 out(output_backward); 365 366 EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == 367 internal::traits<OutputBackward>::Layout, 368 YOU_MADE_A_PROGRAMMING_MISTAKE); 369 370 // stride and in_stride cannot both be larger than 1 371 eigen_assert(!(row_stride > 1 && row_in_stride > 1) && 372 !(col_stride > 1 && col_in_stride > 1)); 373 374 static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor); 375 376 static const int NumDims = internal::traits<Input>::NumDimensions; 377 EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 378 internal::traits<OutputBackward>::NumDimensions, 379 YOU_MADE_A_PROGRAMMING_MISTAKE); 380 381 const TensorIndex inputRows = 382 isColMajor ? in.dimension(1) : in.dimension(NumDims - 2); 383 const TensorIndex inputCols = 384 isColMajor ? in.dimension(2) : in.dimension(NumDims - 3); 385 386 const TensorIndex outputRows = isColMajor 387 ? output_backward.dimension(1) 388 : output_backward.dimension(NumDims - 2); 389 const TensorIndex outputCols = isColMajor 390 ? output_backward.dimension(2) 391 : output_backward.dimension(NumDims - 3); 392 393 // Number of filters to apply. This is the same as the output depth of the 394 // result 395 const TensorIndex kernelFilters = 396 isColMajor ? out.dimensions()[0] : out.dimensions()[NumDims - 1]; 397 398 // Number of channels. This is the same as the input depth. 399 const TensorIndex kernelChannels = 400 isColMajor ? in.dimensions()[0] : in.dimensions()[NumDims - 1]; 401 402 // This is the effective kernel size, taking into account the (*_in_stride - 403 // 1) zero-values 404 // inserted between consecutive kernel elements in atrous convolution 405 const TensorIndex kernelRowsEff = 406 kernelRows + (kernelRows - 1) * (row_in_stride - 1); 407 const TensorIndex kernelColsEff = 408 kernelCols + (kernelCols - 1) * (col_in_stride - 1); 409 410 // Computing the forward padding 411 const TensorIndex padRows = numext::maxi<Index>( 412 0, (outputRows - 1) * row_stride + kernelRowsEff - inputRows); 413 const TensorIndex padCols = numext::maxi<Index>( 414 0, (outputCols - 1) * col_stride + kernelColsEff - inputCols); 415 const TensorIndex padding_top = padRows / 2; 416 const TensorIndex padding_bottom = padRows - padding_top; 417 const TensorIndex padding_left = padCols / 2; 418 const TensorIndex padding_right = padCols - padding_left; 419 420 // Reshaped out 421 DSizes<TensorIndex, 2> output_dims; 422 if (isColMajor) { 423 output_dims[0] = kernelFilters; 424 output_dims[1] = outputRows * outputCols; 425 for (int i = 3; i < NumDims; ++i) { 426 output_dims[1] *= out.dimension(i); 427 } 428 } else { 429 output_dims[1] = kernelFilters; 430 output_dims[0] = outputCols * outputRows; 431 for (int i = 0; i < NumDims - 3; ++i) { 432 output_dims[0] *= out.dimension(i); 433 } 434 } 435 436 // Reshaped extract_image_patches(in) 437 DSizes<TensorIndex, 2> pre_contract_dims; 438 if (isColMajor) { 439 pre_contract_dims[0] = kernelChannels * kernelRows * kernelCols; 440 pre_contract_dims[1] = outputRows * outputCols; 441 for (int i = 3; i < NumDims; ++i) { 442 pre_contract_dims[1] *= in.dimension(i); 443 } 444 eigen_assert(output_dims[1] == pre_contract_dims[1]); 445 } else { 446 pre_contract_dims[1] = kernelCols * kernelRows * kernelChannels; 447 pre_contract_dims[0] = outputRows * outputCols; 448 for (int i = 0; i < NumDims - 3; ++i) { 449 pre_contract_dims[0] *= in.dimension(i); 450 } 451 eigen_assert(output_dims[0] == pre_contract_dims[0]); 452 } 453 454 array<TensorIndex, 2> shuffle_dims; 455 shuffle_dims[0] = 1; 456 shuffle_dims[1] = 0; 457 458 array<IndexPair<TensorIndex>, 1> contract_dims; 459 contract_dims[0] = IndexPair<TensorIndex>(1, 0); 460 461 // After the contraction, the kernel will have the desired shape 462 // out_depth X in_shape X kernel_rows X kernel_cols 463 DSizes<TensorIndex, 4> kernel_dims; 464 if (isColMajor) { 465 kernel_dims[0] = kernelFilters; 466 kernel_dims[1] = kernelChannels; 467 kernel_dims[2] = kernelRows; 468 kernel_dims[3] = kernelCols; 469 } else { 470 kernel_dims[3] = kernelFilters; 471 kernel_dims[2] = kernelChannels; 472 kernel_dims[1] = kernelRows; 473 kernel_dims[0] = kernelCols; 474 } 475 476 return choose( 477 Cond<internal::traits<Input>::Layout == ColMajor>(), 478 output_backward.reshape(output_dims) 479 .contract( 480 input 481 .extract_image_patches( 482 kernelRows, kernelCols, row_stride, col_stride, 483 row_in_stride, col_in_stride, 1, 1, padding_top, 484 padding_bottom, padding_left, padding_right, OutScalar(0)) 485 .reshape(pre_contract_dims) 486 .shuffle(shuffle_dims), 487 contract_dims) 488 .reshape(kernel_dims), 489 input 490 .extract_image_patches(kernelRows, kernelCols, row_stride, col_stride, 491 row_in_stride, col_in_stride, 1, 1, 492 padding_top, padding_bottom, padding_left, 493 padding_right, OutScalar(0)) 494 .reshape(pre_contract_dims) 495 .shuffle(shuffle_dims) 496 .contract(output_backward.reshape(output_dims), contract_dims) 497 .reshape(kernel_dims)); 498 } 499 500 } // end namespace Eigen 501 502 #endif // EIGEN_CXX11_NEURAL_NETWORKS_BACKWARD_SPATIAL_CONVOLUTIONS_H 503
