1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. 14 // Copyright (C) 2009, Willow Garage Inc., all rights reserved. 15 // Third party copyrights are property of their respective owners. 16 // 17 // Redistribution and use in source and binary forms, with or without modification, 18 // are permitted provided that the following conditions are met: 19 // 20 // * Redistribution's of source code must retain the above copyright notice, 21 // this list of conditions and the following disclaimer. 22 // 23 // * Redistribution's in binary form must reproduce the above copyright notice, 24 // this list of conditions and the following disclaimer in the documentation 25 // and/or other materials provided with the distribution. 26 // 27 // * The name of the copyright holders may not be used to endorse or promote products 28 // derived from this software without specific prior written permission. 29 // 30 // This software is provided by the copyright holders and contributors "as is" and 31 // any express or implied warranties, including, but not limited to, the implied 32 // warranties of merchantability and fitness for a particular purpose are disclaimed. 33 // In no event shall the Intel Corporation or contributors be liable for any direct, 34 // indirect, incidental, special, exemplary, or consequential damages 35 // (including, but not limited to, procurement of substitute goods or services; 36 // loss of use, data, or profits; or business interruption) however caused 37 // and on any theory of liability, whether in contract, strict liability, 38 // or tort (including negligence or otherwise) arising in any way out of 39 // the use of this software, even if advised of the possibility of such damage. 40 // 41 //M*/ 42 43 // S. Farsiu , D. Robinson, M. Elad, P. Milanfar. Fast and robust multiframe super resolution. 44 // Dennis Mitzel, Thomas Pock, Thomas Schoenemann, Daniel Cremers. Video Super Resolution using Duality Based TV-L1 Optical Flow. 45 46 #include "precomp.hpp" 47 #include "opencl_kernels_superres.hpp" 48 49 using namespace cv; 50 using namespace cv::superres; 51 using namespace cv::superres::detail; 52 53 namespace 54 { 55 #ifdef HAVE_OPENCL 56 57 bool ocl_calcRelativeMotions(InputArrayOfArrays _forwardMotions, InputArrayOfArrays _backwardMotions, 58 OutputArrayOfArrays _relForwardMotions, OutputArrayOfArrays _relBackwardMotions, 59 int baseIdx, const Size & size) 60 { 61 std::vector<UMat> & forwardMotions = *(std::vector<UMat> *)_forwardMotions.getObj(), 62 & backwardMotions = *(std::vector<UMat> *)_backwardMotions.getObj(), 63 & relForwardMotions = *(std::vector<UMat> *)_relForwardMotions.getObj(), 64 & relBackwardMotions = *(std::vector<UMat> *)_relBackwardMotions.getObj(); 65 66 const int count = static_cast<int>(forwardMotions.size()); 67 68 relForwardMotions.resize(count); 69 relForwardMotions[baseIdx].create(size, CV_32FC2); 70 relForwardMotions[baseIdx].setTo(Scalar::all(0)); 71 72 relBackwardMotions.resize(count); 73 relBackwardMotions[baseIdx].create(size, CV_32FC2); 74 relBackwardMotions[baseIdx].setTo(Scalar::all(0)); 75 76 for (int i = baseIdx - 1; i >= 0; --i) 77 { 78 add(relForwardMotions[i + 1], forwardMotions[i], relForwardMotions[i]); 79 add(relBackwardMotions[i + 1], backwardMotions[i + 1], relBackwardMotions[i]); 80 } 81 82 for (int i = baseIdx + 1; i < count; ++i) 83 { 84 add(relForwardMotions[i - 1], backwardMotions[i], relForwardMotions[i]); 85 add(relBackwardMotions[i - 1], forwardMotions[i - 1], relBackwardMotions[i]); 86 } 87 88 return true; 89 } 90 91 #endif 92 93 void calcRelativeMotions(InputArrayOfArrays _forwardMotions, InputArrayOfArrays _backwardMotions, 94 OutputArrayOfArrays _relForwardMotions, OutputArrayOfArrays _relBackwardMotions, 95 int baseIdx, const Size & size) 96 { 97 CV_OCL_RUN(_forwardMotions.isUMatVector() && _backwardMotions.isUMatVector() && 98 _relForwardMotions.isUMatVector() && _relBackwardMotions.isUMatVector(), 99 ocl_calcRelativeMotions(_forwardMotions, _backwardMotions, _relForwardMotions, 100 _relBackwardMotions, baseIdx, size)) 101 102 std::vector<Mat> & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(), 103 & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj(), 104 & relForwardMotions = *(std::vector<Mat> *)_relForwardMotions.getObj(), 105 & relBackwardMotions = *(std::vector<Mat> *)_relBackwardMotions.getObj(); 106 107 const int count = static_cast<int>(forwardMotions.size()); 108 109 relForwardMotions.resize(count); 110 relForwardMotions[baseIdx].create(size, CV_32FC2); 111 relForwardMotions[baseIdx].setTo(Scalar::all(0)); 112 113 relBackwardMotions.resize(count); 114 relBackwardMotions[baseIdx].create(size, CV_32FC2); 115 relBackwardMotions[baseIdx].setTo(Scalar::all(0)); 116 117 for (int i = baseIdx - 1; i >= 0; --i) 118 { 119 add(relForwardMotions[i + 1], forwardMotions[i], relForwardMotions[i]); 120 add(relBackwardMotions[i + 1], backwardMotions[i + 1], relBackwardMotions[i]); 121 } 122 123 for (int i = baseIdx + 1; i < count; ++i) 124 { 125 add(relForwardMotions[i - 1], backwardMotions[i], relForwardMotions[i]); 126 add(relBackwardMotions[i - 1], forwardMotions[i - 1], relBackwardMotions[i]); 127 } 128 } 129 #ifdef HAVE_OPENCL 130 131 bool ocl_upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale) 132 { 133 std::vector<UMat> & lowResMotions = *(std::vector<UMat> *)_lowResMotions.getObj(), 134 & highResMotions = *(std::vector<UMat> *)_highResMotions.getObj(); 135 136 highResMotions.resize(lowResMotions.size()); 137 138 for (size_t i = 0; i < lowResMotions.size(); ++i) 139 { 140 resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_LINEAR); // TODO 141 multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]); 142 } 143 144 return true; 145 } 146 147 #endif 148 149 void upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale) 150 { 151 CV_OCL_RUN(_lowResMotions.isUMatVector() && _highResMotions.isUMatVector(), 152 ocl_upscaleMotions(_lowResMotions, _highResMotions, scale)) 153 154 std::vector<Mat> & lowResMotions = *(std::vector<Mat> *)_lowResMotions.getObj(), 155 & highResMotions = *(std::vector<Mat> *)_highResMotions.getObj(); 156 157 highResMotions.resize(lowResMotions.size()); 158 159 for (size_t i = 0; i < lowResMotions.size(); ++i) 160 { 161 resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_CUBIC); 162 multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]); 163 } 164 } 165 166 #ifdef HAVE_OPENCL 167 168 bool ocl_buildMotionMaps(InputArray _forwardMotion, InputArray _backwardMotion, 169 OutputArray _forwardMap, OutputArray _backwardMap) 170 { 171 ocl::Kernel k("buildMotionMaps", ocl::superres::superres_btvl1_oclsrc); 172 if (k.empty()) 173 return false; 174 175 UMat forwardMotion = _forwardMotion.getUMat(), backwardMotion = _backwardMotion.getUMat(); 176 Size size = forwardMotion.size(); 177 178 _forwardMap.create(size, CV_32FC2); 179 _backwardMap.create(size, CV_32FC2); 180 181 UMat forwardMap = _forwardMap.getUMat(), backwardMap = _backwardMap.getUMat(); 182 183 k.args(ocl::KernelArg::ReadOnlyNoSize(forwardMotion), 184 ocl::KernelArg::ReadOnlyNoSize(backwardMotion), 185 ocl::KernelArg::WriteOnlyNoSize(forwardMap), 186 ocl::KernelArg::WriteOnly(backwardMap)); 187 188 size_t globalsize[2] = { size.width, size.height }; 189 return k.run(2, globalsize, NULL, false); 190 } 191 192 #endif 193 194 void buildMotionMaps(InputArray _forwardMotion, InputArray _backwardMotion, 195 OutputArray _forwardMap, OutputArray _backwardMap) 196 { 197 CV_OCL_RUN(_forwardMap.isUMat() && _backwardMap.isUMat(), 198 ocl_buildMotionMaps(_forwardMotion, _backwardMotion, _forwardMap, 199 _backwardMap)); 200 201 Mat forwardMotion = _forwardMotion.getMat(), backwardMotion = _backwardMotion.getMat(); 202 203 _forwardMap.create(forwardMotion.size(), CV_32FC2); 204 _backwardMap.create(forwardMotion.size(), CV_32FC2); 205 206 Mat forwardMap = _forwardMap.getMat(), backwardMap = _backwardMap.getMat(); 207 208 for (int y = 0; y < forwardMotion.rows; ++y) 209 { 210 const Point2f* forwardMotionRow = forwardMotion.ptr<Point2f>(y); 211 const Point2f* backwardMotionRow = backwardMotion.ptr<Point2f>(y); 212 Point2f* forwardMapRow = forwardMap.ptr<Point2f>(y); 213 Point2f* backwardMapRow = backwardMap.ptr<Point2f>(y); 214 215 for (int x = 0; x < forwardMotion.cols; ++x) 216 { 217 Point2f base(static_cast<float>(x), static_cast<float>(y)); 218 219 forwardMapRow[x] = base + backwardMotionRow[x]; 220 backwardMapRow[x] = base + forwardMotionRow[x]; 221 } 222 } 223 } 224 225 template <typename T> 226 void upscaleImpl(InputArray _src, OutputArray _dst, int scale) 227 { 228 Mat src = _src.getMat(); 229 _dst.create(src.rows * scale, src.cols * scale, src.type()); 230 _dst.setTo(Scalar::all(0)); 231 Mat dst = _dst.getMat(); 232 233 for (int y = 0, Y = 0; y < src.rows; ++y, Y += scale) 234 { 235 const T * const srcRow = src.ptr<T>(y); 236 T * const dstRow = dst.ptr<T>(Y); 237 238 for (int x = 0, X = 0; x < src.cols; ++x, X += scale) 239 dstRow[X] = srcRow[x]; 240 } 241 } 242 243 #ifdef HAVE_OPENCL 244 245 static bool ocl_upscale(InputArray _src, OutputArray _dst, int scale) 246 { 247 int type = _src.type(), cn = CV_MAT_CN(type); 248 ocl::Kernel k("upscale", ocl::superres::superres_btvl1_oclsrc, 249 format("-D cn=%d", cn)); 250 if (k.empty()) 251 return false; 252 253 UMat src = _src.getUMat(); 254 _dst.create(src.rows * scale, src.cols * scale, type); 255 _dst.setTo(Scalar::all(0)); 256 UMat dst = _dst.getUMat(); 257 258 k.args(ocl::KernelArg::ReadOnly(src), 259 ocl::KernelArg::ReadWriteNoSize(dst), scale); 260 261 size_t globalsize[2] = { src.cols, src.rows }; 262 return k.run(2, globalsize, NULL, false); 263 } 264 265 #endif 266 267 typedef struct _Point4f { float ar[4]; } Point4f; 268 269 void upscale(InputArray _src, OutputArray _dst, int scale) 270 { 271 int cn = _src.channels(); 272 CV_Assert( cn == 1 || cn == 3 || cn == 4 ); 273 274 CV_OCL_RUN(_dst.isUMat(), 275 ocl_upscale(_src, _dst, scale)) 276 277 typedef void (*func_t)(InputArray src, OutputArray dst, int scale); 278 static const func_t funcs[] = 279 { 280 0, upscaleImpl<float>, 0, upscaleImpl<Point3f>, upscaleImpl<Point4f> 281 }; 282 283 const func_t func = funcs[cn]; 284 CV_Assert(func != 0); 285 func(_src, _dst, scale); 286 } 287 288 inline float diffSign(float a, float b) 289 { 290 return a > b ? 1.0f : a < b ? -1.0f : 0.0f; 291 } 292 293 Point3f diffSign(Point3f a, Point3f b) 294 { 295 return Point3f( 296 a.x > b.x ? 1.0f : a.x < b.x ? -1.0f : 0.0f, 297 a.y > b.y ? 1.0f : a.y < b.y ? -1.0f : 0.0f, 298 a.z > b.z ? 1.0f : a.z < b.z ? -1.0f : 0.0f 299 ); 300 } 301 302 #ifdef HAVE_OPENCL 303 304 static bool ocl_diffSign(InputArray _src1, OutputArray _src2, OutputArray _dst) 305 { 306 ocl::Kernel k("diffSign", ocl::superres::superres_btvl1_oclsrc); 307 if (k.empty()) 308 return false; 309 310 UMat src1 = _src1.getUMat(), src2 = _src2.getUMat(); 311 _dst.create(src1.size(), src1.type()); 312 UMat dst = _dst.getUMat(); 313 314 int cn = src1.channels(); 315 k.args(ocl::KernelArg::ReadOnlyNoSize(src1), 316 ocl::KernelArg::ReadOnlyNoSize(src2), 317 ocl::KernelArg::WriteOnly(dst, cn)); 318 319 size_t globalsize[2] = { src1.cols * cn, src1.rows }; 320 return k.run(2, globalsize, NULL, false); 321 } 322 323 #endif 324 325 void diffSign(InputArray _src1, OutputArray _src2, OutputArray _dst) 326 { 327 CV_OCL_RUN(_dst.isUMat(), 328 ocl_diffSign(_src1, _src2, _dst)) 329 330 Mat src1 = _src1.getMat(), src2 = _src2.getMat(); 331 _dst.create(src1.size(), src1.type()); 332 Mat dst = _dst.getMat(); 333 334 const int count = src1.cols * src1.channels(); 335 336 for (int y = 0; y < src1.rows; ++y) 337 { 338 const float * const src1Ptr = src1.ptr<float>(y); 339 const float * const src2Ptr = src2.ptr<float>(y); 340 float* dstPtr = dst.ptr<float>(y); 341 342 for (int x = 0; x < count; ++x) 343 dstPtr[x] = diffSign(src1Ptr[x], src2Ptr[x]); 344 } 345 } 346 347 void calcBtvWeights(int btvKernelSize, double alpha, std::vector<float>& btvWeights) 348 { 349 const size_t size = btvKernelSize * btvKernelSize; 350 351 btvWeights.resize(size); 352 353 const int ksize = (btvKernelSize - 1) / 2; 354 const float alpha_f = static_cast<float>(alpha); 355 356 for (int m = 0, ind = 0; m <= ksize; ++m) 357 { 358 for (int l = ksize; l + m >= 0; --l, ++ind) 359 btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l)); 360 } 361 } 362 363 template <typename T> 364 struct BtvRegularizationBody : ParallelLoopBody 365 { 366 void operator ()(const Range& range) const; 367 368 Mat src; 369 mutable Mat dst; 370 int ksize; 371 const float* btvWeights; 372 }; 373 374 template <typename T> 375 void BtvRegularizationBody<T>::operator ()(const Range& range) const 376 { 377 for (int i = range.start; i < range.end; ++i) 378 { 379 const T * const srcRow = src.ptr<T>(i); 380 T * const dstRow = dst.ptr<T>(i); 381 382 for(int j = ksize; j < src.cols - ksize; ++j) 383 { 384 const T srcVal = srcRow[j]; 385 386 for (int m = 0, ind = 0; m <= ksize; ++m) 387 { 388 const T* srcRow2 = src.ptr<T>(i - m); 389 const T* srcRow3 = src.ptr<T>(i + m); 390 391 for (int l = ksize; l + m >= 0; --l, ++ind) 392 dstRow[j] += btvWeights[ind] * (diffSign(srcVal, srcRow3[j + l]) 393 - diffSign(srcRow2[j - l], srcVal)); 394 } 395 } 396 } 397 } 398 399 template <typename T> 400 void calcBtvRegularizationImpl(InputArray _src, OutputArray _dst, int btvKernelSize, const std::vector<float>& btvWeights) 401 { 402 Mat src = _src.getMat(); 403 _dst.create(src.size(), src.type()); 404 _dst.setTo(Scalar::all(0)); 405 Mat dst = _dst.getMat(); 406 407 const int ksize = (btvKernelSize - 1) / 2; 408 409 BtvRegularizationBody<T> body; 410 411 body.src = src; 412 body.dst = dst; 413 body.ksize = ksize; 414 body.btvWeights = &btvWeights[0]; 415 416 parallel_for_(Range(ksize, src.rows - ksize), body); 417 } 418 419 #ifdef HAVE_OPENCL 420 421 static bool ocl_calcBtvRegularization(InputArray _src, OutputArray _dst, int btvKernelSize, const UMat & ubtvWeights) 422 { 423 int cn = _src.channels(); 424 ocl::Kernel k("calcBtvRegularization", ocl::superres::superres_btvl1_oclsrc, 425 format("-D cn=%d", cn)); 426 if (k.empty()) 427 return false; 428 429 UMat src = _src.getUMat(); 430 _dst.create(src.size(), src.type()); 431 _dst.setTo(Scalar::all(0)); 432 UMat dst = _dst.getUMat(); 433 434 const int ksize = (btvKernelSize - 1) / 2; 435 436 k.args(ocl::KernelArg::ReadOnlyNoSize(src), ocl::KernelArg::WriteOnly(dst), 437 ksize, ocl::KernelArg::PtrReadOnly(ubtvWeights)); 438 439 size_t globalsize[2] = { src.cols, src.rows }; 440 return k.run(2, globalsize, NULL, false); 441 } 442 443 #endif 444 445 void calcBtvRegularization(InputArray _src, OutputArray _dst, int btvKernelSize, 446 const std::vector<float>& btvWeights, const UMat & ubtvWeights) 447 { 448 CV_OCL_RUN(_dst.isUMat(), 449 ocl_calcBtvRegularization(_src, _dst, btvKernelSize, ubtvWeights)) 450 (void)ubtvWeights; 451 452 typedef void (*func_t)(InputArray _src, OutputArray _dst, int btvKernelSize, const std::vector<float>& btvWeights); 453 static const func_t funcs[] = 454 { 455 0, calcBtvRegularizationImpl<float>, 0, calcBtvRegularizationImpl<Point3f>, 0 456 }; 457 458 const func_t func = funcs[_src.channels()]; 459 CV_Assert(func != 0); 460 func(_src, _dst, btvKernelSize, btvWeights); 461 } 462 463 class BTVL1_Base : public cv::superres::SuperResolution 464 { 465 public: 466 BTVL1_Base(); 467 468 void process(InputArrayOfArrays src, OutputArray dst, InputArrayOfArrays forwardMotions, 469 InputArrayOfArrays backwardMotions, int baseIdx); 470 471 void collectGarbage(); 472 473 CV_IMPL_PROPERTY(int, Scale, scale_) 474 CV_IMPL_PROPERTY(int, Iterations, iterations_) 475 CV_IMPL_PROPERTY(double, Tau, tau_) 476 CV_IMPL_PROPERTY(double, Labmda, lambda_) 477 CV_IMPL_PROPERTY(double, Alpha, alpha_) 478 CV_IMPL_PROPERTY(int, KernelSize, btvKernelSize_) 479 CV_IMPL_PROPERTY(int, BlurKernelSize, blurKernelSize_) 480 CV_IMPL_PROPERTY(double, BlurSigma, blurSigma_) 481 CV_IMPL_PROPERTY(int, TemporalAreaRadius, temporalAreaRadius_) 482 CV_IMPL_PROPERTY_S(Ptr<cv::superres::DenseOpticalFlowExt>, OpticalFlow, opticalFlow_) 483 484 protected: 485 int scale_; 486 int iterations_; 487 double tau_; 488 double lambda_; 489 double alpha_; 490 int btvKernelSize_; 491 int blurKernelSize_; 492 double blurSigma_; 493 int temporalAreaRadius_; // not used in some implementations 494 Ptr<cv::superres::DenseOpticalFlowExt> opticalFlow_; 495 496 private: 497 bool ocl_process(InputArrayOfArrays src, OutputArray dst, InputArrayOfArrays forwardMotions, 498 InputArrayOfArrays backwardMotions, int baseIdx); 499 500 //Ptr<FilterEngine> filter_; 501 int curBlurKernelSize_; 502 double curBlurSigma_; 503 int curSrcType_; 504 505 std::vector<float> btvWeights_; 506 UMat ubtvWeights_; 507 508 int curBtvKernelSize_; 509 double curAlpha_; 510 511 // Mat 512 std::vector<Mat> lowResForwardMotions_; 513 std::vector<Mat> lowResBackwardMotions_; 514 515 std::vector<Mat> highResForwardMotions_; 516 std::vector<Mat> highResBackwardMotions_; 517 518 std::vector<Mat> forwardMaps_; 519 std::vector<Mat> backwardMaps_; 520 521 Mat highRes_; 522 523 Mat diffTerm_, regTerm_; 524 Mat a_, b_, c_; 525 526 #ifdef HAVE_OPENCL 527 // UMat 528 std::vector<UMat> ulowResForwardMotions_; 529 std::vector<UMat> ulowResBackwardMotions_; 530 531 std::vector<UMat> uhighResForwardMotions_; 532 std::vector<UMat> uhighResBackwardMotions_; 533 534 std::vector<UMat> uforwardMaps_; 535 std::vector<UMat> ubackwardMaps_; 536 537 UMat uhighRes_; 538 539 UMat udiffTerm_, uregTerm_; 540 UMat ua_, ub_, uc_; 541 #endif 542 }; 543 544 BTVL1_Base::BTVL1_Base() 545 { 546 scale_ = 4; 547 iterations_ = 180; 548 lambda_ = 0.03; 549 tau_ = 1.3; 550 alpha_ = 0.7; 551 btvKernelSize_ = 7; 552 blurKernelSize_ = 5; 553 blurSigma_ = 0.0; 554 temporalAreaRadius_ = 0; 555 opticalFlow_ = createOptFlow_Farneback(); 556 557 curBlurKernelSize_ = -1; 558 curBlurSigma_ = -1.0; 559 curSrcType_ = -1; 560 561 curBtvKernelSize_ = -1; 562 curAlpha_ = -1.0; 563 } 564 565 #ifdef HAVE_OPENCL 566 567 bool BTVL1_Base::ocl_process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions, 568 InputArrayOfArrays _backwardMotions, int baseIdx) 569 { 570 std::vector<UMat> & src = *(std::vector<UMat> *)_src.getObj(), 571 & forwardMotions = *(std::vector<UMat> *)_forwardMotions.getObj(), 572 & backwardMotions = *(std::vector<UMat> *)_backwardMotions.getObj(); 573 574 // update blur filter and btv weights 575 if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_) 576 { 577 //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_); 578 curBlurKernelSize_ = blurKernelSize_; 579 curBlurSigma_ = blurSigma_; 580 curSrcType_ = src[0].type(); 581 } 582 583 if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_) 584 { 585 calcBtvWeights(btvKernelSize_, alpha_, btvWeights_); 586 Mat(btvWeights_, true).copyTo(ubtvWeights_); 587 588 curBtvKernelSize_ = btvKernelSize_; 589 curAlpha_ = alpha_; 590 } 591 592 // calc high res motions 593 calcRelativeMotions(forwardMotions, backwardMotions, ulowResForwardMotions_, ulowResBackwardMotions_, baseIdx, src[0].size()); 594 595 upscaleMotions(ulowResForwardMotions_, uhighResForwardMotions_, scale_); 596 upscaleMotions(ulowResBackwardMotions_, uhighResBackwardMotions_, scale_); 597 598 uforwardMaps_.resize(uhighResForwardMotions_.size()); 599 ubackwardMaps_.resize(uhighResForwardMotions_.size()); 600 for (size_t i = 0; i < uhighResForwardMotions_.size(); ++i) 601 buildMotionMaps(uhighResForwardMotions_[i], uhighResBackwardMotions_[i], uforwardMaps_[i], ubackwardMaps_[i]); 602 603 // initial estimation 604 const Size lowResSize = src[0].size(); 605 const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_); 606 607 resize(src[baseIdx], uhighRes_, highResSize, 0, 0, INTER_LINEAR); // TODO 608 609 // iterations 610 udiffTerm_.create(highResSize, uhighRes_.type()); 611 ua_.create(highResSize, uhighRes_.type()); 612 ub_.create(highResSize, uhighRes_.type()); 613 uc_.create(lowResSize, uhighRes_.type()); 614 615 for (int i = 0; i < iterations_; ++i) 616 { 617 udiffTerm_.setTo(Scalar::all(0)); 618 619 for (size_t k = 0; k < src.size(); ++k) 620 { 621 // a = M * Ih 622 remap(uhighRes_, ua_, ubackwardMaps_[k], noArray(), INTER_NEAREST); 623 // b = HM * Ih 624 GaussianBlur(ua_, ub_, Size(blurKernelSize_, blurKernelSize_), blurSigma_); 625 // c = DHM * Ih 626 resize(ub_, uc_, lowResSize, 0, 0, INTER_NEAREST); 627 628 diffSign(src[k], uc_, uc_); 629 630 // a = Dt * diff 631 upscale(uc_, ua_, scale_); 632 633 // b = HtDt * diff 634 GaussianBlur(ua_, ub_, Size(blurKernelSize_, blurKernelSize_), blurSigma_); 635 // a = MtHtDt * diff 636 remap(ub_, ua_, uforwardMaps_[k], noArray(), INTER_NEAREST); 637 638 add(udiffTerm_, ua_, udiffTerm_); 639 } 640 641 if (lambda_ > 0) 642 { 643 calcBtvRegularization(uhighRes_, uregTerm_, btvKernelSize_, btvWeights_, ubtvWeights_); 644 addWeighted(udiffTerm_, 1.0, uregTerm_, -lambda_, 0.0, udiffTerm_); 645 } 646 647 addWeighted(uhighRes_, 1.0, udiffTerm_, tau_, 0.0, uhighRes_); 648 } 649 650 Rect inner(btvKernelSize_, btvKernelSize_, uhighRes_.cols - 2 * btvKernelSize_, uhighRes_.rows - 2 * btvKernelSize_); 651 uhighRes_(inner).copyTo(_dst); 652 653 return true; 654 } 655 656 #endif 657 658 void BTVL1_Base::process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions, 659 InputArrayOfArrays _backwardMotions, int baseIdx) 660 { 661 CV_Assert( scale_ > 1 ); 662 CV_Assert( iterations_ > 0 ); 663 CV_Assert( tau_ > 0.0 ); 664 CV_Assert( alpha_ > 0.0 ); 665 CV_Assert( btvKernelSize_ > 0 ); 666 CV_Assert( blurKernelSize_ > 0 ); 667 CV_Assert( blurSigma_ >= 0.0 ); 668 669 CV_OCL_RUN(_src.isUMatVector() && _dst.isUMat() && _forwardMotions.isUMatVector() && 670 _backwardMotions.isUMatVector(), 671 ocl_process(_src, _dst, _forwardMotions, _backwardMotions, baseIdx)) 672 673 std::vector<Mat> & src = *(std::vector<Mat> *)_src.getObj(), 674 & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(), 675 & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj(); 676 677 // update blur filter and btv weights 678 if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_) 679 { 680 //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_); 681 curBlurKernelSize_ = blurKernelSize_; 682 curBlurSigma_ = blurSigma_; 683 curSrcType_ = src[0].type(); 684 } 685 686 if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_) 687 { 688 calcBtvWeights(btvKernelSize_, alpha_, btvWeights_); 689 curBtvKernelSize_ = btvKernelSize_; 690 curAlpha_ = alpha_; 691 } 692 693 // calc high res motions 694 calcRelativeMotions(forwardMotions, backwardMotions, lowResForwardMotions_, lowResBackwardMotions_, baseIdx, src[0].size()); 695 696 upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_); 697 upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_); 698 699 forwardMaps_.resize(highResForwardMotions_.size()); 700 backwardMaps_.resize(highResForwardMotions_.size()); 701 for (size_t i = 0; i < highResForwardMotions_.size(); ++i) 702 buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]); 703 704 // initial estimation 705 const Size lowResSize = src[0].size(); 706 const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_); 707 708 resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_CUBIC); 709 710 // iterations 711 diffTerm_.create(highResSize, highRes_.type()); 712 a_.create(highResSize, highRes_.type()); 713 b_.create(highResSize, highRes_.type()); 714 c_.create(lowResSize, highRes_.type()); 715 716 for (int i = 0; i < iterations_; ++i) 717 { 718 diffTerm_.setTo(Scalar::all(0)); 719 720 for (size_t k = 0; k < src.size(); ++k) 721 { 722 // a = M * Ih 723 remap(highRes_, a_, backwardMaps_[k], noArray(), INTER_NEAREST); 724 // b = HM * Ih 725 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_); 726 // c = DHM * Ih 727 resize(b_, c_, lowResSize, 0, 0, INTER_NEAREST); 728 729 diffSign(src[k], c_, c_); 730 731 // a = Dt * diff 732 upscale(c_, a_, scale_); 733 // b = HtDt * diff 734 GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_); 735 // a = MtHtDt * diff 736 remap(b_, a_, forwardMaps_[k], noArray(), INTER_NEAREST); 737 738 add(diffTerm_, a_, diffTerm_); 739 } 740 741 if (lambda_ > 0) 742 { 743 calcBtvRegularization(highRes_, regTerm_, btvKernelSize_, btvWeights_, ubtvWeights_); 744 addWeighted(diffTerm_, 1.0, regTerm_, -lambda_, 0.0, diffTerm_); 745 } 746 747 addWeighted(highRes_, 1.0, diffTerm_, tau_, 0.0, highRes_); 748 } 749 750 Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_); 751 highRes_(inner).copyTo(_dst); 752 } 753 754 void BTVL1_Base::collectGarbage() 755 { 756 // Mat 757 lowResForwardMotions_.clear(); 758 lowResBackwardMotions_.clear(); 759 760 highResForwardMotions_.clear(); 761 highResBackwardMotions_.clear(); 762 763 forwardMaps_.clear(); 764 backwardMaps_.clear(); 765 766 highRes_.release(); 767 768 diffTerm_.release(); 769 regTerm_.release(); 770 a_.release(); 771 b_.release(); 772 c_.release(); 773 774 #ifdef HAVE_OPENCL 775 // UMat 776 ulowResForwardMotions_.clear(); 777 ulowResBackwardMotions_.clear(); 778 779 uhighResForwardMotions_.clear(); 780 uhighResBackwardMotions_.clear(); 781 782 uforwardMaps_.clear(); 783 ubackwardMaps_.clear(); 784 785 uhighRes_.release(); 786 787 udiffTerm_.release(); 788 uregTerm_.release(); 789 ua_.release(); 790 ub_.release(); 791 uc_.release(); 792 #endif 793 } 794 795 //////////////////////////////////////////////////////////////////// 796 797 class BTVL1 : public BTVL1_Base 798 { 799 public: 800 BTVL1(); 801 802 void collectGarbage(); 803 804 protected: 805 void initImpl(Ptr<FrameSource>& frameSource); 806 bool ocl_initImpl(Ptr<FrameSource>& frameSource); 807 808 void processImpl(Ptr<FrameSource>& frameSource, OutputArray output); 809 bool ocl_processImpl(Ptr<FrameSource>& frameSource, OutputArray output); 810 811 private: 812 void readNextFrame(Ptr<FrameSource>& frameSource); 813 bool ocl_readNextFrame(Ptr<FrameSource>& frameSource); 814 815 void processFrame(int idx); 816 bool ocl_processFrame(int idx); 817 818 int storePos_; 819 int procPos_; 820 int outPos_; 821 822 // Mat 823 Mat curFrame_; 824 Mat prevFrame_; 825 826 std::vector<Mat> frames_; 827 std::vector<Mat> forwardMotions_; 828 std::vector<Mat> backwardMotions_; 829 std::vector<Mat> outputs_; 830 831 std::vector<Mat> srcFrames_; 832 std::vector<Mat> srcForwardMotions_; 833 std::vector<Mat> srcBackwardMotions_; 834 Mat finalOutput_; 835 836 #ifdef HAVE_OPENCL 837 // UMat 838 UMat ucurFrame_; 839 UMat uprevFrame_; 840 841 std::vector<UMat> uframes_; 842 std::vector<UMat> uforwardMotions_; 843 std::vector<UMat> ubackwardMotions_; 844 std::vector<UMat> uoutputs_; 845 846 std::vector<UMat> usrcFrames_; 847 std::vector<UMat> usrcForwardMotions_; 848 std::vector<UMat> usrcBackwardMotions_; 849 #endif 850 }; 851 852 BTVL1::BTVL1() 853 { 854 temporalAreaRadius_ = 4; 855 } 856 857 void BTVL1::collectGarbage() 858 { 859 // Mat 860 curFrame_.release(); 861 prevFrame_.release(); 862 863 frames_.clear(); 864 forwardMotions_.clear(); 865 backwardMotions_.clear(); 866 outputs_.clear(); 867 868 srcFrames_.clear(); 869 srcForwardMotions_.clear(); 870 srcBackwardMotions_.clear(); 871 finalOutput_.release(); 872 873 #ifdef HAVE_OPENCL 874 // UMat 875 ucurFrame_.release(); 876 uprevFrame_.release(); 877 878 uframes_.clear(); 879 uforwardMotions_.clear(); 880 ubackwardMotions_.clear(); 881 uoutputs_.clear(); 882 883 usrcFrames_.clear(); 884 usrcForwardMotions_.clear(); 885 usrcBackwardMotions_.clear(); 886 #endif 887 888 SuperResolution::collectGarbage(); 889 BTVL1_Base::collectGarbage(); 890 } 891 892 #ifdef HAVE_OPENCL 893 894 bool BTVL1::ocl_initImpl(Ptr<FrameSource>& frameSource) 895 { 896 const int cacheSize = 2 * temporalAreaRadius_ + 1; 897 898 uframes_.resize(cacheSize); 899 uforwardMotions_.resize(cacheSize); 900 ubackwardMotions_.resize(cacheSize); 901 uoutputs_.resize(cacheSize); 902 903 storePos_ = -1; 904 905 for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t) 906 readNextFrame(frameSource); 907 908 for (int i = 0; i <= temporalAreaRadius_; ++i) 909 processFrame(i); 910 911 procPos_ = temporalAreaRadius_; 912 outPos_ = -1; 913 914 return true; 915 } 916 917 #endif 918 919 void BTVL1::initImpl(Ptr<FrameSource>& frameSource) 920 { 921 const int cacheSize = 2 * temporalAreaRadius_ + 1; 922 923 frames_.resize(cacheSize); 924 forwardMotions_.resize(cacheSize); 925 backwardMotions_.resize(cacheSize); 926 outputs_.resize(cacheSize); 927 928 CV_OCL_RUN(isUmat_, 929 ocl_initImpl(frameSource)) 930 931 storePos_ = -1; 932 933 for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t) 934 readNextFrame(frameSource); 935 936 for (int i = 0; i <= temporalAreaRadius_; ++i) 937 processFrame(i); 938 939 procPos_ = temporalAreaRadius_; 940 outPos_ = -1; 941 } 942 943 #ifdef HAVE_OPENCL 944 945 bool BTVL1::ocl_processImpl(Ptr<FrameSource>& /*frameSource*/, OutputArray _output) 946 { 947 const UMat& curOutput = at(outPos_, uoutputs_); 948 curOutput.convertTo(_output, CV_8U); 949 950 return true; 951 } 952 953 #endif 954 955 void BTVL1::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output) 956 { 957 if (outPos_ >= storePos_) 958 { 959 _output.release(); 960 return; 961 } 962 963 readNextFrame(frameSource); 964 965 if (procPos_ < storePos_) 966 { 967 ++procPos_; 968 processFrame(procPos_); 969 } 970 ++outPos_; 971 972 CV_OCL_RUN(isUmat_, 973 ocl_processImpl(frameSource, _output)) 974 975 const Mat& curOutput = at(outPos_, outputs_); 976 977 if (_output.kind() < _InputArray::OPENGL_BUFFER || _output.isUMat()) 978 curOutput.convertTo(_output, CV_8U); 979 else 980 { 981 curOutput.convertTo(finalOutput_, CV_8U); 982 arrCopy(finalOutput_, _output); 983 } 984 } 985 986 #ifdef HAVE_OPENCL 987 988 bool BTVL1::ocl_readNextFrame(Ptr<FrameSource>& /*frameSource*/) 989 { 990 ucurFrame_.convertTo(at(storePos_, uframes_), CV_32F); 991 992 if (storePos_ > 0) 993 { 994 opticalFlow_->calc(uprevFrame_, ucurFrame_, at(storePos_ - 1, uforwardMotions_)); 995 opticalFlow_->calc(ucurFrame_, uprevFrame_, at(storePos_, ubackwardMotions_)); 996 } 997 998 ucurFrame_.copyTo(uprevFrame_); 999 return true; 1000 } 1001 1002 #endif 1003 1004 void BTVL1::readNextFrame(Ptr<FrameSource>& frameSource) 1005 { 1006 frameSource->nextFrame(curFrame_); 1007 if (curFrame_.empty()) 1008 return; 1009 1010 #ifdef HAVE_OPENCL 1011 if (isUmat_) 1012 curFrame_.copyTo(ucurFrame_); 1013 #endif 1014 ++storePos_; 1015 1016 CV_OCL_RUN(isUmat_, 1017 ocl_readNextFrame(frameSource)) 1018 1019 curFrame_.convertTo(at(storePos_, frames_), CV_32F); 1020 1021 if (storePos_ > 0) 1022 { 1023 opticalFlow_->calc(prevFrame_, curFrame_, at(storePos_ - 1, forwardMotions_)); 1024 opticalFlow_->calc(curFrame_, prevFrame_, at(storePos_, backwardMotions_)); 1025 } 1026 1027 curFrame_.copyTo(prevFrame_); 1028 } 1029 1030 #ifdef HAVE_OPENCL 1031 1032 bool BTVL1::ocl_processFrame(int idx) 1033 { 1034 const int startIdx = std::max(idx - temporalAreaRadius_, 0); 1035 const int procIdx = idx; 1036 const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_); 1037 1038 const int count = endIdx - startIdx + 1; 1039 1040 usrcFrames_.resize(count); 1041 usrcForwardMotions_.resize(count); 1042 usrcBackwardMotions_.resize(count); 1043 1044 int baseIdx = -1; 1045 1046 for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k) 1047 { 1048 if (i == procIdx) 1049 baseIdx = k; 1050 1051 usrcFrames_[k] = at(i, uframes_); 1052 1053 if (i < endIdx) 1054 usrcForwardMotions_[k] = at(i, uforwardMotions_); 1055 if (i > startIdx) 1056 usrcBackwardMotions_[k] = at(i, ubackwardMotions_); 1057 } 1058 1059 process(usrcFrames_, at(idx, uoutputs_), usrcForwardMotions_, usrcBackwardMotions_, baseIdx); 1060 1061 return true; 1062 } 1063 1064 #endif 1065 1066 void BTVL1::processFrame(int idx) 1067 { 1068 CV_OCL_RUN(isUmat_, 1069 ocl_processFrame(idx)) 1070 1071 const int startIdx = std::max(idx - temporalAreaRadius_, 0); 1072 const int procIdx = idx; 1073 const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_); 1074 1075 const int count = endIdx - startIdx + 1; 1076 1077 srcFrames_.resize(count); 1078 srcForwardMotions_.resize(count); 1079 srcBackwardMotions_.resize(count); 1080 1081 int baseIdx = -1; 1082 1083 for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k) 1084 { 1085 if (i == procIdx) 1086 baseIdx = k; 1087 1088 srcFrames_[k] = at(i, frames_); 1089 1090 if (i < endIdx) 1091 srcForwardMotions_[k] = at(i, forwardMotions_); 1092 if (i > startIdx) 1093 srcBackwardMotions_[k] = at(i, backwardMotions_); 1094 } 1095 1096 process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx); 1097 } 1098 } 1099 1100 Ptr<cv::superres::SuperResolution> cv::superres::createSuperResolution_BTVL1() 1101 { 1102 return makePtr<BTVL1>(); 1103 } 1104
