1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. 14 // Copyright (C) 2009, Willow Garage Inc., all rights reserved. 15 // Third party copyrights are property of their respective owners. 16 // 17 // Redistribution and use in source and binary forms, with or without modification, 18 // are permitted provided that the following conditions are met: 19 // 20 // * Redistribution's of source code must retain the above copyright notice, 21 // this list of conditions and the following disclaimer. 22 // 23 // * Redistribution's in binary form must reproduce the above copyright notice, 24 // this list of conditions and the following disclaimer in the documentation 25 // and/or other materials provided with the distribution. 26 // 27 // * The name of the copyright holders may not be used to endorse or promote products 28 // derived from this software without specific prior written permission. 29 // 30 // This software is provided by the copyright holders and contributors "as is" and 31 // any express or implied warranties, including, but not limited to, the implied 32 // warranties of merchantability and fitness for a particular purpose are disclaimed. 33 // In no event shall the Intel Corporation or contributors be liable for any direct, 34 // indirect, incidental, special, exemplary, or consequential damages 35 // (including, but not limited to, procurement of substitute goods or services; 36 // loss of use, data, or profits; or business interruption) however caused 37 // and on any theory of liability, whether in contract, strict liability, 38 // or tort (including negligence or otherwise) arising in any way out of 39 // the use of this software, even if advised of the possibility of such damage. 40 // 41 //M*/ 42 43 // S. Farsiu , D. Robinson, M. Elad, P. Milanfar. Fast and robust multiframe super resolution. 44 // Dennis Mitzel, Thomas Pock, Thomas Schoenemann, Daniel Cremers. Video Super Resolution using Duality Based TV-L1 Optical Flow. 45 46 #include "precomp.hpp" 47 48 using namespace cv; 49 using namespace cv::cuda; 50 using namespace cv::superres; 51 using namespace cv::superres::detail; 52 53 #if !defined(HAVE_CUDA) || !defined(HAVE_OPENCV_CUDAARITHM) || !defined(HAVE_OPENCV_CUDAWARPING) || !defined(HAVE_OPENCV_CUDAFILTERS) 54 55 Ptr<SuperResolution> cv::superres::createSuperResolution_BTVL1_CUDA() 56 { 57 CV_Error(Error::StsNotImplemented, "The called functionality is disabled for current build or platform"); 58 return Ptr<SuperResolution>(); 59 } 60 61 #else // HAVE_CUDA 62 63 namespace btv_l1_cudev 64 { 65 void buildMotionMaps(PtrStepSzf forwardMotionX, PtrStepSzf forwardMotionY, 66 PtrStepSzf backwardMotionX, PtrStepSzf bacwardMotionY, 67 PtrStepSzf forwardMapX, PtrStepSzf forwardMapY, 68 PtrStepSzf backwardMapX, PtrStepSzf backwardMapY); 69 70 template <int cn> 71 void upscale(const PtrStepSzb src, PtrStepSzb dst, int scale, cudaStream_t stream); 72 73 void diffSign(PtrStepSzf src1, PtrStepSzf src2, PtrStepSzf dst, cudaStream_t stream); 74 75 void loadBtvWeights(const float* weights, size_t count); 76 template <int cn> void calcBtvRegularization(PtrStepSzb src, PtrStepSzb dst, int ksize); 77 } 78 79 namespace 80 { 81 void calcRelativeMotions(const std::vector<std::pair<GpuMat, GpuMat> >& forwardMotions, const std::vector<std::pair<GpuMat, GpuMat> >& backwardMotions, 82 std::vector<std::pair<GpuMat, GpuMat> >& relForwardMotions, std::vector<std::pair<GpuMat, GpuMat> >& relBackwardMotions, 83 int baseIdx, Size size) 84 { 85 const int count = static_cast<int>(forwardMotions.size()); 86 87 relForwardMotions.resize(count); 88 relForwardMotions[baseIdx].first.create(size, CV_32FC1); 89 relForwardMotions[baseIdx].first.setTo(Scalar::all(0)); 90 relForwardMotions[baseIdx].second.create(size, CV_32FC1); 91 relForwardMotions[baseIdx].second.setTo(Scalar::all(0)); 92 93 relBackwardMotions.resize(count); 94 relBackwardMotions[baseIdx].first.create(size, CV_32FC1); 95 relBackwardMotions[baseIdx].first.setTo(Scalar::all(0)); 96 relBackwardMotions[baseIdx].second.create(size, CV_32FC1); 97 relBackwardMotions[baseIdx].second.setTo(Scalar::all(0)); 98 99 for (int i = baseIdx - 1; i >= 0; --i) 100 { 101 cuda::add(relForwardMotions[i + 1].first, forwardMotions[i].first, relForwardMotions[i].first); 102 cuda::add(relForwardMotions[i + 1].second, forwardMotions[i].second, relForwardMotions[i].second); 103 104 cuda::add(relBackwardMotions[i + 1].first, backwardMotions[i + 1].first, relBackwardMotions[i].first); 105 cuda::add(relBackwardMotions[i + 1].second, backwardMotions[i + 1].second, relBackwardMotions[i].second); 106 } 107 108 for (int i = baseIdx + 1; i < count; ++i) 109 { 110 cuda::add(relForwardMotions[i - 1].first, backwardMotions[i].first, relForwardMotions[i].first); 111 cuda::add(relForwardMotions[i - 1].second, backwardMotions[i].second, relForwardMotions[i].second); 112 113 cuda::add(relBackwardMotions[i - 1].first, forwardMotions[i - 1].first, relBackwardMotions[i].first); 114 cuda::add(relBackwardMotions[i - 1].second, forwardMotions[i - 1].second, relBackwardMotions[i].second); 115 } 116 } 117 118 void upscaleMotions(const std::vector<std::pair<GpuMat, GpuMat> >& lowResMotions, std::vector<std::pair<GpuMat, GpuMat> >& highResMotions, int scale) 119 { 120 highResMotions.resize(lowResMotions.size()); 121 122 for (size_t i = 0; i < lowResMotions.size(); ++i) 123 { 124 cuda::resize(lowResMotions[i].first, highResMotions[i].first, Size(), scale, scale, INTER_CUBIC); 125 cuda::resize(lowResMotions[i].second, highResMotions[i].second, Size(), scale, scale, INTER_CUBIC); 126 127 cuda::multiply(highResMotions[i].first, Scalar::all(scale), highResMotions[i].first); 128 cuda::multiply(highResMotions[i].second, Scalar::all(scale), highResMotions[i].second); 129 } 130 } 131 132 void buildMotionMaps(const std::pair<GpuMat, GpuMat>& forwardMotion, const std::pair<GpuMat, GpuMat>& backwardMotion, 133 std::pair<GpuMat, GpuMat>& forwardMap, std::pair<GpuMat, GpuMat>& backwardMap) 134 { 135 forwardMap.first.create(forwardMotion.first.size(), CV_32FC1); 136 forwardMap.second.create(forwardMotion.first.size(), CV_32FC1); 137 138 backwardMap.first.create(forwardMotion.first.size(), CV_32FC1); 139 backwardMap.second.create(forwardMotion.first.size(), CV_32FC1); 140 141 btv_l1_cudev::buildMotionMaps(forwardMotion.first, forwardMotion.second, 142 backwardMotion.first, backwardMotion.second, 143 forwardMap.first, forwardMap.second, 144 backwardMap.first, backwardMap.second); 145 } 146 147 void upscale(const GpuMat& src, GpuMat& dst, int scale, Stream& stream) 148 { 149 typedef void (*func_t)(const PtrStepSzb src, PtrStepSzb dst, int scale, cudaStream_t stream); 150 static const func_t funcs[] = 151 { 152 0, btv_l1_cudev::upscale<1>, 0, btv_l1_cudev::upscale<3>, btv_l1_cudev::upscale<4> 153 }; 154 155 CV_Assert( src.channels() == 1 || src.channels() == 3 || src.channels() == 4 ); 156 157 dst.create(src.rows * scale, src.cols * scale, src.type()); 158 dst.setTo(Scalar::all(0)); 159 160 const func_t func = funcs[src.channels()]; 161 162 func(src, dst, scale, StreamAccessor::getStream(stream)); 163 } 164 165 void diffSign(const GpuMat& src1, const GpuMat& src2, GpuMat& dst, Stream& stream) 166 { 167 dst.create(src1.size(), src1.type()); 168 169 btv_l1_cudev::diffSign(src1.reshape(1), src2.reshape(1), dst.reshape(1), StreamAccessor::getStream(stream)); 170 } 171 172 void calcBtvWeights(int btvKernelSize, double alpha, std::vector<float>& btvWeights) 173 { 174 const size_t size = btvKernelSize * btvKernelSize; 175 176 btvWeights.resize(size); 177 178 const int ksize = (btvKernelSize - 1) / 2; 179 const float alpha_f = static_cast<float>(alpha); 180 181 for (int m = 0, ind = 0; m <= ksize; ++m) 182 { 183 for (int l = ksize; l + m >= 0; --l, ++ind) 184 btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l)); 185 } 186 187 btv_l1_cudev::loadBtvWeights(&btvWeights[0], size); 188 } 189 190 void calcBtvRegularization(const GpuMat& src, GpuMat& dst, int btvKernelSize) 191 { 192 typedef void (*func_t)(PtrStepSzb src, PtrStepSzb dst, int ksize); 193 static const func_t funcs[] = 194 { 195 0, 196 btv_l1_cudev::calcBtvRegularization<1>, 197 0, 198 btv_l1_cudev::calcBtvRegularization<3>, 199 btv_l1_cudev::calcBtvRegularization<4> 200 }; 201 202 dst.create(src.size(), src.type()); 203 dst.setTo(Scalar::all(0)); 204 205 const int ksize = (btvKernelSize - 1) / 2; 206 207 funcs[src.channels()](src, dst, ksize); 208 } 209 210 class BTVL1_CUDA_Base : public cv::superres::SuperResolution 211 { 212 public: 213 BTVL1_CUDA_Base(); 214 215 void process(const std::vector<GpuMat>& src, GpuMat& dst, 216 const std::vector<std::pair<GpuMat, GpuMat> >& forwardMotions, const std::vector<std::pair<GpuMat, GpuMat> >& backwardMotions, 217 int baseIdx); 218 219 void collectGarbage(); 220 221 CV_IMPL_PROPERTY(int, Scale, scale_) 222 CV_IMPL_PROPERTY(int, Iterations, iterations_) 223 CV_IMPL_PROPERTY(double, Tau, tau_) 224 CV_IMPL_PROPERTY(double, Labmda, lambda_) 225 CV_IMPL_PROPERTY(double, Alpha, alpha_) 226 CV_IMPL_PROPERTY(int, KernelSize, btvKernelSize_) 227 CV_IMPL_PROPERTY(int, BlurKernelSize, blurKernelSize_) 228 CV_IMPL_PROPERTY(double, BlurSigma, blurSigma_) 229 CV_IMPL_PROPERTY(int, TemporalAreaRadius, temporalAreaRadius_) 230 CV_IMPL_PROPERTY_S(Ptr<cv::superres::DenseOpticalFlowExt>, OpticalFlow, opticalFlow_) 231 232 protected: 233 int scale_; 234 int iterations_; 235 double lambda_; 236 double tau_; 237 double alpha_; 238 int btvKernelSize_; 239 int blurKernelSize_; 240 double blurSigma_; 241 int temporalAreaRadius_; 242 Ptr<cv::superres::DenseOpticalFlowExt> opticalFlow_; 243 244 private: 245 std::vector<Ptr<cuda::Filter> > filters_; 246 int curBlurKernelSize_; 247 double curBlurSigma_; 248 int curSrcType_; 249 250 std::vector<float> btvWeights_; 251 int curBtvKernelSize_; 252 double curAlpha_; 253 254 std::vector<std::pair<GpuMat, GpuMat> > lowResForwardMotions_; 255 std::vector<std::pair<GpuMat, GpuMat> > lowResBackwardMotions_; 256 257 std::vector<std::pair<GpuMat, GpuMat> > highResForwardMotions_; 258 std::vector<std::pair<GpuMat, GpuMat> > highResBackwardMotions_; 259 260 std::vector<std::pair<GpuMat, GpuMat> > forwardMaps_; 261 std::vector<std::pair<GpuMat, GpuMat> > backwardMaps_; 262 263 GpuMat highRes_; 264 265 std::vector<Stream> streams_; 266 std::vector<GpuMat> diffTerms_; 267 std::vector<GpuMat> a_, b_, c_; 268 GpuMat regTerm_; 269 }; 270 271 BTVL1_CUDA_Base::BTVL1_CUDA_Base() 272 { 273 scale_ = 4; 274 iterations_ = 180; 275 lambda_ = 0.03; 276 tau_ = 1.3; 277 alpha_ = 0.7; 278 btvKernelSize_ = 7; 279 blurKernelSize_ = 5; 280 blurSigma_ = 0.0; 281 282 #ifdef HAVE_OPENCV_CUDAOPTFLOW 283 opticalFlow_ = createOptFlow_Farneback_CUDA(); 284 #else 285 opticalFlow_ = createOptFlow_Farneback(); 286 #endif 287 temporalAreaRadius_ = 0; 288 289 curBlurKernelSize_ = -1; 290 curBlurSigma_ = -1.0; 291 curSrcType_ = -1; 292 293 curBtvKernelSize_ = -1; 294 curAlpha_ = -1.0; 295 } 296 297 void BTVL1_CUDA_Base::process(const std::vector<GpuMat>& src, GpuMat& dst, 298 const std::vector<std::pair<GpuMat, GpuMat> >& forwardMotions, const std::vector<std::pair<GpuMat, GpuMat> >& backwardMotions, 299 int baseIdx) 300 { 301 CV_Assert( scale_ > 1 ); 302 CV_Assert( iterations_ > 0 ); 303 CV_Assert( tau_ > 0.0 ); 304 CV_Assert( alpha_ > 0.0 ); 305 CV_Assert( btvKernelSize_ > 0 && btvKernelSize_ <= 16 ); 306 CV_Assert( blurKernelSize_ > 0 ); 307 CV_Assert( blurSigma_ >= 0.0 ); 308 309 // update blur filter and btv weights 310 311 if (filters_.size() != src.size() || blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_) 312 { 313 filters_.resize(src.size()); 314 for (size_t i = 0; i < src.size(); ++i) 315 filters_[i] = cuda::createGaussianFilter(src[0].type(), -1, Size(blurKernelSize_, blurKernelSize_), blurSigma_); 316 curBlurKernelSize_ = blurKernelSize_; 317 curBlurSigma_ = blurSigma_; 318 curSrcType_ = src[0].type(); 319 } 320 321 if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_) 322 { 323 calcBtvWeights(btvKernelSize_, alpha_, btvWeights_); 324 curBtvKernelSize_ = btvKernelSize_; 325 curAlpha_ = alpha_; 326 } 327 328 // calc motions between input frames 329 330 calcRelativeMotions(forwardMotions, backwardMotions, lowResForwardMotions_, lowResBackwardMotions_, baseIdx, src[0].size()); 331 332 upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_); 333 upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_); 334 335 forwardMaps_.resize(highResForwardMotions_.size()); 336 backwardMaps_.resize(highResForwardMotions_.size()); 337 for (size_t i = 0; i < highResForwardMotions_.size(); ++i) 338 buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]); 339 340 // initial estimation 341 342 const Size lowResSize = src[0].size(); 343 const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_); 344 345 cuda::resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_CUBIC); 346 347 // iterations 348 349 streams_.resize(src.size()); 350 diffTerms_.resize(src.size()); 351 a_.resize(src.size()); 352 b_.resize(src.size()); 353 c_.resize(src.size()); 354 355 for (int i = 0; i < iterations_; ++i) 356 { 357 for (size_t k = 0; k < src.size(); ++k) 358 { 359 // a = M * Ih 360 cuda::remap(highRes_, a_[k], backwardMaps_[k].first, backwardMaps_[k].second, INTER_NEAREST, BORDER_REPLICATE, Scalar(), streams_[k]); 361 // b = HM * Ih 362 filters_[k]->apply(a_[k], b_[k], streams_[k]); 363 // c = DHF * Ih 364 cuda::resize(b_[k], c_[k], lowResSize, 0, 0, INTER_NEAREST, streams_[k]); 365 366 diffSign(src[k], c_[k], c_[k], streams_[k]); 367 368 // a = Dt * diff 369 upscale(c_[k], a_[k], scale_, streams_[k]); 370 // b = HtDt * diff 371 filters_[k]->apply(a_[k], b_[k], streams_[k]); 372 // diffTerm = MtHtDt * diff 373 cuda::remap(b_[k], diffTerms_[k], forwardMaps_[k].first, forwardMaps_[k].second, INTER_NEAREST, BORDER_REPLICATE, Scalar(), streams_[k]); 374 } 375 376 if (lambda_ > 0) 377 { 378 calcBtvRegularization(highRes_, regTerm_, btvKernelSize_); 379 cuda::addWeighted(highRes_, 1.0, regTerm_, -tau_ * lambda_, 0.0, highRes_); 380 } 381 382 for (size_t k = 0; k < src.size(); ++k) 383 { 384 streams_[k].waitForCompletion(); 385 cuda::addWeighted(highRes_, 1.0, diffTerms_[k], tau_, 0.0, highRes_); 386 } 387 } 388 389 Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_); 390 highRes_(inner).copyTo(dst); 391 } 392 393 void BTVL1_CUDA_Base::collectGarbage() 394 { 395 filters_.clear(); 396 397 lowResForwardMotions_.clear(); 398 lowResBackwardMotions_.clear(); 399 400 highResForwardMotions_.clear(); 401 highResBackwardMotions_.clear(); 402 403 forwardMaps_.clear(); 404 backwardMaps_.clear(); 405 406 highRes_.release(); 407 408 diffTerms_.clear(); 409 a_.clear(); 410 b_.clear(); 411 c_.clear(); 412 regTerm_.release(); 413 } 414 415 //////////////////////////////////////////////////////////// 416 417 class BTVL1_CUDA : public BTVL1_CUDA_Base 418 { 419 public: 420 BTVL1_CUDA(); 421 422 void collectGarbage(); 423 424 protected: 425 void initImpl(Ptr<FrameSource>& frameSource); 426 void processImpl(Ptr<FrameSource>& frameSource, OutputArray output); 427 428 private: 429 void readNextFrame(Ptr<FrameSource>& frameSource); 430 void processFrame(int idx); 431 432 GpuMat curFrame_; 433 GpuMat prevFrame_; 434 435 std::vector<GpuMat> frames_; 436 std::vector<std::pair<GpuMat, GpuMat> > forwardMotions_; 437 std::vector<std::pair<GpuMat, GpuMat> > backwardMotions_; 438 std::vector<GpuMat> outputs_; 439 440 int storePos_; 441 int procPos_; 442 int outPos_; 443 444 std::vector<GpuMat> srcFrames_; 445 std::vector<std::pair<GpuMat, GpuMat> > srcForwardMotions_; 446 std::vector<std::pair<GpuMat, GpuMat> > srcBackwardMotions_; 447 GpuMat finalOutput_; 448 }; 449 450 BTVL1_CUDA::BTVL1_CUDA() 451 { 452 temporalAreaRadius_ = 4; 453 } 454 455 void BTVL1_CUDA::collectGarbage() 456 { 457 curFrame_.release(); 458 prevFrame_.release(); 459 460 frames_.clear(); 461 forwardMotions_.clear(); 462 backwardMotions_.clear(); 463 outputs_.clear(); 464 465 srcFrames_.clear(); 466 srcForwardMotions_.clear(); 467 srcBackwardMotions_.clear(); 468 finalOutput_.release(); 469 470 SuperResolution::collectGarbage(); 471 BTVL1_CUDA_Base::collectGarbage(); 472 } 473 474 void BTVL1_CUDA::initImpl(Ptr<FrameSource>& frameSource) 475 { 476 const int cacheSize = 2 * temporalAreaRadius_ + 1; 477 478 frames_.resize(cacheSize); 479 forwardMotions_.resize(cacheSize); 480 backwardMotions_.resize(cacheSize); 481 outputs_.resize(cacheSize); 482 483 storePos_ = -1; 484 485 for (int t = -temporalAreaRadius_; t <= temporalAreaRadius_; ++t) 486 readNextFrame(frameSource); 487 488 for (int i = 0; i <= temporalAreaRadius_; ++i) 489 processFrame(i); 490 491 procPos_ = temporalAreaRadius_; 492 outPos_ = -1; 493 } 494 495 void BTVL1_CUDA::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output) 496 { 497 if (outPos_ >= storePos_) 498 { 499 _output.release(); 500 return; 501 } 502 503 readNextFrame(frameSource); 504 505 if (procPos_ < storePos_) 506 { 507 ++procPos_; 508 processFrame(procPos_); 509 } 510 511 ++outPos_; 512 const GpuMat& curOutput = at(outPos_, outputs_); 513 514 if (_output.kind() == _InputArray::CUDA_GPU_MAT) 515 curOutput.convertTo(_output.getGpuMatRef(), CV_8U); 516 else 517 { 518 curOutput.convertTo(finalOutput_, CV_8U); 519 arrCopy(finalOutput_, _output); 520 } 521 } 522 523 void BTVL1_CUDA::readNextFrame(Ptr<FrameSource>& frameSource) 524 { 525 frameSource->nextFrame(curFrame_); 526 527 if (curFrame_.empty()) 528 return; 529 530 ++storePos_; 531 curFrame_.convertTo(at(storePos_, frames_), CV_32F); 532 533 if (storePos_ > 0) 534 { 535 std::pair<GpuMat, GpuMat>& forwardMotion = at(storePos_ - 1, forwardMotions_); 536 std::pair<GpuMat, GpuMat>& backwardMotion = at(storePos_, backwardMotions_); 537 538 opticalFlow_->calc(prevFrame_, curFrame_, forwardMotion.first, forwardMotion.second); 539 opticalFlow_->calc(curFrame_, prevFrame_, backwardMotion.first, backwardMotion.second); 540 } 541 542 curFrame_.copyTo(prevFrame_); 543 } 544 545 void BTVL1_CUDA::processFrame(int idx) 546 { 547 const int startIdx = std::max(idx - temporalAreaRadius_, 0); 548 const int procIdx = idx; 549 const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_); 550 551 const int count = endIdx - startIdx + 1; 552 553 srcFrames_.resize(count); 554 srcForwardMotions_.resize(count); 555 srcBackwardMotions_.resize(count); 556 557 int baseIdx = -1; 558 559 for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k) 560 { 561 if (i == procIdx) 562 baseIdx = k; 563 564 srcFrames_[k] = at(i, frames_); 565 566 if (i < endIdx) 567 srcForwardMotions_[k] = at(i, forwardMotions_); 568 if (i > startIdx) 569 srcBackwardMotions_[k] = at(i, backwardMotions_); 570 } 571 572 process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx); 573 } 574 } 575 576 Ptr<SuperResolution> cv::superres::createSuperResolution_BTVL1_CUDA() 577 { 578 return makePtr<BTVL1_CUDA>(); 579 } 580 581 #endif // HAVE_CUDA 582
