1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. 14 // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. 15 // Third party copyrights are property of their respective owners. 16 // 17 // Redistribution and use in source and binary forms, with or without modification, 18 // are permitted provided that the following conditions are met: 19 // 20 // * Redistribution's of source code must retain the above copyright notice, 21 // this list of conditions and the following disclaimer. 22 // 23 // * Redistribution's in binary form must reproduce the above copyright notice, 24 // this list of conditions and the following disclaimer in the documentation 25 // and/or other materials provided with the distribution. 26 // 27 // * The name of the copyright holders may not be used to endorse or promote products 28 // derived from this software without specific prior written permission. 29 // 30 // This software is provided by the copyright holders and contributors "as is" and 31 // any express or implied warranties, including, but not limited to, the implied 32 // warranties of merchantability and fitness for a particular purpose are disclaimed. 33 // In no event shall the Intel Corporation or contributors be liable for any direct, 34 // indirect, incidental, special, exemplary, or consequential damages 35 // (including, but not limited to, procurement of substitute goods or services; 36 // loss of use, data, or profits; or business interruption) however caused 37 // and on any theory of liability, whether in contract, strict liability, 38 // or tort (including negligence or otherwise) arising in any way out of 39 // the use of this software, even if advised of the possibility of such damage. 40 // 41 //M*/ 42 43 #include "precomp.hpp" 44 #include "opencv2/videostab/stabilizer.hpp" 45 #include "opencv2/videostab/ring_buffer.hpp" 46 47 // for debug purposes 48 #define SAVE_MOTIONS 0 49 50 namespace cv 51 { 52 namespace videostab 53 { 54 55 StabilizerBase::StabilizerBase() 56 { 57 setLog(makePtr<LogToStdout>()); 58 setFrameSource(makePtr<NullFrameSource>()); 59 setMotionEstimator(makePtr<KeypointBasedMotionEstimator>(makePtr<MotionEstimatorRansacL2>())); 60 setDeblurer(makePtr<NullDeblurer>()); 61 setInpainter(makePtr<NullInpainter>()); 62 setRadius(15); 63 setTrimRatio(0); 64 setCorrectionForInclusion(false); 65 setBorderMode(BORDER_REPLICATE); 66 } 67 68 69 void StabilizerBase::reset() 70 { 71 frameSize_ = Size(0, 0); 72 frameMask_ = Mat(); 73 curPos_ = -1; 74 curStabilizedPos_ = -1; 75 doDeblurring_ = false; 76 preProcessedFrame_ = Mat(); 77 doInpainting_ = false; 78 inpaintingMask_ = Mat(); 79 frames_.clear(); 80 motions_.clear(); 81 blurrinessRates_.clear(); 82 stabilizedFrames_.clear(); 83 stabilizedMasks_.clear(); 84 stabilizationMotions_.clear(); 85 processingStartTime_ = 0; 86 } 87 88 89 Mat StabilizerBase::nextStabilizedFrame() 90 { 91 // check if we've processed all frames already 92 if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1) 93 { 94 logProcessingTime(); 95 return Mat(); 96 } 97 98 bool processed; 99 do processed = doOneIteration(); 100 while (processed && curStabilizedPos_ == -1); 101 102 // check if the frame source is empty 103 if (curStabilizedPos_ == -1) 104 { 105 logProcessingTime(); 106 return Mat(); 107 } 108 109 return postProcessFrame(at(curStabilizedPos_, stabilizedFrames_)); 110 } 111 112 113 bool StabilizerBase::doOneIteration() 114 { 115 Mat frame = frameSource_->nextFrame(); 116 if (!frame.empty()) 117 { 118 curPos_++; 119 120 if (curPos_ > 0) 121 { 122 at(curPos_, frames_) = frame; 123 124 if (doDeblurring_) 125 at(curPos_, blurrinessRates_) = calcBlurriness(frame); 126 127 at(curPos_ - 1, motions_) = estimateMotion(); 128 129 if (curPos_ >= radius_) 130 { 131 curStabilizedPos_ = curPos_ - radius_; 132 stabilizeFrame(); 133 } 134 } 135 else 136 setUp(frame); 137 138 log_->print("."); 139 return true; 140 } 141 142 if (curStabilizedPos_ < curPos_) 143 { 144 curStabilizedPos_++; 145 at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_); 146 at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F); 147 stabilizeFrame(); 148 149 log_->print("."); 150 return true; 151 } 152 153 return false; 154 } 155 156 157 void StabilizerBase::setUp(const Mat &firstFrame) 158 { 159 InpainterBase *inpaint = inpainter_.get(); 160 doInpainting_ = dynamic_cast<NullInpainter*>(inpaint) == 0; 161 if (doInpainting_) 162 { 163 inpainter_->setMotionModel(motionEstimator_->motionModel()); 164 inpainter_->setFrames(frames_); 165 inpainter_->setMotions(motions_); 166 inpainter_->setStabilizedFrames(stabilizedFrames_); 167 inpainter_->setStabilizationMotions(stabilizationMotions_); 168 } 169 170 DeblurerBase *deblurer = deblurer_.get(); 171 doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0; 172 if (doDeblurring_) 173 { 174 blurrinessRates_.resize(2*radius_ + 1); 175 float blurriness = calcBlurriness(firstFrame); 176 for (int i = -radius_; i <= 0; ++i) 177 at(i, blurrinessRates_) = blurriness; 178 deblurer_->setFrames(frames_); 179 deblurer_->setMotions(motions_); 180 deblurer_->setBlurrinessRates(blurrinessRates_); 181 } 182 183 log_->print("processing frames"); 184 processingStartTime_ = clock(); 185 } 186 187 188 void StabilizerBase::stabilizeFrame() 189 { 190 Mat stabilizationMotion = estimateStabilizationMotion(); 191 if (doCorrectionForInclusion_) 192 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); 193 194 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; 195 196 if (doDeblurring_) 197 { 198 at(curStabilizedPos_, frames_).copyTo(preProcessedFrame_); 199 deblurer_->deblur(curStabilizedPos_, preProcessedFrame_); 200 } 201 else 202 preProcessedFrame_ = at(curStabilizedPos_, frames_); 203 204 // apply stabilization transformation 205 206 if (motionEstimator_->motionModel() != MM_HOMOGRAPHY) 207 warpAffine( 208 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), 209 stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_); 210 else 211 warpPerspective( 212 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_), 213 stabilizationMotion, frameSize_, INTER_LINEAR, borderMode_); 214 215 if (doInpainting_) 216 { 217 if (motionEstimator_->motionModel() != MM_HOMOGRAPHY) 218 warpAffine( 219 frameMask_, at(curStabilizedPos_, stabilizedMasks_), 220 stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST); 221 else 222 warpPerspective( 223 frameMask_, at(curStabilizedPos_, stabilizedMasks_), 224 stabilizationMotion, frameSize_, INTER_NEAREST); 225 226 erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_), 227 Mat()); 228 229 at(curStabilizedPos_, stabilizedMasks_).copyTo(inpaintingMask_); 230 231 inpainter_->inpaint( 232 curStabilizedPos_, at(curStabilizedPos_, stabilizedFrames_), inpaintingMask_); 233 } 234 } 235 236 237 Mat StabilizerBase::postProcessFrame(const Mat &frame) 238 { 239 // trim frame 240 int dx = static_cast<int>(floor(trimRatio_ * frame.cols)); 241 int dy = static_cast<int>(floor(trimRatio_ * frame.rows)); 242 return frame(Rect(dx, dy, frame.cols - 2*dx, frame.rows - 2*dy)); 243 } 244 245 246 void StabilizerBase::logProcessingTime() 247 { 248 clock_t elapsedTime = clock() - processingStartTime_; 249 log_->print("\nprocessing time: %.3f sec\n", static_cast<double>(elapsedTime) / CLOCKS_PER_SEC); 250 } 251 252 253 OnePassStabilizer::OnePassStabilizer() 254 { 255 setMotionFilter(makePtr<GaussianMotionFilter>()); 256 reset(); 257 } 258 259 260 void OnePassStabilizer::reset() 261 { 262 StabilizerBase::reset(); 263 } 264 265 266 void OnePassStabilizer::setUp(const Mat &firstFrame) 267 { 268 frameSize_ = firstFrame.size(); 269 frameMask_.create(frameSize_, CV_8U); 270 frameMask_.setTo(255); 271 272 int cacheSize = 2*radius_ + 1; 273 frames_.resize(cacheSize); 274 stabilizedFrames_.resize(cacheSize); 275 stabilizedMasks_.resize(cacheSize); 276 motions_.resize(cacheSize); 277 stabilizationMotions_.resize(cacheSize); 278 279 for (int i = -radius_; i < 0; ++i) 280 { 281 at(i, motions_) = Mat::eye(3, 3, CV_32F); 282 at(i, frames_) = firstFrame; 283 } 284 285 at(0, frames_) = firstFrame; 286 287 StabilizerBase::setUp(firstFrame); 288 } 289 290 291 Mat OnePassStabilizer::estimateMotion() 292 { 293 return motionEstimator_->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_)); 294 } 295 296 297 Mat OnePassStabilizer::estimateStabilizationMotion() 298 { 299 return motionFilter_->stabilize(curStabilizedPos_, motions_, std::make_pair(0, curPos_)); 300 } 301 302 303 Mat OnePassStabilizer::postProcessFrame(const Mat &frame) 304 { 305 return StabilizerBase::postProcessFrame(frame); 306 } 307 308 309 TwoPassStabilizer::TwoPassStabilizer() 310 { 311 setMotionStabilizer(makePtr<GaussianMotionFilter>()); 312 setWobbleSuppressor(makePtr<NullWobbleSuppressor>()); 313 setEstimateTrimRatio(false); 314 reset(); 315 } 316 317 318 void TwoPassStabilizer::reset() 319 { 320 StabilizerBase::reset(); 321 frameCount_ = 0; 322 isPrePassDone_ = false; 323 doWobbleSuppression_ = false; 324 motions2_.clear(); 325 suppressedFrame_ = Mat(); 326 } 327 328 329 Mat TwoPassStabilizer::nextFrame() 330 { 331 runPrePassIfNecessary(); 332 return StabilizerBase::nextStabilizedFrame(); 333 } 334 335 336 #if SAVE_MOTIONS 337 static void saveMotions( 338 int frameCount, const std::vector<Mat> &motions, const std::vector<Mat> &stabilizationMotions) 339 { 340 std::ofstream fm("log_motions.csv"); 341 for (int i = 0; i < frameCount - 1; ++i) 342 { 343 Mat_<float> M = at(i, motions); 344 fm << M(0,0) << " " << M(0,1) << " " << M(0,2) << " " 345 << M(1,0) << " " << M(1,1) << " " << M(1,2) << " " 346 << M(2,0) << " " << M(2,1) << " " << M(2,2) << std::endl; 347 } 348 std::ofstream fo("log_orig.csv"); 349 for (int i = 0; i < frameCount; ++i) 350 { 351 Mat_<float> M = getMotion(0, i, motions); 352 fo << M(0,0) << " " << M(0,1) << " " << M(0,2) << " " 353 << M(1,0) << " " << M(1,1) << " " << M(1,2) << " " 354 << M(2,0) << " " << M(2,1) << " " << M(2,2) << std::endl; 355 } 356 std::ofstream fs("log_stab.csv"); 357 for (int i = 0; i < frameCount; ++i) 358 { 359 Mat_<float> M = stabilizationMotions[i] * getMotion(0, i, motions); 360 fs << M(0,0) << " " << M(0,1) << " " << M(0,2) << " " 361 << M(1,0) << " " << M(1,1) << " " << M(1,2) << " " 362 << M(2,0) << " " << M(2,1) << " " << M(2,2) << std::endl; 363 } 364 } 365 #endif 366 367 368 void TwoPassStabilizer::runPrePassIfNecessary() 369 { 370 if (!isPrePassDone_) 371 { 372 // check if we must do wobble suppression 373 374 WobbleSuppressorBase *wobble = wobbleSuppressor_.get(); 375 doWobbleSuppression_ = dynamic_cast<NullWobbleSuppressor*>(wobble) == 0; 376 377 // estimate motions 378 379 clock_t startTime = clock(); 380 log_->print("first pass: estimating motions"); 381 382 Mat prevFrame, frame; 383 bool ok = true, ok2 = true; 384 385 while (!(frame = frameSource_->nextFrame()).empty()) 386 { 387 if (frameCount_ > 0) 388 { 389 motions_.push_back(motionEstimator_->estimate(prevFrame, frame, &ok)); 390 391 if (doWobbleSuppression_) 392 { 393 Mat M = wobbleSuppressor_->motionEstimator()->estimate(prevFrame, frame, &ok2); 394 if (ok2) 395 motions2_.push_back(M); 396 else 397 motions2_.push_back(motions_.back()); 398 } 399 400 if (ok) 401 { 402 if (ok2) log_->print("."); 403 else log_->print("?"); 404 } 405 else log_->print("x"); 406 } 407 else 408 { 409 frameSize_ = frame.size(); 410 frameMask_.create(frameSize_, CV_8U); 411 frameMask_.setTo(255); 412 } 413 414 prevFrame = frame; 415 frameCount_++; 416 } 417 418 clock_t elapsedTime = clock() - startTime; 419 log_->print("\nmotion estimation time: %.3f sec\n", 420 static_cast<double>(elapsedTime) / CLOCKS_PER_SEC); 421 422 // add aux. motions 423 424 for (int i = 0; i < radius_; ++i) 425 motions_.push_back(Mat::eye(3, 3, CV_32F)); 426 427 // stabilize 428 429 startTime = clock(); 430 431 stabilizationMotions_.resize(frameCount_); 432 motionStabilizer_->stabilize( 433 frameCount_, motions_, std::make_pair(0, frameCount_ - 1), &stabilizationMotions_[0]); 434 435 elapsedTime = clock() - startTime; 436 log_->print("motion stabilization time: %.3f sec\n", 437 static_cast<double>(elapsedTime) / CLOCKS_PER_SEC); 438 439 // estimate optimal trim ratio if necessary 440 441 if (mustEstTrimRatio_) 442 { 443 trimRatio_ = 0; 444 for (int i = 0; i < frameCount_; ++i) 445 { 446 Mat S = stabilizationMotions_[i]; 447 trimRatio_ = std::max(trimRatio_, estimateOptimalTrimRatio(S, frameSize_)); 448 } 449 log_->print("estimated trim ratio: %f\n", static_cast<double>(trimRatio_)); 450 } 451 452 #if SAVE_MOTIONS 453 saveMotions(frameCount_, motions_, stabilizationMotions_); 454 #endif 455 456 isPrePassDone_ = true; 457 frameSource_->reset(); 458 } 459 } 460 461 462 void TwoPassStabilizer::setUp(const Mat &firstFrame) 463 { 464 int cacheSize = 2*radius_ + 1; 465 frames_.resize(cacheSize); 466 stabilizedFrames_.resize(cacheSize); 467 stabilizedMasks_.resize(cacheSize); 468 469 for (int i = -radius_; i <= 0; ++i) 470 at(i, frames_) = firstFrame; 471 472 WobbleSuppressorBase *wobble = wobbleSuppressor_.get(); 473 doWobbleSuppression_ = dynamic_cast<NullWobbleSuppressor*>(wobble) == 0; 474 if (doWobbleSuppression_) 475 { 476 wobbleSuppressor_->setFrameCount(frameCount_); 477 wobbleSuppressor_->setMotions(motions_); 478 wobbleSuppressor_->setMotions2(motions2_); 479 wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_); 480 } 481 482 StabilizerBase::setUp(firstFrame); 483 } 484 485 486 Mat TwoPassStabilizer::estimateMotion() 487 { 488 return motions_[curPos_ - 1].clone(); 489 } 490 491 492 Mat TwoPassStabilizer::estimateStabilizationMotion() 493 { 494 return stabilizationMotions_[curStabilizedPos_].clone(); 495 } 496 497 498 Mat TwoPassStabilizer::postProcessFrame(const Mat &frame) 499 { 500 wobbleSuppressor_->suppress(curStabilizedPos_, frame, suppressedFrame_); 501 return StabilizerBase::postProcessFrame(suppressedFrame_); 502 } 503 504 } // namespace videostab 505 } // namespace cv 506
