1 /* 2 * stablizer.cpp - abstract for DVS (Digital Video Stabilizer) 3 * 4 * Copyright (c) 2014-2016 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Zong Wei <wei.zong (at) intel.com> 19 */ 20 21 #include "stabilizer.h" 22 23 using namespace cv; 24 using namespace cv::videostab; 25 using namespace std; 26 27 Mat 28 OnePassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos) 29 { 30 if (!(frame->data.empty())) 31 { 32 Mat image; 33 frame->data.getMat(ACCESS_READ).copyTo(image); 34 35 curPos_++; 36 37 if (curPos_ > 0) 38 { 39 at(curPos_, frames_) = image; 40 41 if (doDeblurring_) 42 at(curPos_, blurrinessRates_) = calcBlurriness(image); 43 44 at(curPos_ - 1, motions_) = estimateMotion(); 45 46 if (curPos_ >= radius_) 47 { 48 curStabilizedPos_ = curPos_ - radius_; 49 Mat stabilizationMotion = estimateStabilizationMotion(); 50 if (doCorrectionForInclusion_) 51 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); 52 53 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; 54 stablizedPos = curStabilizedPos_; 55 56 return stabilizationMotion; 57 } 58 } 59 else 60 setUpFrame(image); 61 62 log_->print("."); 63 return Mat(); 64 } 65 66 if (curStabilizedPos_ < curPos_) 67 { 68 curStabilizedPos_++; 69 stablizedPos = curStabilizedPos_; 70 at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_); 71 at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F); 72 73 Mat stabilizationMotion = estimateStabilizationMotion(); 74 if (doCorrectionForInclusion_) 75 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); 76 77 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; 78 79 log_->print("."); 80 81 return stabilizationMotion; 82 } 83 84 return Mat(); 85 } 86 87 88 Mat 89 OnePassVideoStabilizer::estimateMotion() 90 { 91 #if ENABLE_DVS_CL_PATH 92 cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ); 93 cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ); 94 95 cv::UMat ugrayImage0; 96 cv::UMat ugrayImage1; 97 if ( frame0.type() != CV_8U ) 98 { 99 cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY ); 100 } 101 else 102 { 103 ugrayImage0 = frame0; 104 } 105 106 if ( frame1.type() != CV_8U ) 107 { 108 cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY ); 109 } 110 else 111 { 112 ugrayImage1 = frame1; 113 } 114 115 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1); 116 #else 117 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_)); 118 #endif 119 } 120 121 void 122 OnePassVideoStabilizer::setUpFrame(const Mat &firstFrame) 123 { 124 frameSize_ = firstFrame.size(); 125 frameMask_.create(frameSize_, CV_8U); 126 frameMask_.setTo(255); 127 128 int cacheSize = 2 * radius_ + 1; 129 frames_.resize(2); 130 motions_.resize(cacheSize); 131 stabilizationMotions_.resize(cacheSize); 132 133 for (int i = -radius_; i < 0; ++i) 134 { 135 at(i, motions_) = Mat::eye(3, 3, CV_32F); 136 at(i, frames_) = firstFrame; 137 } 138 139 at(0, frames_) = firstFrame; 140 141 StabilizerBase::setUp(firstFrame); 142 } 143 144 145 Mat 146 TwoPassVideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos) 147 { 148 if (!(frame->data.empty())) 149 { 150 Mat image; 151 frame->data.getMat(ACCESS_READ).copyTo(image); 152 153 curPos_++; 154 155 if (curPos_ > 0) 156 { 157 at(curPos_, frames_) = image; 158 159 if (doDeblurring_) 160 at(curPos_, blurrinessRates_) = calcBlurriness(image); 161 162 at(curPos_ - 1, motions_) = estimateMotion(); 163 164 if (curPos_ >= radius_) 165 { 166 curStabilizedPos_ = curPos_ - radius_; 167 Mat stabilizationMotion = estimateStabilizationMotion(); 168 if (doCorrectionForInclusion_) 169 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); 170 171 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; 172 stablizedPos = curStabilizedPos_; 173 174 return stabilizationMotion; 175 } 176 } 177 else 178 setUpFrame(image); 179 180 log_->print("."); 181 return Mat(); 182 } 183 184 if (curStabilizedPos_ < curPos_) 185 { 186 curStabilizedPos_++; 187 stablizedPos = curStabilizedPos_; 188 at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_); 189 at(curStabilizedPos_ + radius_ - 1, motions_) = Mat::eye(3, 3, CV_32F); 190 191 Mat stabilizationMotion = estimateStabilizationMotion(); 192 if (doCorrectionForInclusion_) 193 stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_); 194 195 at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion; 196 197 log_->print("."); 198 199 return stabilizationMotion; 200 } 201 202 return Mat(); 203 } 204 205 206 Mat 207 TwoPassVideoStabilizer::estimateMotion() 208 { 209 #if ENABLE_DVS_CL_PATH 210 cv::UMat frame0 = at(curPos_ - 1, frames_).getUMat(ACCESS_READ); 211 cv::UMat frame1 = at(curPos_, frames_).getUMat(ACCESS_READ); 212 213 cv::UMat ugrayImage0; 214 cv::UMat ugrayImage1; 215 if ( frame0.type() != CV_8U ) 216 { 217 cvtColor( frame0, ugrayImage0, COLOR_BGR2GRAY ); 218 } 219 else 220 { 221 ugrayImage0 = frame0; 222 } 223 224 if ( frame1.type() != CV_8U ) 225 { 226 cvtColor( frame1, ugrayImage1, COLOR_BGR2GRAY ); 227 } 228 else 229 { 230 ugrayImage1 = frame1; 231 } 232 233 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(ugrayImage0, ugrayImage1); 234 #else 235 return motionEstimator_.dynamicCast<KeypointBasedMotionEstimator>()->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_)); 236 #endif 237 } 238 239 void 240 TwoPassVideoStabilizer::setUpFrame(const Mat &firstFrame) 241 { 242 //int cacheSize = 2*radius_ + 1; 243 frames_.resize(2); 244 stabilizedFrames_.resize(2); 245 stabilizedMasks_.resize(2); 246 247 for (int i = -1; i <= 0; ++i) 248 at(i, frames_) = firstFrame; 249 250 StabilizerBase::setUp(firstFrame); 251 } 252 253 VideoStabilizer::VideoStabilizer( 254 bool isTwoPass, 255 bool wobbleSuppress, 256 bool deblur, 257 bool inpainter) 258 : isTwoPass_ (isTwoPass) 259 , trimRatio_ (0.05f) 260 { 261 Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(MM_HOMOGRAPHY); 262 Ptr<IOutlierRejector> outlierRejector = makePtr<TranslationBasedLocalOutlierRejector>(); 263 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est); 264 kbest->setDetector(GFTTDetector::create(1000, 0.01, 15)); 265 kbest->setOutlierRejector(outlierRejector); 266 267 if (isTwoPass) 268 { 269 Ptr<TwoPassVideoStabilizer> twoPassStabilizer = makePtr<TwoPassVideoStabilizer>(); 270 stabilizer_ = twoPassStabilizer; 271 twoPassStabilizer->setEstimateTrimRatio(false); 272 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(15, 10)); 273 274 if (wobbleSuppress) { 275 Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>(); 276 277 ws->setMotionEstimator(kbest); 278 ws->setPeriod(30); 279 twoPassStabilizer->setWobbleSuppressor(ws); 280 } 281 } else { 282 Ptr<OnePassVideoStabilizer> onePassStabilizer = makePtr<OnePassVideoStabilizer>(); 283 stabilizer_ = onePassStabilizer; 284 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(15, 10)); 285 } 286 287 stabilizer_->setMotionEstimator(kbest); 288 289 stabilizer_->setRadius(15); 290 291 if (deblur) 292 { 293 Ptr<WeightingDeblurer> deblurrer = makePtr<WeightingDeblurer>(); 294 deblurrer->setRadius(3); 295 deblurrer->setSensitivity(0.001f); 296 stabilizer_->setDeblurer(deblurrer); 297 } 298 299 if (inpainter) 300 { 301 bool inpaintMosaic = true; 302 bool inpaintColorAverage = true; 303 bool inpaintColorNs = false; 304 bool inpaintColorTelea = false; 305 306 // init inpainter 307 InpaintingPipeline *inpainters = new InpaintingPipeline(); 308 Ptr<InpainterBase> inpainters_(inpainters); 309 if (true == inpaintMosaic) 310 { 311 Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>(); 312 inp->setStdevThresh(10.0f); 313 inpainters->pushBack(inp); 314 } 315 if (true == inpaintColorAverage) 316 inpainters->pushBack(makePtr<ColorAverageInpainter>()); 317 else if (true == inpaintColorNs) 318 inpainters->pushBack(makePtr<ColorInpainter>(0, 2)); 319 else if (true == inpaintColorTelea) 320 inpainters->pushBack(makePtr<ColorInpainter>(1, 2)); 321 if (!inpainters->empty()) 322 { 323 inpainters->setRadius(2); 324 stabilizer_->setInpainter(inpainters_); 325 } 326 } 327 } 328 329 VideoStabilizer::~VideoStabilizer() {} 330 331 void 332 VideoStabilizer::configFeatureDetector(int features, double minDistance) 333 { 334 Ptr<ImageMotionEstimatorBase> estimator = stabilizer_->motionEstimator(); 335 Ptr<FeatureDetector> detector = estimator.dynamicCast<KeypointBasedMotionEstimator>()->detector(); 336 if (NULL == detector) { 337 return; 338 } 339 340 detector.dynamicCast<GFTTDetector>()->setMaxFeatures(features); 341 detector.dynamicCast<GFTTDetector>()->setMinDistance(minDistance); 342 } 343 344 void 345 VideoStabilizer::configMotionFilter(int radius, float stdev) 346 { 347 if (isTwoPass_) { 348 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>(); 349 Ptr<IMotionStabilizer> motionStabilizer = stab->motionStabilizer(); 350 motionStabilizer.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev); 351 } else { 352 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>(); 353 Ptr<MotionFilterBase> motionFilter = stab->motionFilter(); 354 motionFilter.dynamicCast<GaussianMotionFilter>()->setParams(radius, stdev); 355 } 356 stabilizer_->setRadius(radius); 357 } 358 359 void 360 VideoStabilizer::configDeblurrer(int radius, double sensitivity) 361 { 362 Ptr<DeblurerBase> deblurrer = stabilizer_->deblurrer(); 363 if (NULL == deblurrer) { 364 return; 365 } 366 367 deblurrer->setRadius(radius); 368 deblurrer.dynamicCast<WeightingDeblurer>()->setSensitivity(sensitivity); 369 } 370 371 void 372 VideoStabilizer::setFrameSize(Size frameSize) 373 { 374 frameSize_ = frameSize; 375 } 376 377 Mat 378 VideoStabilizer::nextFrame() 379 { 380 if (isTwoPass_) { 381 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>(); 382 return stab->nextFrame(); 383 } else { 384 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>(); 385 return stab->nextFrame(); 386 } 387 } 388 389 Mat 390 VideoStabilizer::nextStabilizedMotion(DvsData* frame, int& stablizedPos) 391 { 392 Mat HMatrix; 393 394 if (isTwoPass_) { 395 Ptr<TwoPassVideoStabilizer> stab = stabilizer_.dynamicCast<TwoPassVideoStabilizer>(); 396 HMatrix = stab->nextStabilizedMotion(frame, stablizedPos); 397 } else { 398 Ptr<OnePassVideoStabilizer> stab = stabilizer_.dynamicCast<OnePassVideoStabilizer>(); 399 HMatrix = stab->nextStabilizedMotion(frame, stablizedPos); 400 } 401 402 return HMatrix; 403 } 404 405 Size 406 VideoStabilizer::trimedVideoSize(Size frameSize) 407 { 408 Size outputFrameSize; 409 outputFrameSize.width = ((int)((float)frameSize.width * (1 - 2 * trimRatio_)) >> 3) << 3; 410 outputFrameSize.height = ((int)((float)frameSize.height * (1 - 2 * trimRatio_)) >> 3) << 3; 411 412 return (outputFrameSize); 413 } 414 415 Mat 416 VideoStabilizer::cropVideoFrame(Mat& frame) 417 { 418 Rect cropROI; 419 Size inputFrameSize = frame.size(); 420 Size outputFrameSize = trimedVideoSize(inputFrameSize); 421 422 cropROI.x = (inputFrameSize.width - outputFrameSize.width) >> 1; 423 cropROI.y = (inputFrameSize.height - outputFrameSize.height) >> 1; 424 cropROI.width = outputFrameSize.width; 425 cropROI.height = outputFrameSize.height; 426 427 Mat croppedFrame = frame(cropROI).clone(); 428 429 return croppedFrame; 430 } 431 432
