1 #include <string> 2 #include <iostream> 3 #include <fstream> 4 #include <sstream> 5 #include <stdexcept> 6 #include "opencv2/core.hpp" 7 #include <opencv2/core/utility.hpp> 8 #include "opencv2/video.hpp" 9 #include "opencv2/imgproc.hpp" 10 #include "opencv2/videoio.hpp" 11 #include "opencv2/highgui.hpp" 12 #include "opencv2/videostab.hpp" 13 #include "opencv2/opencv_modules.hpp" 14 15 #define arg(name) cmd.get<string>(name) 16 #define argb(name) cmd.get<bool>(name) 17 #define argi(name) cmd.get<int>(name) 18 #define argf(name) cmd.get<float>(name) 19 #define argd(name) cmd.get<double>(name) 20 21 using namespace std; 22 using namespace cv; 23 using namespace cv::videostab; 24 25 Ptr<IFrameSource> stabilizedFrames; 26 string saveMotionsPath; 27 double outputFps; 28 string outputPath; 29 bool quietMode; 30 31 void run(); 32 void saveMotionsIfNecessary(); 33 void printHelp(); 34 MotionModel motionModel(const string &str); 35 36 37 void run() 38 { 39 VideoWriter writer; 40 Mat stabilizedFrame; 41 int nframes = 0; 42 43 // for each stabilized frame 44 while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty()) 45 { 46 nframes++; 47 48 // init writer (once) and save stabilized frame 49 if (!outputPath.empty()) 50 { 51 if (!writer.isOpened()) 52 writer.open(outputPath, VideoWriter::fourcc('X','V','I','D'), 53 outputFps, stabilizedFrame.size()); 54 writer << stabilizedFrame; 55 } 56 57 // show stabilized frame 58 if (!quietMode) 59 { 60 imshow("stabilizedFrame", stabilizedFrame); 61 char key = static_cast<char>(waitKey(3)); 62 if (key == 27) { cout << endl; break; } 63 } 64 } 65 66 cout << "processed frames: " << nframes << endl 67 << "finished\n"; 68 } 69 70 71 void printHelp() 72 { 73 cout << "OpenCV video stabilizer.\n" 74 "Usage: videostab <file_path> [arguments]\n\n" 75 "Arguments:\n" 76 " -m, --model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" 77 " Set motion model. The default is affine.\n" 78 " -lp, --lin-prog-motion-est=(yes|no)\n" 79 " Turn on/off LP based motion estimation. The default is no.\n" 80 " --subset=(<int_number>|auto)\n" 81 " Number of random samples per one motion hypothesis. The default is auto.\n" 82 " --thresh=(<float_number>|auto)\n" 83 " Maximum error to classify match as inlier. The default is auto.\n" 84 " --outlier-ratio=<float_number>\n" 85 " Motion estimation outlier ratio hypothesis. The default is 0.5.\n" 86 " --min-inlier-ratio=<float_number>\n" 87 " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n" 88 " --nkps=<int_number>\n" 89 " Number of keypoints to find in each frame. The default is 1000.\n" 90 " --local-outlier-rejection=(yes|no)\n" 91 " Perform local outlier rejection. The default is no.\n\n" 92 " -sm, --save-motions=(<file_path>|no)\n" 93 " Save estimated motions into file. The default is no.\n" 94 " -lm, --load-motions=(<file_path>|no)\n" 95 " Load motions from file. The default is no.\n\n" 96 " -r, --radius=<int_number>\n" 97 " Set sliding window radius. The default is 15.\n" 98 " --stdev=(<float_number>|auto)\n" 99 " Set smoothing weights standard deviation. The default is auto\n" 100 " (i.e. sqrt(radius)).\n" 101 " -lps, --lin-prog-stab=(yes|no)\n" 102 " Turn on/off linear programming based stabilization method.\n" 103 " --lps-trim-ratio=(<float_number>|auto)\n" 104 " Trimming ratio used in linear programming based method.\n" 105 " --lps-w1=(<float_number>|1)\n" 106 " 1st derivative weight. The default is 1.\n" 107 " --lps-w2=(<float_number>|10)\n" 108 " 2nd derivative weight. The default is 10.\n" 109 " --lps-w3=(<float_number>|100)\n" 110 " 3rd derivative weight. The default is 100.\n" 111 " --lps-w4=(<float_number>|100)\n" 112 " Non-translation motion components weight. The default is 100.\n\n" 113 " --deblur=(yes|no)\n" 114 " Do deblurring.\n" 115 " --deblur-sens=<float_number>\n" 116 " Set deblurring sensitivity (from 0 to +inf). The default is 0.1.\n\n" 117 " -t, --trim-ratio=<float_number>\n" 118 " Set trimming ratio (from 0 to 0.5). The default is 0.1.\n" 119 " -et, --est-trim=(yes|no)\n" 120 " Estimate trim ratio automatically. The default is yes.\n" 121 " -ic, --incl-constr=(yes|no)\n" 122 " Ensure the inclusion constraint is always satisfied. The default is no.\n\n" 123 " -bm, --border-mode=(replicate|reflect|const)\n" 124 " Set border extrapolation mode. The default is replicate.\n\n" 125 " --mosaic=(yes|no)\n" 126 " Do consistent mosaicing. The default is no.\n" 127 " --mosaic-stdev=<float_number>\n" 128 " Consistent mosaicing stdev threshold. The default is 10.0.\n\n" 129 " -mi, --motion-inpaint=(yes|no)\n" 130 " Do motion inpainting (requires CUDA support). The default is no.\n" 131 " --mi-dist-thresh=<float_number>\n" 132 " Estimated flow distance threshold for motion inpainting. The default is 5.0.\n\n" 133 " -ci, --color-inpaint=(no|average|ns|telea)\n" 134 " Do color inpainting. The defailt is no.\n" 135 " --ci-radius=<float_number>\n" 136 " Set color inpainting radius (for ns and telea options only).\n" 137 " The default is 2.0\n\n" 138 " -ws, --wobble-suppress=(yes|no)\n" 139 " Perform wobble suppression. The default is no.\n" 140 " --ws-lp=(yes|no)\n" 141 " Turn on/off LP based motion estimation. The default is no.\n" 142 " --ws-period=<int_number>\n" 143 " Set wobble suppression period. The default is 30.\n" 144 " --ws-model=(transl|transl_and_scale|rigid|similarity|affine|homography)\n" 145 " Set wobble suppression motion model (must have more DOF than motion \n" 146 " estimation model). The default is homography.\n" 147 " --ws-subset=(<int_number>|auto)\n" 148 " Number of random samples per one motion hypothesis. The default is auto.\n" 149 " --ws-thresh=(<float_number>|auto)\n" 150 " Maximum error to classify match as inlier. The default is auto.\n" 151 " --ws-outlier-ratio=<float_number>\n" 152 " Motion estimation outlier ratio hypothesis. The default is 0.5.\n" 153 " --ws-min-inlier-ratio=<float_number>\n" 154 " Minimum inlier ratio to decide if estimated motion is OK. The default is 0.1.\n" 155 " --ws-nkps=<int_number>\n" 156 " Number of keypoints to find in each frame. The default is 1000.\n" 157 " --ws-local-outlier-rejection=(yes|no)\n" 158 " Perform local outlier rejection. The default is no.\n\n" 159 " -sm2, --save-motions2=(<file_path>|no)\n" 160 " Save motions estimated for wobble suppression. The default is no.\n" 161 " -lm2, --load-motions2=(<file_path>|no)\n" 162 " Load motions for wobble suppression from file. The default is no.\n\n" 163 " -gpu=(yes|no)\n" 164 " Use CUDA optimization whenever possible. The default is no.\n\n" 165 " -o, --output=(no|<file_path>)\n" 166 " Set output file path explicitely. The default is stabilized.avi.\n" 167 " --fps=(<float_number>|auto)\n" 168 " Set output video FPS explicitely. By default the source FPS is used (auto).\n" 169 " -q, --quiet\n" 170 " Don't show output video frames.\n\n" 171 " -h, --help\n" 172 " Print help.\n\n" 173 "Note: some argument configurations lead to two passes, some to single pass.\n\n"; 174 } 175 176 // motion estimator builders are for concise creation of motion estimators 177 178 class IMotionEstimatorBuilder 179 { 180 public: 181 virtual ~IMotionEstimatorBuilder() {} 182 virtual Ptr<ImageMotionEstimatorBase> build() = 0; 183 protected: 184 IMotionEstimatorBuilder(CommandLineParser &command) : cmd(command) {} 185 CommandLineParser cmd; 186 }; 187 188 189 class MotionEstimatorRansacL2Builder : public IMotionEstimatorBuilder 190 { 191 public: 192 MotionEstimatorRansacL2Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "") 193 : IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {} 194 195 virtual Ptr<ImageMotionEstimatorBase> build() 196 { 197 Ptr<MotionEstimatorRansacL2> est = makePtr<MotionEstimatorRansacL2>(motionModel(arg(prefix + "model"))); 198 199 RansacParams ransac = est->ransacParams(); 200 if (arg(prefix + "subset") != "auto") 201 ransac.size = argi(prefix + "subset"); 202 if (arg(prefix + "thresh") != "auto") 203 ransac.thresh = argf(prefix + "thresh"); 204 ransac.eps = argf(prefix + "outlier-ratio"); 205 est->setRansacParams(ransac); 206 207 est->setMinInlierRatio(argf(prefix + "min-inlier-ratio")); 208 209 Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>(); 210 if (arg(prefix + "local-outlier-rejection") == "yes") 211 { 212 Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>(); 213 RansacParams ransacParams = tblor->ransacParams(); 214 if (arg(prefix + "thresh") != "auto") 215 ransacParams.thresh = argf(prefix + "thresh"); 216 tblor->setRansacParams(ransacParams); 217 outlierRejector = tblor; 218 } 219 220 #if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW) 221 if (gpu) 222 { 223 Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est); 224 kbest->setOutlierRejector(outlierRejector); 225 return kbest; 226 } 227 #endif 228 229 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est); 230 kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps"))); 231 kbest->setOutlierRejector(outlierRejector); 232 return kbest; 233 } 234 private: 235 bool gpu; 236 string prefix; 237 }; 238 239 240 class MotionEstimatorL1Builder : public IMotionEstimatorBuilder 241 { 242 public: 243 MotionEstimatorL1Builder(CommandLineParser &command, bool use_gpu, const string &_prefix = "") 244 : IMotionEstimatorBuilder(command), gpu(use_gpu), prefix(_prefix) {} 245 246 virtual Ptr<ImageMotionEstimatorBase> build() 247 { 248 Ptr<MotionEstimatorL1> est = makePtr<MotionEstimatorL1>(motionModel(arg(prefix + "model"))); 249 250 Ptr<IOutlierRejector> outlierRejector = makePtr<NullOutlierRejector>(); 251 if (arg(prefix + "local-outlier-rejection") == "yes") 252 { 253 Ptr<TranslationBasedLocalOutlierRejector> tblor = makePtr<TranslationBasedLocalOutlierRejector>(); 254 RansacParams ransacParams = tblor->ransacParams(); 255 if (arg(prefix + "thresh") != "auto") 256 ransacParams.thresh = argf(prefix + "thresh"); 257 tblor->setRansacParams(ransacParams); 258 outlierRejector = tblor; 259 } 260 261 #if defined(HAVE_OPENCV_CUDAIMGPROC) && defined(HAVE_OPENCV_CUDAOPTFLOW) 262 if (gpu) 263 { 264 Ptr<KeypointBasedMotionEstimatorGpu> kbest = makePtr<KeypointBasedMotionEstimatorGpu>(est); 265 kbest->setOutlierRejector(outlierRejector); 266 return kbest; 267 } 268 #endif 269 270 Ptr<KeypointBasedMotionEstimator> kbest = makePtr<KeypointBasedMotionEstimator>(est); 271 kbest->setDetector(GFTTDetector::create(argi(prefix + "nkps"))); 272 kbest->setOutlierRejector(outlierRejector); 273 return kbest; 274 } 275 private: 276 bool gpu; 277 string prefix; 278 }; 279 280 281 int main(int argc, const char **argv) 282 { 283 try 284 { 285 const char *keys = 286 "{ @1 | | }" 287 "{ m model | affine | }" 288 "{ lp lin-prog-motion-est | no | }" 289 "{ subset | auto | }" 290 "{ thresh | auto | }" 291 "{ outlier-ratio | 0.5 | }" 292 "{ min-inlier-ratio | 0.1 | }" 293 "{ nkps | 1000 | }" 294 "{ extra-kps | 0 | }" 295 "{ local-outlier-rejection | no | }" 296 "{ sm save-motions | no | }" 297 "{ lm load-motions | no | }" 298 "{ r radius | 15 | }" 299 "{ stdev | auto | }" 300 "{ lps lin-prog-stab | no | }" 301 "{ lps-trim-ratio | auto | }" 302 "{ lps-w1 | 1 | }" 303 "{ lps-w2 | 10 | }" 304 "{ lps-w3 | 100 | }" 305 "{ lps-w4 | 100 | }" 306 "{ deblur | no | }" 307 "{ deblur-sens | 0.1 | }" 308 "{ et est-trim | yes | }" 309 "{ t trim-ratio | 0.1 | }" 310 "{ ic incl-constr | no | }" 311 "{ bm border-mode | replicate | }" 312 "{ mosaic | no | }" 313 "{ ms mosaic-stdev | 10.0 | }" 314 "{ mi motion-inpaint | no | }" 315 "{ mi-dist-thresh | 5.0 | }" 316 "{ ci color-inpaint | no | }" 317 "{ ci-radius | 2 | }" 318 "{ ws wobble-suppress | no | }" 319 "{ ws-period | 30 | }" 320 "{ ws-model | homography | }" 321 "{ ws-subset | auto | }" 322 "{ ws-thresh | auto | }" 323 "{ ws-outlier-ratio | 0.5 | }" 324 "{ ws-min-inlier-ratio | 0.1 | }" 325 "{ ws-nkps | 1000 | }" 326 "{ ws-extra-kps | 0 | }" 327 "{ ws-local-outlier-rejection | no | }" 328 "{ ws-lp | no | }" 329 "{ sm2 save-motions2 | no | }" 330 "{ lm2 load-motions2 | no | }" 331 "{ gpu | no | }" 332 "{ o output | stabilized.avi | }" 333 "{ fps | auto | }" 334 "{ q quiet | | }" 335 "{ h help | | }"; 336 CommandLineParser cmd(argc, argv, keys); 337 338 // parse command arguments 339 340 if (argb("help")) 341 { 342 printHelp(); 343 return 0; 344 } 345 346 if (arg("gpu") == "yes") 347 { 348 cout << "initializing GPU..."; cout.flush(); 349 Mat hostTmp = Mat::zeros(1, 1, CV_32F); 350 cuda::GpuMat deviceTmp; 351 deviceTmp.upload(hostTmp); 352 cout << endl; 353 } 354 355 StabilizerBase *stabilizer = 0; 356 357 // check if source video is specified 358 359 string inputPath = arg(0); 360 if (inputPath.empty()) 361 throw runtime_error("specify video file path"); 362 363 // get source video parameters 364 365 Ptr<VideoFileSource> source = makePtr<VideoFileSource>(inputPath); 366 cout << "frame count (rough): " << source->count() << endl; 367 if (arg("fps") == "auto") 368 outputFps = source->fps(); 369 else 370 outputFps = argd("fps"); 371 372 // prepare motion estimation builders 373 374 Ptr<IMotionEstimatorBuilder> motionEstBuilder; 375 if (arg("lin-prog-motion-est") == "yes") 376 motionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes")); 377 else 378 motionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes")); 379 380 Ptr<IMotionEstimatorBuilder> wsMotionEstBuilder; 381 if (arg("ws-lp") == "yes") 382 wsMotionEstBuilder.reset(new MotionEstimatorL1Builder(cmd, arg("gpu") == "yes", "ws-")); 383 else 384 wsMotionEstBuilder.reset(new MotionEstimatorRansacL2Builder(cmd, arg("gpu") == "yes", "ws-")); 385 386 // determine whether we must use one pass or two pass stabilizer 387 bool isTwoPass = 388 arg("est-trim") == "yes" || arg("wobble-suppress") == "yes" || arg("lin-prog-stab") == "yes"; 389 390 if (isTwoPass) 391 { 392 // we must use two pass stabilizer 393 394 TwoPassStabilizer *twoPassStabilizer = new TwoPassStabilizer(); 395 stabilizer = twoPassStabilizer; 396 twoPassStabilizer->setEstimateTrimRatio(arg("est-trim") == "yes"); 397 398 // determine stabilization technique 399 400 if (arg("lin-prog-stab") == "yes") 401 { 402 Ptr<LpMotionStabilizer> stab = makePtr<LpMotionStabilizer>(); 403 stab->setFrameSize(Size(source->width(), source->height())); 404 stab->setTrimRatio(arg("lps-trim-ratio") == "auto" ? argf("trim-ratio") : argf("lps-trim-ratio")); 405 stab->setWeight1(argf("lps-w1")); 406 stab->setWeight2(argf("lps-w2")); 407 stab->setWeight3(argf("lps-w3")); 408 stab->setWeight4(argf("lps-w4")); 409 twoPassStabilizer->setMotionStabilizer(stab); 410 } 411 else if (arg("stdev") == "auto") 412 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius"))); 413 else 414 twoPassStabilizer->setMotionStabilizer(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev"))); 415 416 // init wobble suppressor if necessary 417 418 if (arg("wobble-suppress") == "yes") 419 { 420 Ptr<MoreAccurateMotionWobbleSuppressorBase> ws = makePtr<MoreAccurateMotionWobbleSuppressor>(); 421 if (arg("gpu") == "yes") 422 #ifdef HAVE_OPENCV_CUDAWARPING 423 ws = makePtr<MoreAccurateMotionWobbleSuppressorGpu>(); 424 #else 425 throw runtime_error("OpenCV is built without CUDA support"); 426 #endif 427 428 ws->setMotionEstimator(wsMotionEstBuilder->build()); 429 ws->setPeriod(argi("ws-period")); 430 twoPassStabilizer->setWobbleSuppressor(ws); 431 432 MotionModel model = ws->motionEstimator()->motionModel(); 433 if (arg("load-motions2") != "no") 434 { 435 ws->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions2"))); 436 ws->motionEstimator()->setMotionModel(model); 437 } 438 if (arg("save-motions2") != "no") 439 { 440 ws->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions2"), ws->motionEstimator())); 441 ws->motionEstimator()->setMotionModel(model); 442 } 443 } 444 } 445 else 446 { 447 // we must use one pass stabilizer 448 449 OnePassStabilizer *onePassStabilizer = new OnePassStabilizer(); 450 stabilizer = onePassStabilizer; 451 if (arg("stdev") == "auto") 452 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius"))); 453 else 454 onePassStabilizer->setMotionFilter(makePtr<GaussianMotionFilter>(argi("radius"), argf("stdev"))); 455 } 456 457 stabilizer->setFrameSource(source); 458 stabilizer->setMotionEstimator(motionEstBuilder->build()); 459 460 // cast stabilizer to simple frame source interface to read stabilized frames 461 stabilizedFrames.reset(dynamic_cast<IFrameSource*>(stabilizer)); 462 463 MotionModel model = stabilizer->motionEstimator()->motionModel(); 464 if (arg("load-motions") != "no") 465 { 466 stabilizer->setMotionEstimator(makePtr<FromFileMotionReader>(arg("load-motions"))); 467 stabilizer->motionEstimator()->setMotionModel(model); 468 } 469 if (arg("save-motions") != "no") 470 { 471 stabilizer->setMotionEstimator(makePtr<ToFileMotionWriter>(arg("save-motions"), stabilizer->motionEstimator())); 472 stabilizer->motionEstimator()->setMotionModel(model); 473 } 474 475 stabilizer->setRadius(argi("radius")); 476 477 // init deblurer 478 if (arg("deblur") == "yes") 479 { 480 Ptr<WeightingDeblurer> deblurer = makePtr<WeightingDeblurer>(); 481 deblurer->setRadius(argi("radius")); 482 deblurer->setSensitivity(argf("deblur-sens")); 483 stabilizer->setDeblurer(deblurer); 484 } 485 486 // set up trimming paramters 487 stabilizer->setTrimRatio(argf("trim-ratio")); 488 stabilizer->setCorrectionForInclusion(arg("incl-constr") == "yes"); 489 490 if (arg("border-mode") == "reflect") 491 stabilizer->setBorderMode(BORDER_REFLECT); 492 else if (arg("border-mode") == "replicate") 493 stabilizer->setBorderMode(BORDER_REPLICATE); 494 else if (arg("border-mode") == "const") 495 stabilizer->setBorderMode(BORDER_CONSTANT); 496 else 497 throw runtime_error("unknown border extrapolation mode: " 498 + cmd.get<string>("border-mode")); 499 500 // init inpainter 501 InpaintingPipeline *inpainters = new InpaintingPipeline(); 502 Ptr<InpainterBase> inpainters_(inpainters); 503 if (arg("mosaic") == "yes") 504 { 505 Ptr<ConsistentMosaicInpainter> inp = makePtr<ConsistentMosaicInpainter>(); 506 inp->setStdevThresh(argf("mosaic-stdev")); 507 inpainters->pushBack(inp); 508 } 509 if (arg("motion-inpaint") == "yes") 510 { 511 Ptr<MotionInpainter> inp = makePtr<MotionInpainter>(); 512 inp->setDistThreshold(argf("mi-dist-thresh")); 513 inpainters->pushBack(inp); 514 } 515 if (arg("color-inpaint") == "average") 516 inpainters->pushBack(makePtr<ColorAverageInpainter>()); 517 else if (arg("color-inpaint") == "ns") 518 inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_NS), argd("ci-radius"))); 519 else if (arg("color-inpaint") == "telea") 520 inpainters->pushBack(makePtr<ColorInpainter>(int(INPAINT_TELEA), argd("ci-radius"))); 521 else if (arg("color-inpaint") != "no") 522 throw runtime_error("unknown color inpainting method: " + arg("color-inpaint")); 523 if (!inpainters->empty()) 524 { 525 inpainters->setRadius(argi("radius")); 526 stabilizer->setInpainter(inpainters_); 527 } 528 529 if (arg("output") != "no") 530 outputPath = arg("output"); 531 532 quietMode = argb("quiet"); 533 534 run(); 535 } 536 catch (const exception &e) 537 { 538 cout << "error: " << e.what() << endl; 539 stabilizedFrames.release(); 540 return -1; 541 } 542 stabilizedFrames.release(); 543 return 0; 544 } 545 546 547 MotionModel motionModel(const string &str) 548 { 549 if (str == "transl") 550 return MM_TRANSLATION; 551 if (str == "transl_and_scale") 552 return MM_TRANSLATION_AND_SCALE; 553 if (str == "rigid") 554 return MM_RIGID; 555 if (str == "similarity") 556 return MM_SIMILARITY; 557 if (str == "affine") 558 return MM_AFFINE; 559 if (str == "homography") 560 return MM_HOMOGRAPHY; 561 throw runtime_error("unknown motion model: " + str); 562 } 563
