1 /*M/////////////////////////////////////////////////////////////////////////////////////// 2 // 3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4 // 5 // By downloading, copying, installing or using the software you agree to this license. 6 // If you do not agree to this license, do not download, install, 7 // copy or use the software. 8 // 9 // 10 // License Agreement 11 // For Open Source Computer Vision Library 12 // 13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved. 14 // Third party copyrights are property of their respective owners. 15 // 16 // Redistribution and use in source and binary forms, with or without modification, 17 // are permitted provided that the following conditions are met: 18 // 19 // * Redistribution's of source code must retain the above copyright notice, 20 // this list of conditions and the following disclaimer. 21 // 22 // * Redistribution's in binary form must reproduce the above copyright notice, 23 // this list of conditions and the following disclaimer in the documentation 24 // and/or other materials provided with the distribution. 25 // 26 // * The name of the copyright holders may not be used to endorse or promote products 27 // derived from this software without specific prior written permission. 28 // 29 // This software is provided by the copyright holders and contributors "as is" and 30 // any express or implied warranties, including, but not limited to, the implied 31 // warranties of merchantability and fitness for a particular purpose are disclaimed. 32 // In no event shall the Intel Corporation or contributors be liable for any direct, 33 // indirect, incidental, special, exemplary, or consequential damages 34 // (including, but not limited to, procurement of substitute goods or services; 35 // loss of use, data, or profits; or business interruption) however caused 36 // and on any theory of liability, whether in contract, strict liability, 37 // or tort (including negligence or otherwise) arising in any way out of 38 // the use of this software, even if advised of the possibility of such damage. 39 // 40 //M*/ 41 42 #include "seamless_cloning.hpp" 43 44 using namespace cv; 45 using namespace std; 46 47 48 void Cloning::computeGradientX( const Mat &img, Mat &gx) 49 { 50 Mat kernel = Mat::zeros(1, 3, CV_8S); 51 kernel.at<char>(0,2) = 1; 52 kernel.at<char>(0,1) = -1; 53 54 if(img.channels() == 3) 55 { 56 filter2D(img, gx, CV_32F, kernel); 57 } 58 else if (img.channels() == 1) 59 { 60 Mat tmp[3]; 61 for(int chan = 0 ; chan < 3 ; ++chan) 62 { 63 filter2D(img, tmp[chan], CV_32F, kernel); 64 } 65 merge(tmp, 3, gx); 66 } 67 } 68 69 void Cloning::computeGradientY( const Mat &img, Mat &gy) 70 { 71 Mat kernel = Mat::zeros(3, 1, CV_8S); 72 kernel.at<char>(2,0) = 1; 73 kernel.at<char>(1,0) = -1; 74 75 if(img.channels() == 3) 76 { 77 filter2D(img, gy, CV_32F, kernel); 78 } 79 else if (img.channels() == 1) 80 { 81 Mat tmp[3]; 82 for(int chan = 0 ; chan < 3 ; ++chan) 83 { 84 filter2D(img, tmp[chan], CV_32F, kernel); 85 } 86 merge(tmp, 3, gy); 87 } 88 } 89 90 void Cloning::computeLaplacianX( const Mat &img, Mat &laplacianX) 91 { 92 Mat kernel = Mat::zeros(1, 3, CV_8S); 93 kernel.at<char>(0,0) = -1; 94 kernel.at<char>(0,1) = 1; 95 filter2D(img, laplacianX, CV_32F, kernel); 96 } 97 98 void Cloning::computeLaplacianY( const Mat &img, Mat &laplacianY) 99 { 100 Mat kernel = Mat::zeros(3, 1, CV_8S); 101 kernel.at<char>(0,0) = -1; 102 kernel.at<char>(1,0) = 1; 103 filter2D(img, laplacianY, CV_32F, kernel); 104 } 105 106 void Cloning::dst(const Mat& src, Mat& dest, bool invert) 107 { 108 Mat temp = Mat::zeros(src.rows, 2 * src.cols + 2, CV_32F); 109 110 int flag = invert ? DFT_ROWS + DFT_SCALE + DFT_INVERSE: DFT_ROWS; 111 112 src.copyTo(temp(Rect(1,0, src.cols, src.rows))); 113 114 for(int j = 0 ; j < src.rows ; ++j) 115 { 116 float * tempLinePtr = temp.ptr<float>(j); 117 const float * srcLinePtr = src.ptr<float>(j); 118 for(int i = 0 ; i < src.cols ; ++i) 119 { 120 tempLinePtr[src.cols + 2 + i] = - srcLinePtr[src.cols - 1 - i]; 121 } 122 } 123 124 Mat planes[] = {temp, Mat::zeros(temp.size(), CV_32F)}; 125 Mat complex; 126 127 merge(planes, 2, complex); 128 dft(complex, complex, flag); 129 split(complex, planes); 130 temp = Mat::zeros(src.cols, 2 * src.rows + 2, CV_32F); 131 132 for(int j = 0 ; j < src.cols ; ++j) 133 { 134 float * tempLinePtr = temp.ptr<float>(j); 135 for(int i = 0 ; i < src.rows ; ++i) 136 { 137 float val = planes[1].ptr<float>(i)[j + 1]; 138 tempLinePtr[i + 1] = val; 139 tempLinePtr[temp.cols - 1 - i] = - val; 140 } 141 } 142 143 Mat planes2[] = {temp, Mat::zeros(temp.size(), CV_32F)}; 144 145 merge(planes2, 2, complex); 146 dft(complex, complex, flag); 147 split(complex, planes2); 148 149 temp = planes2[1].t(); 150 dest = Mat::zeros(src.size(), CV_32F); 151 temp(Rect( 0, 1, src.cols, src.rows)).copyTo(dest); 152 } 153 154 void Cloning::idst(const Mat& src, Mat& dest) 155 { 156 dst(src, dest, true); 157 } 158 159 void Cloning::solve(const Mat &img, Mat& mod_diff, Mat &result) 160 { 161 const int w = img.cols; 162 const int h = img.rows; 163 164 Mat res; 165 dst(mod_diff, res); 166 167 for(int j = 0 ; j < h-2; j++) 168 { 169 float * resLinePtr = res.ptr<float>(j); 170 for(int i = 0 ; i < w-2; i++) 171 { 172 resLinePtr[i] /= (filter_X[i] + filter_Y[j] - 4); 173 } 174 } 175 176 idst(res, mod_diff); 177 178 unsigned char * resLinePtr = result.ptr<unsigned char>(0); 179 const unsigned char * imgLinePtr = img.ptr<unsigned char>(0); 180 const float * interpLinePtr = NULL; 181 182 //first col 183 for(int i = 0 ; i < w ; ++i) 184 result.ptr<unsigned char>(0)[i] = img.ptr<unsigned char>(0)[i]; 185 186 for(int j = 1 ; j < h-1 ; ++j) 187 { 188 resLinePtr = result.ptr<unsigned char>(j); 189 imgLinePtr = img.ptr<unsigned char>(j); 190 interpLinePtr = mod_diff.ptr<float>(j-1); 191 192 //first row 193 resLinePtr[0] = imgLinePtr[0]; 194 195 for(int i = 1 ; i < w-1 ; ++i) 196 { 197 //saturate cast is not used here, because it behaves differently from the previous implementation 198 //most notable, saturate_cast rounds before truncating, here it's the opposite. 199 float value = interpLinePtr[i-1]; 200 if(value < 0.) 201 resLinePtr[i] = 0; 202 else if (value > 255.0) 203 resLinePtr[i] = 255; 204 else 205 resLinePtr[i] = static_cast<unsigned char>(value); 206 } 207 208 //last row 209 resLinePtr[w-1] = imgLinePtr[w-1]; 210 } 211 212 //last col 213 resLinePtr = result.ptr<unsigned char>(h-1); 214 imgLinePtr = img.ptr<unsigned char>(h-1); 215 for(int i = 0 ; i < w ; ++i) 216 resLinePtr[i] = imgLinePtr[i]; 217 } 218 219 void Cloning::poissonSolver(const Mat &img, Mat &laplacianX , Mat &laplacianY, Mat &result) 220 { 221 const int w = img.cols; 222 const int h = img.rows; 223 224 Mat lap = Mat(img.size(),CV_32FC1); 225 226 lap = laplacianX + laplacianY; 227 228 Mat bound = img.clone(); 229 230 rectangle(bound, Point(1, 1), Point(img.cols-2, img.rows-2), Scalar::all(0), -1); 231 Mat boundary_points; 232 Laplacian(bound, boundary_points, CV_32F); 233 234 boundary_points = lap - boundary_points; 235 236 Mat mod_diff = boundary_points(Rect(1, 1, w-2, h-2)); 237 238 solve(img,mod_diff,result); 239 } 240 241 void Cloning::initVariables(const Mat &destination, const Mat &binaryMask) 242 { 243 destinationGradientX = Mat(destination.size(),CV_32FC3); 244 destinationGradientY = Mat(destination.size(),CV_32FC3); 245 patchGradientX = Mat(destination.size(),CV_32FC3); 246 patchGradientY = Mat(destination.size(),CV_32FC3); 247 248 binaryMaskFloat = Mat(binaryMask.size(),CV_32FC1); 249 binaryMaskFloatInverted = Mat(binaryMask.size(),CV_32FC1); 250 251 //init of the filters used in the dst 252 const int w = destination.cols; 253 filter_X.resize(w - 2); 254 for(int i = 0 ; i < w-2 ; ++i) 255 filter_X[i] = 2.0f * std::cos(static_cast<float>(CV_PI) * (i + 1) / (w - 1)); 256 257 const int h = destination.rows; 258 filter_Y.resize(h - 2); 259 for(int j = 0 ; j < h - 2 ; ++j) 260 filter_Y[j] = 2.0f * std::cos(static_cast<float>(CV_PI) * (j + 1) / (h - 1)); 261 } 262 263 void Cloning::computeDerivatives(const Mat& destination, const Mat &patch, const Mat &binaryMask) 264 { 265 initVariables(destination,binaryMask); 266 267 computeGradientX(destination,destinationGradientX); 268 computeGradientY(destination,destinationGradientY); 269 270 computeGradientX(patch,patchGradientX); 271 computeGradientY(patch,patchGradientY); 272 273 Mat Kernel(Size(3, 3), CV_8UC1); 274 Kernel.setTo(Scalar(1)); 275 erode(binaryMask, binaryMask, Kernel, Point(-1,-1), 3); 276 277 binaryMask.convertTo(binaryMaskFloat,CV_32FC1,1.0/255.0); 278 } 279 280 void Cloning::scalarProduct(Mat mat, float r, float g, float b) 281 { 282 vector <Mat> channels; 283 split(mat,channels); 284 multiply(channels[2],r,channels[2]); 285 multiply(channels[1],g,channels[1]); 286 multiply(channels[0],b,channels[0]); 287 merge(channels,mat); 288 } 289 290 void Cloning::arrayProduct(const cv::Mat& lhs, const cv::Mat& rhs, cv::Mat& result) const 291 { 292 vector <Mat> lhs_channels; 293 vector <Mat> result_channels; 294 295 split(lhs,lhs_channels); 296 split(result,result_channels); 297 298 for(int chan = 0 ; chan < 3 ; ++chan) 299 multiply(lhs_channels[chan],rhs,result_channels[chan]); 300 301 merge(result_channels,result); 302 } 303 304 void Cloning::poisson(const Mat &destination) 305 { 306 Mat laplacianX = Mat(destination.size(),CV_32FC3); 307 Mat laplacianY = Mat(destination.size(),CV_32FC3); 308 309 laplacianX = destinationGradientX + patchGradientX; 310 laplacianY = destinationGradientY + patchGradientY; 311 312 computeLaplacianX(laplacianX,laplacianX); 313 computeLaplacianY(laplacianY,laplacianY); 314 315 split(laplacianX,rgbx_channel); 316 split(laplacianY,rgby_channel); 317 318 split(destination,output); 319 320 for(int chan = 0 ; chan < 3 ; ++chan) 321 { 322 poissonSolver(output[chan], rgbx_channel[chan], rgby_channel[chan], output[chan]); 323 } 324 } 325 326 void Cloning::evaluate(const Mat &I, const Mat &wmask, const Mat &cloned) 327 { 328 bitwise_not(wmask,wmask); 329 330 wmask.convertTo(binaryMaskFloatInverted,CV_32FC1,1.0/255.0); 331 332 arrayProduct(destinationGradientX,binaryMaskFloatInverted, destinationGradientX); 333 arrayProduct(destinationGradientY,binaryMaskFloatInverted, destinationGradientY); 334 335 poisson(I); 336 337 merge(output,cloned); 338 } 339 340 void Cloning::normalClone(const Mat &destination, const Mat &patch, const Mat &binaryMask, Mat &cloned, int flag) 341 { 342 const int w = destination.cols; 343 const int h = destination.rows; 344 const int channel = destination.channels(); 345 const int n_elem_in_line = w * channel; 346 347 computeDerivatives(destination,patch,binaryMask); 348 349 switch(flag) 350 { 351 case NORMAL_CLONE: 352 arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); 353 arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); 354 break; 355 356 case MIXED_CLONE: 357 { 358 AutoBuffer<int> maskIndices(n_elem_in_line); 359 for (int i = 0; i < n_elem_in_line; ++i) 360 maskIndices[i] = i / channel; 361 362 for(int i=0;i < h; i++) 363 { 364 float * patchXLinePtr = patchGradientX.ptr<float>(i); 365 float * patchYLinePtr = patchGradientY.ptr<float>(i); 366 const float * destinationXLinePtr = destinationGradientX.ptr<float>(i); 367 const float * destinationYLinePtr = destinationGradientY.ptr<float>(i); 368 const float * binaryMaskLinePtr = binaryMaskFloat.ptr<float>(i); 369 370 for(int j=0; j < n_elem_in_line; j++) 371 { 372 int maskIndex = maskIndices[j]; 373 374 if(abs(patchXLinePtr[j] - patchYLinePtr[j]) > 375 abs(destinationXLinePtr[j] - destinationYLinePtr[j])) 376 { 377 patchXLinePtr[j] *= binaryMaskLinePtr[maskIndex]; 378 patchYLinePtr[j] *= binaryMaskLinePtr[maskIndex]; 379 } 380 else 381 { 382 patchXLinePtr[j] = destinationXLinePtr[j] 383 * binaryMaskLinePtr[maskIndex]; 384 patchYLinePtr[j] = destinationYLinePtr[j] 385 * binaryMaskLinePtr[maskIndex]; 386 } 387 } 388 } 389 } 390 break; 391 392 case MONOCHROME_TRANSFER: 393 Mat gray = Mat(patch.size(),CV_8UC1); 394 cvtColor(patch, gray, COLOR_BGR2GRAY ); 395 396 computeGradientX(gray,patchGradientX); 397 computeGradientY(gray,patchGradientY); 398 399 arrayProduct(patchGradientX, binaryMaskFloat, patchGradientX); 400 arrayProduct(patchGradientY, binaryMaskFloat, patchGradientY); 401 break; 402 403 } 404 405 evaluate(destination,binaryMask,cloned); 406 } 407 408 void Cloning::localColorChange(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float red_mul=1.0, 409 float green_mul=1.0, float blue_mul=1.0) 410 { 411 computeDerivatives(I,mask,wmask); 412 413 arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); 414 arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); 415 scalarProduct(patchGradientX,red_mul,green_mul,blue_mul); 416 scalarProduct(patchGradientY,red_mul,green_mul,blue_mul); 417 418 evaluate(I,wmask,cloned); 419 } 420 421 void Cloning::illuminationChange(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float alpha, float beta) 422 { 423 computeDerivatives(I,mask,wmask); 424 425 arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); 426 arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); 427 428 Mat mag = Mat(I.size(),CV_32FC3); 429 magnitude(patchGradientX,patchGradientY,mag); 430 431 Mat multX, multY, multx_temp, multy_temp; 432 433 multiply(patchGradientX,pow(alpha,beta),multX); 434 pow(mag,-1*beta, multx_temp); 435 multiply(multX,multx_temp, patchGradientX); 436 patchNaNs(patchGradientX); 437 438 multiply(patchGradientY,pow(alpha,beta),multY); 439 pow(mag,-1*beta, multy_temp); 440 multiply(multY,multy_temp,patchGradientY); 441 patchNaNs(patchGradientY); 442 443 Mat zeroMask = (patchGradientX != 0); 444 445 patchGradientX.copyTo(patchGradientX, zeroMask); 446 patchGradientY.copyTo(patchGradientY, zeroMask); 447 448 evaluate(I,wmask,cloned); 449 } 450 451 void Cloning::textureFlatten(Mat &I, Mat &mask, Mat &wmask, float low_threshold, 452 float high_threshold, int kernel_size, Mat &cloned) 453 { 454 computeDerivatives(I,mask,wmask); 455 456 Mat out = Mat(mask.size(),CV_8UC1); 457 Canny(mask,out,low_threshold,high_threshold,kernel_size); 458 459 Mat zeros(patchGradientX.size(), CV_32FC3); 460 zeros.setTo(0); 461 Mat zerosMask = (out != 255); 462 zeros.copyTo(patchGradientX, zerosMask); 463 zeros.copyTo(patchGradientY, zerosMask); 464 465 arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); 466 arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); 467 468 evaluate(I,wmask,cloned); 469 } 470