1 /* 2 * cl_post_image_processor.cpp - CL post image processor 3 * 4 * Copyright (c) 2015 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Wind Yuan <feng.yuan (at) intel.com> 19 * Author: Yinhang Liu <yinhangx.liu (at) intel.com> 20 */ 21 22 #include "cl_post_image_processor.h" 23 #include "cl_context.h" 24 25 #include "cl_tnr_handler.h" 26 #include "cl_retinex_handler.h" 27 #include "cl_defog_dcp_handler.h" 28 #include "cl_wavelet_denoise_handler.h" 29 #include "cl_newwavelet_denoise_handler.h" 30 #include "cl_3d_denoise_handler.h" 31 #include "cl_image_scaler.h" 32 #include "cl_wire_frame_handler.h" 33 #include "cl_csc_handler.h" 34 #include "cl_image_warp_handler.h" 35 #include "cl_image_360_stitch.h" 36 #include "cl_video_stabilizer.h" 37 38 #define XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE 6 39 #define XCAM_CL_POST_IMAGE_MAX_POOL_SIZE 12 40 41 namespace XCam { 42 43 CLPostImageProcessor::CLPostImageProcessor () 44 : CLImageProcessor ("CLPostImageProcessor") 45 , _output_fourcc (V4L2_PIX_FMT_NV12) 46 , _out_sample_type (OutSampleYuv) 47 , _scaler_factor (1.0) 48 , _tnr_mode (TnrYuv) 49 , _defog_mode (CLPostImageProcessor::DefogDisabled) 50 , _wavelet_basis (CL_WAVELET_DISABLED) 51 , _wavelet_channel (CL_IMAGE_CHANNEL_UV) 52 , _wavelet_bayes_shrink (false) 53 , _3d_denoise_mode (CLPostImageProcessor::Denoise3DDisabled) 54 , _3d_denoise_ref_count (3) 55 , _enable_scaler (false) 56 , _enable_wireframe (false) 57 , _enable_image_warp (false) 58 , _enable_stitch (false) 59 , _stitch_enable_seam (false) 60 , _stitch_fisheye_map (false) 61 , _stitch_lsc (false) 62 , _stitch_fm_ocl (false) 63 , _stitch_scale_mode (CLBlenderScaleLocal) 64 , _stitch_width (0) 65 , _stitch_height (0) 66 , _stitch_res_mode (0) 67 , _surround_mode (SphereView) 68 { 69 XCAM_LOG_DEBUG ("CLPostImageProcessor constructed"); 70 } 71 72 CLPostImageProcessor::~CLPostImageProcessor () 73 { 74 XCAM_LOG_DEBUG ("CLPostImageProcessor destructed"); 75 } 76 77 bool 78 CLPostImageProcessor::set_output_format (uint32_t fourcc) 79 { 80 switch (fourcc) { 81 case XCAM_PIX_FMT_RGBA64: 82 case V4L2_PIX_FMT_XBGR32: 83 case V4L2_PIX_FMT_ABGR32: 84 case V4L2_PIX_FMT_BGR32: 85 case V4L2_PIX_FMT_RGB32: 86 case V4L2_PIX_FMT_ARGB32: 87 case V4L2_PIX_FMT_XRGB32: 88 _out_sample_type = OutSampleRGB; 89 break; 90 case V4L2_PIX_FMT_NV12: 91 _out_sample_type = OutSampleYuv; 92 break; 93 default: 94 XCAM_LOG_WARNING ( 95 "cl post processor doesn't support output format: %s", 96 xcam_fourcc_to_string(fourcc)); 97 return false; 98 } 99 100 _output_fourcc = fourcc; 101 return true; 102 } 103 104 void 105 CLPostImageProcessor::set_stats_callback (const SmartPtr<StatsCallback> &callback) 106 { 107 XCAM_ASSERT (callback.ptr ()); 108 _stats_callback = callback; 109 } 110 111 bool 112 CLPostImageProcessor::set_scaler_factor (const double factor) 113 { 114 _scaler_factor = factor; 115 116 return true; 117 } 118 119 bool 120 CLPostImageProcessor::can_process_result (SmartPtr < X3aResult > & result) 121 { 122 if (!result.ptr ()) 123 return false; 124 125 switch (result->get_type ()) { 126 case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: 127 case XCAM_3A_RESULT_3D_NOISE_REDUCTION: 128 case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: 129 case XCAM_3A_RESULT_FACE_DETECTION: 130 case XCAM_3A_RESULT_DVS: 131 return true; 132 default: 133 return false; 134 } 135 136 return false; 137 } 138 139 XCamReturn 140 CLPostImageProcessor::apply_3a_results (X3aResultList &results) 141 { 142 XCamReturn ret = XCAM_RETURN_NO_ERROR; 143 144 for (X3aResultList::iterator iter = results.begin (); iter != results.end (); ++iter) 145 { 146 SmartPtr<X3aResult> &result = *iter; 147 ret = apply_3a_result (result); 148 if (ret != XCAM_RETURN_NO_ERROR) 149 break; 150 } 151 152 return ret; 153 } 154 155 XCamReturn 156 CLPostImageProcessor::apply_3a_result (SmartPtr<X3aResult> &result) 157 { 158 STREAM_LOCK; 159 160 if (!result.ptr ()) 161 return XCAM_RETURN_BYPASS; 162 163 uint32_t res_type = result->get_type (); 164 165 switch (res_type) { 166 case XCAM_3A_RESULT_TEMPORAL_NOISE_REDUCTION_YUV: { 167 SmartPtr<X3aTemporalNoiseReduction> tnr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> (); 168 XCAM_ASSERT (tnr_res.ptr ()); 169 if (_tnr.ptr ()) { 170 if (_defog_mode != CLPostImageProcessor::DefogDisabled) { 171 XCam3aResultTemporalNoiseReduction config; 172 xcam_mem_clear (config); 173 // isp processor 174 // config.gain = 0.12; 175 176 // cl processor 177 config.gain = 0.22; 178 179 config.threshold [0] = 0.00081; 180 config.threshold [1] = 0.00072; 181 _tnr->set_yuv_config (config); 182 } else { 183 _tnr->set_yuv_config (tnr_res->get_standard_result ()); 184 } 185 } 186 break; 187 } 188 case XCAM_3A_RESULT_3D_NOISE_REDUCTION: { 189 SmartPtr<X3aTemporalNoiseReduction> nr_res = result.dynamic_cast_ptr<X3aTemporalNoiseReduction> (); 190 XCAM_ASSERT (nr_res.ptr ()); 191 if (_3d_denoise.ptr ()) { 192 _3d_denoise->set_denoise_config (nr_res->get_standard_result ()); 193 } 194 break; 195 } 196 case XCAM_3A_RESULT_WAVELET_NOISE_REDUCTION: { 197 SmartPtr<X3aWaveletNoiseReduction> wavelet_res = result.dynamic_cast_ptr<X3aWaveletNoiseReduction> (); 198 XCAM_ASSERT (wavelet_res.ptr ()); 199 if (_wavelet.ptr()) { 200 _wavelet->set_denoise_config (wavelet_res->get_standard_result ()); 201 } 202 if (_newwavelet.ptr()) { 203 _newwavelet->set_denoise_config (wavelet_res->get_standard_result ()); 204 } 205 break; 206 } 207 case XCAM_3A_RESULT_FACE_DETECTION: { 208 SmartPtr<X3aFaceDetectionResult> fd_res = result.dynamic_cast_ptr<X3aFaceDetectionResult> (); 209 XCAM_ASSERT (fd_res.ptr ()); 210 if (_wireframe.ptr ()) { 211 _wireframe->set_wire_frame_config (fd_res->get_standard_result_ptr (), get_scaler_factor ()); 212 } 213 break; 214 } 215 case XCAM_3A_RESULT_DVS: { 216 SmartPtr<X3aDVSResult> dvs_res = result.dynamic_cast_ptr<X3aDVSResult> (); 217 XCAM_ASSERT (dvs_res.ptr ()); 218 if (_image_warp.ptr ()) { 219 _image_warp->set_warp_config (dvs_res->get_standard_result ()); 220 } 221 break; 222 } 223 default: 224 XCAM_LOG_WARNING ("CLPostImageProcessor unknown 3a result: %d", res_type); 225 break; 226 } 227 228 return XCAM_RETURN_NO_ERROR; 229 } 230 231 232 XCamReturn 233 CLPostImageProcessor::create_handlers () 234 { 235 SmartPtr<CLImageHandler> image_handler; 236 SmartPtr<CLContext> context = get_cl_context (); 237 238 XCAM_ASSERT (context.ptr ()); 239 240 /* defog: retinex */ 241 image_handler = create_cl_retinex_image_handler (context); 242 _retinex = image_handler.dynamic_cast_ptr<CLRetinexImageHandler> (); 243 XCAM_FAIL_RETURN ( 244 WARNING, 245 _retinex.ptr (), 246 XCAM_RETURN_ERROR_CL, 247 "CLPostImageProcessor create retinex handler failed"); 248 _retinex->enable_handler (_defog_mode == CLPostImageProcessor::DefogRetinex); 249 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 250 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 251 add_handler (image_handler); 252 253 /* defog: dark channel prior */ 254 image_handler = create_cl_defog_dcp_image_handler (context); 255 _defog_dcp = image_handler.dynamic_cast_ptr<CLDefogDcpImageHandler> (); 256 XCAM_FAIL_RETURN ( 257 WARNING, 258 _defog_dcp.ptr (), 259 XCAM_RETURN_ERROR_CL, 260 "CLPostImageProcessor create defog handler failed"); 261 _defog_dcp->enable_handler (_defog_mode == CLPostImageProcessor::DefogDarkChannelPrior); 262 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 263 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 264 add_handler (image_handler); 265 266 /* Temporal Noise Reduction */ 267 if (_defog_mode != CLPostImageProcessor::DefogDisabled) { 268 switch (_tnr_mode) { 269 case TnrYuv: { 270 image_handler = create_cl_tnr_image_handler (context, CL_TNR_TYPE_YUV); 271 _tnr = image_handler.dynamic_cast_ptr<CLTnrImageHandler> (); 272 XCAM_FAIL_RETURN ( 273 WARNING, 274 _tnr.ptr (), 275 XCAM_RETURN_ERROR_CL, 276 "CLPostImageProcessor create tnr handler failed"); 277 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 278 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE); 279 add_handler (image_handler); 280 break; 281 } 282 case TnrDisable: 283 XCAM_LOG_DEBUG ("CLPostImageProcessor disable tnr"); 284 break; 285 default: 286 XCAM_LOG_WARNING ("CLPostImageProcessor unknown tnr mode (%d)", _tnr_mode); 287 break; 288 } 289 } 290 291 /* wavelet denoise */ 292 switch (_wavelet_basis) { 293 case CL_WAVELET_HAT: { 294 image_handler = create_cl_wavelet_denoise_image_handler (context, _wavelet_channel); 295 _wavelet = image_handler.dynamic_cast_ptr<CLWaveletDenoiseImageHandler> (); 296 XCAM_FAIL_RETURN ( 297 WARNING, 298 _wavelet.ptr (), 299 XCAM_RETURN_ERROR_CL, 300 "CLPostImageProcessor create wavelet denoise handler failed"); 301 _wavelet->enable_handler (true); 302 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 303 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE); 304 add_handler (image_handler); 305 break; 306 } 307 case CL_WAVELET_HAAR: { 308 image_handler = create_cl_newwavelet_denoise_image_handler (context, _wavelet_channel, _wavelet_bayes_shrink); 309 _newwavelet = image_handler.dynamic_cast_ptr<CLNewWaveletDenoiseImageHandler> (); 310 XCAM_FAIL_RETURN ( 311 WARNING, 312 _newwavelet.ptr (), 313 XCAM_RETURN_ERROR_CL, 314 "CLPostImageProcessor create new wavelet denoise handler failed"); 315 _newwavelet->enable_handler (true); 316 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 317 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE); 318 add_handler (image_handler); 319 break; 320 } 321 case CL_WAVELET_DISABLED: 322 default : 323 XCAM_LOG_DEBUG ("unknown or disable wavelet (%d)", _wavelet_basis); 324 break; 325 } 326 327 /* 3D noise reduction */ 328 if (_3d_denoise_mode != CLPostImageProcessor::Denoise3DDisabled) { 329 uint32_t denoise_channel = CL_IMAGE_CHANNEL_UV; 330 331 if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DUV) { 332 denoise_channel = CL_IMAGE_CHANNEL_UV; 333 } else if (_3d_denoise_mode == CLPostImageProcessor::Denoise3DYuv) { 334 denoise_channel = CL_IMAGE_CHANNEL_Y | CL_IMAGE_CHANNEL_UV; 335 } 336 337 image_handler = create_cl_3d_denoise_image_handler (context, denoise_channel, _3d_denoise_ref_count); 338 _3d_denoise = image_handler.dynamic_cast_ptr<CL3DDenoiseImageHandler> (); 339 XCAM_FAIL_RETURN ( 340 WARNING, 341 _3d_denoise.ptr (), 342 XCAM_RETURN_ERROR_CL, 343 "CL3aImageProcessor create 3D noise reduction handler failed"); 344 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 345 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 346 image_handler->enable_handler (true); 347 add_handler (image_handler); 348 } 349 350 /* image scaler */ 351 image_handler = create_cl_image_scaler_handler (context, V4L2_PIX_FMT_NV12); 352 _scaler = image_handler.dynamic_cast_ptr<CLImageScaler> (); 353 XCAM_FAIL_RETURN ( 354 WARNING, 355 _scaler.ptr (), 356 XCAM_RETURN_ERROR_CL, 357 "CLPostImageProcessor create scaler handler failed"); 358 _scaler->set_scaler_factor (_scaler_factor, _scaler_factor); 359 _scaler->set_buffer_callback (_stats_callback); 360 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 361 image_handler->enable_handler (_enable_scaler); 362 add_handler (image_handler); 363 364 /* wire frame */ 365 image_handler = create_cl_wire_frame_image_handler (context); 366 _wireframe = image_handler.dynamic_cast_ptr<CLWireFrameImageHandler> (); 367 XCAM_FAIL_RETURN ( 368 WARNING, 369 _wireframe.ptr (), 370 XCAM_RETURN_ERROR_CL, 371 "CLPostImageProcessor create wire frame handler failed"); 372 _wireframe->enable_handler (_enable_wireframe); 373 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 374 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE); 375 add_handler (image_handler); 376 377 /* image warp */ 378 image_handler = create_cl_image_warp_handler (context); 379 _image_warp = image_handler.dynamic_cast_ptr<CLImageWarpHandler> (); 380 XCAM_FAIL_RETURN ( 381 WARNING, 382 _image_warp.ptr (), 383 XCAM_RETURN_ERROR_CL, 384 "CLPostImageProcessor create image warp handler failed"); 385 _image_warp->enable_handler (_enable_image_warp); 386 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 387 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 388 add_handler (image_handler); 389 390 /* video stabilization */ 391 image_handler = create_cl_video_stab_handler (context); 392 _video_stab = image_handler.dynamic_cast_ptr<CLVideoStabilizer> (); 393 XCAM_FAIL_RETURN ( 394 WARNING, 395 _video_stab.ptr (), 396 XCAM_RETURN_ERROR_CL, 397 "CLPostImageProcessor create video stabilizer failed"); 398 _video_stab->enable_handler (false); 399 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 400 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 401 add_handler (image_handler); 402 403 /* image stitch */ 404 image_handler = 405 create_image_360_stitch (context, _stitch_enable_seam, _stitch_scale_mode, 406 _stitch_fisheye_map, _stitch_lsc, (SurroundMode) _surround_mode, (StitchResMode) _stitch_res_mode); 407 _stitch = image_handler.dynamic_cast_ptr<CLImage360Stitch> (); 408 XCAM_FAIL_RETURN ( 409 WARNING, 410 _stitch.ptr (), 411 XCAM_RETURN_ERROR_CL, 412 "CLPostImageProcessor create image stitch handler failed"); 413 _stitch->set_output_size (_stitch_width, _stitch_height); 414 #if HAVE_OPENCV 415 _stitch->set_feature_match_ocl (_stitch_fm_ocl); 416 #endif 417 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 418 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_MAX_POOL_SIZE); 419 image_handler->enable_handler (_enable_stitch); 420 add_handler (image_handler); 421 422 /* csc (nv12torgba) */ 423 image_handler = create_cl_csc_image_handler (context, CL_CSC_TYPE_NV12TORGBA); 424 _csc = image_handler.dynamic_cast_ptr<CLCscImageHandler> (); 425 XCAM_FAIL_RETURN ( 426 WARNING, 427 _csc .ptr (), 428 XCAM_RETURN_ERROR_CL, 429 "CLPostImageProcessor create csc handler failed"); 430 _csc->enable_handler (_out_sample_type == OutSampleRGB); 431 _csc->set_output_format (_output_fourcc); 432 image_handler->set_pool_type (CLImageHandler::CLVideoPoolType); 433 image_handler->set_pool_size (XCAM_CL_POST_IMAGE_DEFAULT_POOL_SIZE); 434 add_handler (image_handler); 435 436 return XCAM_RETURN_NO_ERROR; 437 } 438 439 bool 440 CLPostImageProcessor::set_tnr (CLTnrMode mode) 441 { 442 _tnr_mode = mode; 443 444 STREAM_LOCK; 445 446 return true; 447 } 448 449 bool 450 CLPostImageProcessor::set_defog_mode (CLDefogMode mode) 451 { 452 _defog_mode = mode; 453 454 STREAM_LOCK; 455 456 return true; 457 } 458 459 bool 460 CLPostImageProcessor::set_wavelet (CLWaveletBasis basis, uint32_t channel, bool bayes_shrink) 461 { 462 _wavelet_basis = basis; 463 _wavelet_channel = (CLImageChannel) channel; 464 _wavelet_bayes_shrink = bayes_shrink; 465 466 STREAM_LOCK; 467 468 return true; 469 } 470 471 bool 472 CLPostImageProcessor::set_3ddenoise_mode (CL3DDenoiseMode mode, uint8_t ref_frame_count) 473 { 474 _3d_denoise_mode = mode; 475 _3d_denoise_ref_count = ref_frame_count; 476 477 STREAM_LOCK; 478 479 return true; 480 } 481 482 bool 483 CLPostImageProcessor::set_scaler (bool enable) 484 { 485 _enable_scaler = enable; 486 487 STREAM_LOCK; 488 489 return true; 490 } 491 492 bool 493 CLPostImageProcessor::set_wireframe (bool enable) 494 { 495 _enable_wireframe = enable; 496 497 STREAM_LOCK; 498 499 return true; 500 } 501 502 bool 503 CLPostImageProcessor::set_image_warp (bool enable) 504 { 505 _enable_image_warp = enable; 506 507 STREAM_LOCK; 508 509 return true; 510 } 511 512 bool 513 CLPostImageProcessor::set_image_stitch ( 514 bool enable_stitch, bool enable_seam, CLBlenderScaleMode scale_mode, bool enable_fisheye_map, 515 bool lsc, bool fm_ocl, uint32_t stitch_width, uint32_t stitch_height, uint32_t res_mode) 516 { 517 XCAM_ASSERT (scale_mode < CLBlenderScaleMax); 518 519 _enable_stitch = enable_stitch; 520 if (enable_stitch) 521 _stitch_enable_seam = enable_seam; 522 else 523 _stitch_enable_seam = false; 524 525 _stitch_scale_mode = scale_mode; 526 _stitch_fisheye_map = enable_fisheye_map; 527 _stitch_lsc = lsc; 528 _stitch_width = stitch_width; 529 _stitch_height = stitch_height; 530 _stitch_res_mode = res_mode; 531 532 #if HAVE_OPENCV 533 _stitch_fm_ocl = fm_ocl; 534 #else 535 XCAM_UNUSED (fm_ocl); 536 #endif 537 538 STREAM_LOCK; 539 540 return true; 541 } 542 543 }; 544
