1 /* 2 * cl_tnr_handler.cpp - CL tnr handler 3 * 4 * Copyright (c) 2015 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Wei Zong <wei.zong (at) intel.com> 19 */ 20 21 #include "cl_tnr_handler.h" 22 23 #define TNR_PROCESSING_FRAME_COUNT 4 24 #define TNR_LIST_FRAME_COUNT 4 25 #define TNR_MOTION_THRESHOLD 2 26 27 namespace XCam { 28 29 static const XCamKernelInfo kernel_tnr_yuv_info = { 30 "kernel_tnr_yuv", 31 #include "kernel_tnr.clx" 32 , 0 33 }; 34 35 static const XCamKernelInfo kernel_tnr_rgb_info = { 36 "kernel_tnr_rgb", 37 #include "kernel_tnr.clx" 38 , 0, 39 }; 40 41 CLTnrImageHandler::CLTnrMotionInfo::CLTnrMotionInfo () 42 : hor_shift (0) 43 , ver_shift (0) 44 , hor_corr (0) 45 , ver_corr (0) 46 { 47 } 48 49 CLTnrImageHandler::CLTnrHistogram::CLTnrHistogram() { 50 hor_hist_bin = 0; 51 ver_hist_bin = 0; 52 hor_hist_current = NULL; 53 hor_hist_reference = NULL; 54 ver_hist_current = NULL; 55 ver_hist_reference = NULL; 56 }; 57 58 CLTnrImageHandler::CLTnrHistogram::CLTnrHistogram(uint32_t width, uint32_t height) { 59 hor_hist_bin = width; 60 ver_hist_bin = height; 61 if ((NULL == hor_hist_current) && (hor_hist_bin != 0)) { 62 hor_hist_current = (float*)xcam_malloc0(hor_hist_bin * sizeof(float)); 63 } 64 if ((NULL == ver_hist_current) && (ver_hist_bin != 0)) { 65 ver_hist_current = (float*)xcam_malloc0(ver_hist_bin * sizeof(float)); 66 } 67 if ((NULL == hor_hist_reference) && (hor_hist_bin != 0)) { 68 hor_hist_reference = (float*)xcam_malloc0(hor_hist_bin * sizeof(float)); 69 } 70 if ((NULL == ver_hist_reference) && (ver_hist_bin != 0)) { 71 ver_hist_reference = (float*)xcam_malloc0(ver_hist_bin * sizeof(float)); 72 } 73 }; 74 75 CLTnrImageHandler::CLTnrHistogram::~CLTnrHistogram() { 76 if (NULL != hor_hist_current) { 77 xcam_free(hor_hist_current); 78 hor_hist_current = NULL; 79 } 80 if (NULL != ver_hist_current) { 81 xcam_free(ver_hist_current); 82 ver_hist_current = NULL; 83 } 84 if (NULL != hor_hist_reference) { 85 xcam_free(hor_hist_reference); 86 hor_hist_reference = NULL; 87 } 88 if (NULL != ver_hist_reference) { 89 xcam_free(ver_hist_reference); 90 ver_hist_reference = NULL; 91 } 92 hor_hist_bin = 0; 93 ver_hist_bin = 0; 94 } 95 96 CLTnrImageKernel::CLTnrImageKernel ( 97 const SmartPtr<CLContext> &context, CLTnrType type) 98 : CLImageKernel (context) 99 , _type (type) 100 { 101 } 102 103 bool 104 CLTnrImageHandler::calculate_image_histogram (XCam3AStats* stats, CLTnrHistogramType type, float* histogram) 105 { 106 if ( NULL == stats || NULL == histogram ) { 107 return false; 108 } 109 110 uint32_t normalize_factor = (1 << stats->info.bit_depth) - 1; 111 uint32_t image_width = stats->info.width; 112 uint32_t image_height = stats->info.height; 113 uint32_t image_aligned_width = stats->info.aligned_width; 114 uint32_t hor_hist_bin = image_width; 115 uint32_t ver_hist_bin = image_height; 116 117 switch (type) { 118 case CL_TNR_HIST_HOR_PROJECTION : 119 for (uint32_t bin = 0; bin < hor_hist_bin; bin++) { 120 for (uint32_t row_index = 0; row_index < image_height; row_index++) { 121 histogram[bin] += (float)(stats->stats[row_index * image_aligned_width + bin].avg_y) 122 / (1.0 * normalize_factor); 123 } 124 } 125 break; 126 case CL_TNR_HIST_VER_PROJECTION : 127 for (uint32_t bin = 0; bin < ver_hist_bin; bin++) { 128 for (uint32_t col_index = 0; col_index < image_width; col_index++) { 129 histogram[bin] += (float)(stats->stats[col_index + bin * image_aligned_width].avg_y) 130 / (1.0 * normalize_factor); 131 } 132 } 133 break; 134 case CL_TNR_HIST_BRIGHTNESS : 135 for (uint32_t row_index = 0; row_index < image_height; row_index++) { 136 for (uint32_t col_index = 0; col_index < image_width; col_index++) { 137 uint8_t bin = (stats->stats[row_index * image_aligned_width + col_index].avg_y * 255) 138 / normalize_factor; 139 histogram[bin]++; 140 } 141 } 142 break; 143 default : 144 break; 145 } 146 147 return true; 148 } 149 150 bool 151 CLTnrImageHandler::calculate_image_histogram (SmartPtr<VideoBuffer> &input, CLTnrHistogramType type, float* histogram) 152 { 153 if ( NULL == histogram ) { 154 return false; 155 } 156 157 uint32_t normalize_factor = (1 << input->get_video_info ().color_bits) - 1; 158 uint32_t image_width = input->get_video_info ().width; 159 uint32_t image_height = input->get_video_info ().height; 160 uint32_t image_aligned_width = input->get_video_info ().aligned_width; 161 uint32_t stride = input->get_video_info ().strides[0]; 162 163 uint32_t hor_hist_bin = image_width; 164 uint32_t ver_hist_bin = image_height; 165 uint32_t pxiel_bytes = stride / image_aligned_width; 166 167 uint32_t format = input->get_video_info ().format; 168 if (XCAM_PIX_FMT_RGBA64 != format) { 169 XCAM_LOG_ERROR ("Only support RGBA64 format !"); 170 return false; 171 } 172 173 uint8_t* image_buffer = input->map(); 174 if (NULL == image_buffer) { 175 return false; 176 } 177 178 switch (type) { 179 case CL_TNR_HIST_HOR_PROJECTION : 180 for (uint32_t bin = 0; bin < hor_hist_bin; bin++) { 181 for (uint32_t row_index = 0; row_index < image_height; row_index++) { 182 histogram[bin] += (float)(image_buffer[row_index * stride + pxiel_bytes * bin] + 183 (image_buffer[row_index * stride + pxiel_bytes * bin + 1] << 8) + 184 image_buffer[row_index * stride + pxiel_bytes * bin + 2] + 185 (image_buffer[row_index * stride + pxiel_bytes * bin + 3] << 8) + 186 image_buffer[row_index * stride + pxiel_bytes * bin + 4] + 187 (image_buffer[row_index * stride + pxiel_bytes * bin + 5] << 8) ) 188 / (3.0 * normalize_factor); 189 } 190 } 191 break; 192 case CL_TNR_HIST_VER_PROJECTION : 193 for (uint32_t bin = 0; bin < ver_hist_bin; bin++) { 194 for (uint32_t col_index = 0; col_index < stride; col_index += pxiel_bytes) { 195 histogram[bin] += (float)(image_buffer[col_index + bin * stride] + 196 (image_buffer[col_index + bin * stride + 1] << 8) + 197 image_buffer[col_index + bin * stride + 2] + 198 (image_buffer[col_index + bin * stride + 3] << 8) + 199 image_buffer[col_index + bin * stride + 4] + 200 (image_buffer[col_index + bin * stride + 5] << 8) ) 201 / (3.0 * normalize_factor); 202 } 203 } 204 break; 205 case CL_TNR_HIST_BRIGHTNESS : 206 for (uint32_t row_index = 0; row_index < image_height; row_index++) { 207 for (uint32_t col_index = 0; col_index < stride; col_index += pxiel_bytes) { 208 uint8_t bin = (image_buffer[row_index * stride + col_index] + 209 (image_buffer[row_index * stride + col_index + 1] << 8) + 210 image_buffer[row_index * stride + col_index + 2] + 211 (image_buffer[row_index * stride + col_index + 3] << 8) + 212 image_buffer[row_index * stride + col_index + 4] + 213 (image_buffer[row_index * stride + col_index + 5] << 8) ) * 255 214 / (3 * normalize_factor); 215 histogram[bin]++; 216 } 217 } 218 break; 219 default : 220 break; 221 } 222 223 input->unmap(); 224 225 return true; 226 } 227 228 void 229 CLTnrImageHandler::print_image_histogram () 230 { 231 uint32_t hor_hist_bin = _image_histogram.hor_hist_bin; 232 uint32_t ver_hist_bin = _image_histogram.ver_hist_bin; 233 234 XCAM_LOG_DEBUG ("hor hist bin = %d, ver hist bin = %d", hor_hist_bin, ver_hist_bin); 235 236 printf("float hor_hist_current[] = { "); 237 for (uint32_t i = 0; i < hor_hist_bin; i++) { 238 printf("%f, ", _image_histogram.hor_hist_current[i]); 239 } 240 printf(" }; \n\n\n"); 241 242 printf("float ver_hist_current[] = { "); 243 for (uint32_t i = 0; i < ver_hist_bin; i++) { 244 printf("%f, ", _image_histogram.ver_hist_current[i]); 245 } 246 printf(" }; \n\n\n"); 247 248 printf("float hor_hist_reference[] = { "); 249 for (uint32_t i = 0; i < hor_hist_bin; i++) { 250 printf("%f, ", _image_histogram.hor_hist_reference[i]); 251 } 252 printf(" }; \n\n\n"); 253 254 printf("float ver_hist_reference[] = { "); 255 for (uint32_t i = 0; i < ver_hist_bin; i++) { 256 printf("%f, ", _image_histogram.ver_hist_reference[i]); 257 } 258 printf(" }; \n\n\n"); 259 } 260 261 CLTnrImageHandler::CLTnrImageHandler (const SmartPtr<CLContext> &context, CLTnrType type, const char *name) 262 : CLImageHandler (context, name) 263 , _type (type) 264 , _gain_yuv (1.0) 265 , _thr_y (0.05) 266 , _thr_uv (0.05) 267 , _gain_rgb (0.0) 268 , _thr_r (0.064) // set high initial threshold to get strong denoise effect 269 , _thr_g (0.045) 270 , _thr_b (0.073) 271 , _frame_count (TNR_PROCESSING_FRAME_COUNT) 272 { 273 } 274 275 bool 276 CLTnrImageHandler::set_tnr_kernel(SmartPtr<CLTnrImageKernel> &kernel) 277 { 278 SmartPtr<CLImageKernel> image_kernel = kernel; 279 add_kernel (image_kernel); 280 _tnr_kernel = kernel; 281 return true; 282 } 283 284 bool 285 CLTnrImageHandler::set_framecount (uint8_t count) 286 { 287 if (!_tnr_kernel->is_valid ()) { 288 XCAM_LOG_ERROR ("set framecount error, invalid TNR kernel !"); 289 return false; 290 } 291 292 XCAM_ASSERT (count >= 2 && count <= 4); 293 _frame_count = count; 294 295 return true; 296 } 297 298 bool 299 CLTnrImageHandler::set_rgb_config (const XCam3aResultTemporalNoiseReduction& config) 300 301 { 302 if (!_tnr_kernel->is_valid ()) { 303 XCAM_LOG_ERROR ("set threshold error, invalid TNR kernel !"); 304 return false; 305 } 306 _gain_rgb = (float)config.gain; 307 _thr_r = (float)config.threshold[0]; 308 _thr_g = (float)config.threshold[1]; 309 _thr_b = (float)config.threshold[2]; 310 XCAM_LOG_DEBUG ("set TNR RGB config: _gain(%f), _thr_r(%f), _thr_g(%f), _thr_b(%f)", 311 _gain_rgb, _thr_r, _thr_g, _thr_b); 312 313 return true; 314 } 315 316 bool 317 CLTnrImageHandler::set_yuv_config (const XCam3aResultTemporalNoiseReduction& config) 318 319 { 320 if (!_tnr_kernel->is_valid ()) { 321 XCAM_LOG_ERROR ("set threshold error, invalid TNR kernel !"); 322 return false; 323 } 324 325 _gain_yuv = (float)config.gain; 326 _thr_y = (float)config.threshold[0]; 327 _thr_uv = (float)config.threshold[1]; 328 329 XCAM_LOG_DEBUG ("set TNR YUV config: _gain(%f), _thr_y(%f), _thr_uv(%f)", 330 _gain_yuv, _thr_y, _thr_uv); 331 332 return true; 333 } 334 335 XCamReturn 336 CLTnrImageHandler::prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output) 337 { 338 SmartPtr<CLContext> context = get_context (); 339 const VideoBufferInfo & video_info = input->get_video_info (); 340 CLArgList args; 341 CLWorkSize work_size; 342 XCamReturn ret = XCAM_RETURN_NO_ERROR; 343 344 XCAM_ASSERT (_tnr_kernel.ptr ()); 345 346 CLImageDesc desc; 347 if (CL_TNR_TYPE_YUV == _type) { 348 desc.format.image_channel_order = CL_R; 349 desc.format.image_channel_data_type = CL_UNORM_INT8; 350 desc.width = video_info.aligned_width; 351 desc.height = video_info.aligned_height + video_info.height / 2; 352 desc.row_pitch = video_info.strides[0]; 353 desc.array_size = 2; 354 desc.slice_pitch = video_info.strides [0] * video_info.aligned_height; 355 } else if (CL_TNR_TYPE_RGB == _type) { 356 desc.format.image_channel_order = CL_RGBA; 357 desc.format.image_channel_data_type = CL_UNORM_INT8; 358 desc.width = video_info.aligned_width; 359 desc.height = video_info.height; 360 desc.row_pitch = video_info.strides[0]; 361 desc.array_size = 0; 362 desc.slice_pitch = 0; 363 } 364 365 SmartPtr<CLImage> image_in = convert_to_climage (context, input, desc); 366 SmartPtr<CLImage> image_out = convert_to_climage (context, output, desc); 367 368 XCAM_FAIL_RETURN ( 369 WARNING, 370 image_in->is_valid () && image_out->is_valid (), 371 XCAM_RETURN_ERROR_MEM, 372 "cl image kernel(%s) in/out memory not available", _tnr_kernel->get_kernel_name ()); 373 374 if (CL_TNR_TYPE_YUV == _type) { 375 if (!_image_out_prev.ptr ()) { 376 _image_out_prev = image_in; 377 } 378 } else if (CL_TNR_TYPE_RGB == _type) { 379 // analyze motion between the latest adjacent two frames 380 // Todo: enable analyze when utilize motion compensation next step 381 382 if (_image_in_list.size () < TNR_LIST_FRAME_COUNT) { 383 while (_image_in_list.size () < TNR_LIST_FRAME_COUNT) { 384 _image_in_list.push_back (image_in); 385 } 386 } else { 387 _image_in_list.pop_front (); 388 _image_in_list.push_back (image_in); 389 } 390 } 391 392 uint32_t vertical_offset = video_info.aligned_height; 393 394 //set args; 395 work_size.dim = XCAM_DEFAULT_IMAGE_DIM; 396 work_size.local[0] = 8; 397 work_size.local[1] = 4; 398 if (CL_TNR_TYPE_YUV == _type) { 399 args.push_back (new CLMemArgument (image_in)); 400 args.push_back (new CLMemArgument (_image_out_prev)); 401 args.push_back (new CLMemArgument (image_out)); 402 args.push_back (new CLArgumentT<uint> (vertical_offset)); 403 404 args.push_back (new CLArgumentT<float> (_gain_yuv)); 405 args.push_back (new CLArgumentT<float> (_thr_y)); 406 args.push_back (new CLArgumentT<float> (_thr_uv)); 407 408 work_size.global[0] = video_info.width / 2; 409 work_size.global[1] = video_info.height / 2; 410 } 411 else if (CL_TNR_TYPE_RGB == _type) { 412 const CLImageDesc out_info = image_out->get_image_desc (); 413 work_size.global[0] = out_info.width; 414 work_size.global[1] = out_info.height; 415 416 args.push_back (new CLMemArgument (image_out)); 417 args.push_back (new CLArgumentT<float> (_gain_rgb)); 418 args.push_back (new CLArgumentT<float> (_thr_r)); 419 args.push_back (new CLArgumentT<float> (_thr_g)); 420 args.push_back (new CLArgumentT<float> (_thr_b)); 421 args.push_back (new CLArgumentT<uint8_t> (_frame_count)); 422 423 for (std::list<SmartPtr<CLImage>>::iterator it = _image_in_list.begin (); it != _image_in_list.end (); it++) { 424 args.push_back (new CLMemArgument (*it)); 425 } 426 } 427 428 XCAM_ASSERT (_tnr_kernel.ptr ()); 429 ret = _tnr_kernel->set_arguments (args, work_size); 430 XCAM_FAIL_RETURN ( 431 WARNING, ret == XCAM_RETURN_NO_ERROR, ret, 432 "tnr kernel set arguments failed."); 433 434 _image_out_prev = image_out; 435 return XCAM_RETURN_NO_ERROR; 436 } 437 438 SmartPtr<CLImageHandler> 439 create_cl_tnr_image_handler (const SmartPtr<CLContext> &context, CLTnrType type) 440 { 441 SmartPtr<CLTnrImageHandler> tnr_handler; 442 SmartPtr<CLTnrImageKernel> tnr_kernel; 443 XCamReturn ret = XCAM_RETURN_NO_ERROR; 444 445 tnr_kernel = new CLTnrImageKernel (context, type); 446 XCAM_ASSERT (tnr_kernel.ptr ()); 447 if (CL_TNR_TYPE_YUV == type) { 448 ret = tnr_kernel->build_kernel (kernel_tnr_yuv_info, NULL); 449 } else if (CL_TNR_TYPE_RGB == type) { 450 ret = tnr_kernel->build_kernel (kernel_tnr_rgb_info, NULL); 451 } else { 452 XCAM_LOG_ERROR ("create cl tnr image handler failed, unknown type:%d", type); 453 return NULL; 454 } 455 456 XCAM_FAIL_RETURN ( 457 ERROR, ret == XCAM_RETURN_NO_ERROR, NULL, 458 "build tnr kernel failed"); 459 460 tnr_handler = new CLTnrImageHandler (context, type, "cl_handler_tnr"); 461 XCAM_ASSERT (tnr_kernel->is_valid ()); 462 tnr_handler->set_tnr_kernel (tnr_kernel); 463 464 return tnr_handler; 465 } 466 467 }; 468
