1 /* 2 * cl_image_360_stitch.cpp - CL Image 360 stitch 3 * 4 * Copyright (c) 2016 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Wind Yuan <feng.yuan (at) intel.com> 19 */ 20 21 #include "cl_utils.h" 22 #include "cl_image_360_stitch.h" 23 #if HAVE_OPENCV 24 #include "cv_feature_match.h" 25 #endif 26 27 #define XCAM_BLENDER_GLOBAL_SCALE_EXT_WIDTH 64 28 29 #define STITCH_CHECK(ret, msg, ...) \ 30 if ((ret) != XCAM_RETURN_NO_ERROR) { \ 31 XCAM_LOG_WARNING (msg, ## __VA_ARGS__); \ 32 return ret; \ 33 } 34 35 namespace XCam { 36 37 CLBlenderGlobalScaleKernel::CLBlenderGlobalScaleKernel ( 38 const SmartPtr<CLContext> &context, SmartPtr<CLImage360Stitch> &stitch, bool is_uv) 39 : CLBlenderScaleKernel (context, is_uv) 40 , _stitch (stitch) 41 { 42 } 43 44 SmartPtr<CLImage> 45 CLBlenderGlobalScaleKernel::get_input_image () { 46 SmartPtr<CLContext> context = get_context (); 47 SmartPtr<VideoBuffer> input = _stitch->get_global_scale_input (); 48 49 CLImageDesc cl_desc; 50 SmartPtr<CLImage> cl_image; 51 const VideoBufferInfo &buf_info = input->get_video_info (); 52 53 cl_desc.format.image_channel_data_type = CL_UNORM_INT8; 54 if (_is_uv) { 55 cl_desc.format.image_channel_order = CL_RG; 56 cl_desc.width = buf_info.width / 2; 57 cl_desc.height = buf_info.height / 2; 58 cl_desc.row_pitch = buf_info.strides[1]; 59 cl_image = convert_to_climage (context, input, cl_desc, buf_info.offsets[1]); 60 } else { 61 cl_desc.format.image_channel_order = CL_R; 62 cl_desc.width = buf_info.width; 63 cl_desc.height = buf_info.height; 64 cl_desc.row_pitch = buf_info.strides[0]; 65 cl_image = convert_to_climage (context, input, cl_desc, buf_info.offsets[0]); 66 } 67 68 return cl_image; 69 } 70 71 SmartPtr<CLImage> 72 CLBlenderGlobalScaleKernel::get_output_image () { 73 SmartPtr<CLContext> context = get_context (); 74 SmartPtr<VideoBuffer> output = _stitch->get_global_scale_output (); 75 76 CLImageDesc cl_desc; 77 SmartPtr<CLImage> cl_image; 78 const VideoBufferInfo &buf_info = output->get_video_info (); 79 80 cl_desc.format.image_channel_data_type = CL_UNSIGNED_INT16; 81 cl_desc.format.image_channel_order = CL_RGBA; 82 if (_is_uv) { 83 cl_desc.width = buf_info.width / 8; 84 cl_desc.height = buf_info.height / 2; 85 cl_desc.row_pitch = buf_info.strides[1]; 86 cl_image = convert_to_climage (context, output, cl_desc, buf_info.offsets[1]); 87 } else { 88 cl_desc.width = buf_info.width / 8; 89 cl_desc.height = buf_info.height; 90 cl_desc.row_pitch = buf_info.strides[0]; 91 cl_image = convert_to_climage (context, output, cl_desc, buf_info.offsets[0]); 92 } 93 94 return cl_image; 95 } 96 97 bool 98 CLBlenderGlobalScaleKernel::get_output_info ( 99 uint32_t &out_width, uint32_t &out_height, int &out_offset_x) 100 { 101 SmartPtr<VideoBuffer> output = _stitch->get_global_scale_output (); 102 const VideoBufferInfo &output_info = output->get_video_info (); 103 104 out_width = output_info.width / 8; 105 out_height = _is_uv ? output_info.height / 2 : output_info.height; 106 out_offset_x = 0; 107 108 return true; 109 } 110 111 #if HAVE_OPENCV 112 static CVFMConfig 113 get_fm_default_config (StitchResMode res_mode) 114 { 115 CVFMConfig config; 116 117 switch (res_mode) { 118 case StitchRes1080P: { 119 config.sitch_min_width = 56; 120 config.min_corners = 8; 121 config.offset_factor = 0.8f; 122 config.delta_mean_offset = 5.0f; 123 config.recur_offset_error = 8.0f; 124 config.max_adjusted_offset = 12.0f; 125 config.max_valid_offset_y = 8.0f; 126 config.max_track_error = 24.0f; 127 128 break; 129 } 130 case StitchRes1080P4: { 131 config.sitch_min_width = 128; 132 config.min_corners = 4; 133 config.offset_factor = 0.8f; 134 config.delta_mean_offset = 24.0f; 135 config.recur_offset_error = 12.0f; 136 config.max_adjusted_offset = 24.0f; 137 config.max_valid_offset_y = 64.0f; 138 config.max_track_error = 32.0f; 139 140 break; 141 } 142 case StitchRes4K: { 143 config.sitch_min_width = 160; 144 config.min_corners = 8; 145 config.offset_factor = 0.8f; 146 config.delta_mean_offset = 5.0f; 147 config.recur_offset_error = 8.0f; 148 config.max_adjusted_offset = 12.0f; 149 config.max_valid_offset_y = 8.0f; 150 config.max_track_error = 24.0f; 151 152 break; 153 } 154 default: 155 XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode); 156 break; 157 } 158 159 return config; 160 } 161 #endif 162 163 static StitchInfo 164 get_default_stitch_info (StitchResMode res_mode) 165 { 166 StitchInfo stitch_info; 167 168 switch (res_mode) { 169 case StitchRes1080P: { 170 stitch_info.merge_width[0] = 56; 171 stitch_info.merge_width[1] = 56; 172 173 stitch_info.crop[0].left = 96; 174 stitch_info.crop[0].right = 96; 175 stitch_info.crop[0].top = 0; 176 stitch_info.crop[0].bottom = 0; 177 stitch_info.crop[1].left = 96; 178 stitch_info.crop[1].right = 96; 179 stitch_info.crop[1].top = 0; 180 stitch_info.crop[1].bottom = 0; 181 182 stitch_info.fisheye_info[0].center_x = 480.0f; 183 stitch_info.fisheye_info[0].center_y = 480.0f; 184 stitch_info.fisheye_info[0].wide_angle = 202.8f; 185 stitch_info.fisheye_info[0].radius = 480.0f; 186 stitch_info.fisheye_info[0].rotate_angle = -90.0f; 187 stitch_info.fisheye_info[1].center_x = 1440.0f; 188 stitch_info.fisheye_info[1].center_y = 480.0f; 189 stitch_info.fisheye_info[1].wide_angle = 202.8f; 190 stitch_info.fisheye_info[1].radius = 480.0f; 191 stitch_info.fisheye_info[1].rotate_angle = 89.4f; 192 break; 193 } 194 case StitchRes1080P4: { 195 stitch_info.merge_width[0] = 288; 196 stitch_info.merge_width[1] = 288; 197 stitch_info.merge_width[2] = 288; 198 stitch_info.merge_width[3] = 288; 199 200 stitch_info.crop[0].left = 0; 201 stitch_info.crop[0].right = 0; 202 stitch_info.crop[0].top = 0; 203 stitch_info.crop[0].bottom = 0; 204 stitch_info.crop[1].left = 0; 205 stitch_info.crop[1].right = 0; 206 stitch_info.crop[1].top = 0; 207 stitch_info.crop[1].bottom = 0; 208 stitch_info.crop[2].left = 0; 209 stitch_info.crop[2].right = 0; 210 stitch_info.crop[2].top = 0; 211 stitch_info.crop[2].bottom = 0; 212 stitch_info.crop[3].left = 0; 213 stitch_info.crop[3].right = 0; 214 stitch_info.crop[3].top = 0; 215 stitch_info.crop[3].bottom = 0; 216 217 stitch_info.fisheye_info[0].center_x = 640.0f; 218 stitch_info.fisheye_info[0].center_y = 400.0f; 219 stitch_info.fisheye_info[0].wide_angle = 120.0f; 220 stitch_info.fisheye_info[0].radius = 640.0f; 221 stitch_info.fisheye_info[0].rotate_angle = 0.0f; 222 stitch_info.fisheye_info[1].center_x = 640.0f; 223 stitch_info.fisheye_info[1].center_y = 400.0f; 224 stitch_info.fisheye_info[1].wide_angle = 120.0f; 225 stitch_info.fisheye_info[1].radius = 640.0f; 226 stitch_info.fisheye_info[1].rotate_angle = 0.0f; 227 stitch_info.fisheye_info[2].center_x = 640.0f; 228 stitch_info.fisheye_info[2].center_y = 400.0f; 229 stitch_info.fisheye_info[2].wide_angle = 120.0f; 230 stitch_info.fisheye_info[2].radius = 640.0f; 231 stitch_info.fisheye_info[2].rotate_angle = 0.0f; 232 stitch_info.fisheye_info[3].center_x = 640.0f; 233 stitch_info.fisheye_info[3].center_y = 400.0f; 234 stitch_info.fisheye_info[3].wide_angle = 120.0f; 235 stitch_info.fisheye_info[3].radius = 640.0f; 236 stitch_info.fisheye_info[3].rotate_angle = 0.0f; 237 break; 238 } 239 case StitchRes4K: { 240 stitch_info.merge_width[0] = 160; 241 stitch_info.merge_width[1] = 160; 242 243 stitch_info.crop[0].left = 64; 244 stitch_info.crop[0].right = 64; 245 stitch_info.crop[0].top = 0; 246 stitch_info.crop[0].bottom = 0; 247 stitch_info.crop[1].left = 64; 248 stitch_info.crop[1].right = 64; 249 stitch_info.crop[1].top = 0; 250 stitch_info.crop[1].bottom = 0; 251 252 stitch_info.fisheye_info[0].center_x = 1024.0f; 253 stitch_info.fisheye_info[0].center_y = 1024.0f; 254 stitch_info.fisheye_info[0].wide_angle = 195.0f; 255 stitch_info.fisheye_info[0].radius = 1040.0f; 256 stitch_info.fisheye_info[0].rotate_angle = 0.0f; 257 258 stitch_info.fisheye_info[1].center_x = 3072.0f; 259 stitch_info.fisheye_info[1].center_y = 1016.0f; 260 stitch_info.fisheye_info[1].wide_angle = 192.0f; 261 stitch_info.fisheye_info[1].radius = 1040.0f; 262 stitch_info.fisheye_info[1].rotate_angle = 0.4f; 263 break; 264 } 265 default: 266 XCAM_LOG_DEBUG ("unknown reslution mode (%d)", res_mode); 267 break; 268 } 269 270 return stitch_info; 271 } 272 273 CLImage360Stitch::CLImage360Stitch ( 274 const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, SurroundMode surround_mode, 275 StitchResMode res_mode, int fisheye_num, bool all_in_one_img) 276 : CLMultiImageHandler (context, "CLImage360Stitch") 277 , _context (context) 278 , _output_width (0) 279 , _output_height (0) 280 , _scale_mode (scale_mode) 281 , _surround_mode (surround_mode) 282 , _res_mode (res_mode) 283 , _is_stitch_inited (false) 284 , _fisheye_num (fisheye_num) 285 , _all_in_one_img (all_in_one_img) 286 { 287 #if HAVE_OPENCV 288 for (int i = 0; i < fisheye_num; i++) { 289 _feature_match[i] = new CVFeatureMatch (); 290 XCAM_ASSERT (_feature_match[i].ptr ()); 291 _feature_match[i]->set_config (get_fm_default_config (res_mode)); 292 _feature_match[i]->set_fm_index (i); 293 } 294 #endif 295 } 296 297 bool 298 CLImage360Stitch::set_stitch_info (StitchInfo stitch_info) 299 { 300 if (_is_stitch_inited) { 301 XCAM_LOG_WARNING ("stitching info was initialized and can't be set twice"); 302 return false; 303 } 304 305 for (int index = 0; index < _fisheye_num; ++index) { 306 _fisheye[index].handler->set_fisheye_info (stitch_info.fisheye_info[index]); 307 } 308 309 _stitch_info = stitch_info; 310 _is_stitch_inited = true; 311 312 return true; 313 } 314 315 StitchInfo 316 CLImage360Stitch::get_stitch_info () 317 { 318 if (!_is_stitch_inited) { 319 XCAM_LOG_WARNING ("stitch-info was not initialized, return default parameters"); 320 return get_default_stitch_info (_res_mode); 321 } 322 323 return _stitch_info; 324 } 325 326 bool 327 CLImage360Stitch::set_fisheye_handler (SmartPtr<CLFisheyeHandler> fisheye, int index) 328 { 329 XCAM_ASSERT (index < _fisheye_num); 330 331 _fisheye[index].handler = fisheye; 332 SmartPtr<CLImageHandler> handler = fisheye; 333 return add_image_handler (handler); 334 } 335 336 bool 337 CLImage360Stitch::set_blender (SmartPtr<CLBlender> blender, int idx) 338 { 339 _blender[idx] = blender; 340 341 SmartPtr<CLImageHandler> handler = blender; 342 return add_image_handler (handler); 343 } 344 345 void 346 CLImage360Stitch::set_fisheye_intrinsic (IntrinsicParameter intrinsic_param, int index) 347 { 348 _fisheye[index].handler->set_intrinsic_param(intrinsic_param); 349 } 350 351 void 352 CLImage360Stitch::set_fisheye_extrinsic (ExtrinsicParameter extrinsic_param, int index) 353 { 354 _fisheye[index].handler->set_extrinsic_param(extrinsic_param); 355 } 356 357 const BowlDataConfig & 358 CLImage360Stitch::get_fisheye_bowl_config (int index) 359 { 360 XCAM_ASSERT (index < _fisheye_num); 361 return _fisheye[index].handler->get_bowl_config (); 362 } 363 364 bool 365 CLImage360Stitch::set_image_overlap (const int idx, const Rect &overlap0, const Rect &overlap1) 366 { 367 XCAM_ASSERT (idx < _fisheye_num); 368 _overlaps[idx][0] = overlap0; 369 _overlaps[idx][1] = overlap1; 370 return true; 371 } 372 373 void 374 CLImage360Stitch::set_feature_match_ocl (bool fm_ocl) 375 { 376 #if HAVE_OPENCV 377 for (int i = 0; i < _fisheye_num; i++) { 378 _feature_match[i]->set_ocl (fm_ocl); 379 } 380 #else 381 XCAM_UNUSED (fm_ocl); 382 XCAM_LOG_WARNING ("non-OpenCV mode, failed to set ocl for feature match"); 383 #endif 384 } 385 386 #if HAVE_OPENCV 387 void 388 CLImage360Stitch::set_feature_match_config (const int idx, CVFMConfig config) 389 { 390 _feature_match[idx]->set_config (config); 391 } 392 393 CVFMConfig 394 CLImage360Stitch::get_feature_match_config (const int idx) 395 { 396 return _feature_match[idx]->get_config (); 397 } 398 #endif 399 400 void 401 CLImage360Stitch::calc_fisheye_initial_info (SmartPtr<VideoBuffer> &output) 402 { 403 const VideoBufferInfo &out_info = output->get_video_info (); 404 405 if(_surround_mode == SphereView) { 406 uint32_t fisheye_width_sum = out_info.width; 407 for (int i = 0; i < _fisheye_num; i++) { 408 fisheye_width_sum += _stitch_info.merge_width[i] + _stitch_info.crop[i].left + _stitch_info.crop[i].right; 409 } 410 _fisheye[0].width = fisheye_width_sum / _fisheye_num; 411 _fisheye[0].width = XCAM_ALIGN_UP (_fisheye[0].width, 16); 412 _fisheye[0].height = out_info.height + _stitch_info.crop[0].top + _stitch_info.crop[0].bottom; 413 XCAM_LOG_INFO ( 414 "fisheye correction output size width:%d height:%d", 415 _fisheye[0].width, _fisheye[0].height); 416 417 for (int i = 1; i < _fisheye_num; i++) { 418 _fisheye[i].width = _fisheye[0].width; 419 _fisheye[i].height = _fisheye[0].height; 420 } 421 422 float max_dst_longitude, max_dst_latitude; 423 for (int i = 0; i < _fisheye_num; ++i) { 424 max_dst_latitude = (_stitch_info.fisheye_info[i].wide_angle > 180.0f) ? 425 180.0f : _stitch_info.fisheye_info[i].wide_angle; 426 max_dst_longitude = max_dst_latitude * _fisheye[i].width / _fisheye[i].height; 427 428 _fisheye[i].handler->set_dst_range (max_dst_longitude, max_dst_latitude); 429 _fisheye[i].handler->set_output_size (_fisheye[i].width, _fisheye[i].height); 430 } 431 } else { 432 _fisheye[0].height = out_info.height + _stitch_info.crop[0].top + _stitch_info.crop[0].bottom; 433 434 float view_angle[XCAM_STITCH_FISHEYE_MAX_NUM]; 435 436 view_angle[0] = 68.0f; 437 _fisheye[0].width = view_angle[0] / 360.0f * out_info.width; 438 _fisheye[0].width = XCAM_ALIGN_UP (_fisheye[0].width, 32); 439 440 view_angle[1] = 152.0f; 441 _fisheye[1].width = view_angle[1] / 360.0f * out_info.width; 442 _fisheye[1].width = XCAM_ALIGN_UP (_fisheye[1].width, 32); 443 444 view_angle[2] = 68.0f; 445 _fisheye[2].width = view_angle[2] / 360.0f * out_info.width; 446 _fisheye[2].width = XCAM_ALIGN_UP (_fisheye[2].width, 32); 447 448 view_angle[3] = 152.0f; 449 _fisheye[3].width = view_angle[3] / 360.0f * out_info.width; 450 _fisheye[3].width = XCAM_ALIGN_UP (_fisheye[3].width, 32); 451 452 XCAM_LOG_INFO ( 453 "fisheye correction output size width:%d height:%d", 454 _fisheye[0].width, _fisheye[0].height); 455 456 BowlDataConfig bowl_data_config[XCAM_STITCH_FISHEYE_MAX_NUM]; 457 458 bowl_data_config[0].angle_start = -view_angle[0] / 2; 459 bowl_data_config[0].angle_end = view_angle[0] / 2; 460 461 for (int i = 1; i < _fisheye_num; i++) { 462 _fisheye[i].height = _fisheye[0].height; 463 float angle_center = 360.0f / _fisheye_num * i; 464 bowl_data_config[i].angle_start = angle_center - view_angle[i] / 2; 465 bowl_data_config[i].angle_end = angle_center + view_angle[i] / 2; 466 } 467 468 for(int i = 0; i < _fisheye_num; i++) { 469 _fisheye[i].handler->set_bowl_config(bowl_data_config[i]); 470 _fisheye[i].handler->set_output_size (_fisheye[i].width, _fisheye[i].height); 471 } 472 473 for(int i = 0; i < _fisheye_num; i++) { 474 _stitch_info.merge_width[i] = XCAM_ALIGN_UP((uint32_t)(20.0f / 360.0f * out_info.width), 32); 475 } 476 } 477 } 478 479 void 480 CLImage360Stitch::update_image_overlap () 481 { 482 static bool is_merge_info_inited = false; 483 if (!is_merge_info_inited) { 484 int idx_next = 1; 485 for (int i = 0; i < _fisheye_num; i++) { 486 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); 487 488 _img_merge_info[i].left.pos_x = _stitch_info.crop[i].left; 489 _img_merge_info[i].left.pos_y = _stitch_info.crop[i].top; 490 _img_merge_info[i].left.width = _stitch_info.merge_width[i]; 491 _img_merge_info[i].left.height = _fisheye[i].height - _stitch_info.crop[i].top 492 - _stitch_info.crop[i].bottom; 493 494 _img_merge_info[i].right.pos_x = _fisheye[i].width - _stitch_info.crop[i].right 495 - _stitch_info.merge_width[idx_next]; 496 _img_merge_info[i].right.pos_y = _stitch_info.crop[i].top; 497 _img_merge_info[i].right.width = _stitch_info.merge_width[idx_next]; 498 _img_merge_info[i].right.height = _fisheye[i].height - _stitch_info.crop[i].top 499 - _stitch_info.crop[i].bottom; 500 } 501 502 is_merge_info_inited = true; 503 } 504 505 for (int i = 0; i < _fisheye_num; i++) { 506 set_image_overlap (i, _img_merge_info[i].left, _img_merge_info[i].right); 507 } 508 } 509 510 XCamReturn 511 CLImage360Stitch::prepare_buffer_pool_video_info ( 512 const VideoBufferInfo &input, VideoBufferInfo &output) 513 { 514 if (_output_width == 0 || _output_height == 0) { 515 XCAM_LOG_ERROR ("incorrect output size: width:%d height:%d", _output_width, _output_height); 516 return XCAM_RETURN_ERROR_PARAM; 517 } 518 519 // aligned at least XCAM_CL_BLENDER_ALIGNMENT_X 520 uint32_t aligned_width = XCAM_MAX (16, XCAM_CL_BLENDER_ALIGNMENT_X); 521 output.init ( 522 input.format, _output_width, _output_height, 523 XCAM_ALIGN_UP(_output_width, aligned_width), XCAM_ALIGN_UP(_output_height, 16)); 524 525 return XCAM_RETURN_NO_ERROR; 526 } 527 528 XCamReturn 529 CLImage360Stitch::ensure_fisheye_parameters ( 530 SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output) 531 { 532 static bool is_fisheye_inited = false; 533 534 if (!is_fisheye_inited) { 535 calc_fisheye_initial_info (output); 536 is_fisheye_inited = true; 537 } 538 539 SmartPtr<VideoBuffer> pre_buf; 540 SmartPtr<VideoBuffer> cur_buf = input; 541 for (int i = 0; i < _fisheye_num; i++) { 542 if (!_fisheye[i].pool.ptr ()) 543 create_buffer_pool (_fisheye[i].pool, _fisheye[i].width, _fisheye[i].height); 544 545 _fisheye[i].buf = _fisheye[i].pool->get_buffer (_fisheye[i].pool); 546 XCAM_ASSERT (_fisheye[i].buf.ptr ()); 547 548 XCamReturn ret = ensure_handler_parameters (_fisheye[i].handler, cur_buf, _fisheye[i].buf); 549 STITCH_CHECK (ret, "execute fisheye prepare_parameters failed"); 550 551 if (!_all_in_one_img) { 552 pre_buf = cur_buf; 553 cur_buf = cur_buf->find_typed_attach<VideoBuffer> (); 554 if (!cur_buf.ptr () && (i != (_fisheye_num - 1))) { 555 XCAM_LOG_ERROR ("conflicting attached buffers and fisheye number"); 556 return XCAM_RETURN_ERROR_PARAM; 557 } 558 pre_buf->detach_buffer (cur_buf); 559 } 560 } 561 562 return XCAM_RETURN_NO_ERROR; 563 } 564 565 XCamReturn 566 CLImage360Stitch::prepare_global_scale_blender_parameters ( 567 SmartPtr<VideoBuffer> &input0, SmartPtr<VideoBuffer> &input1, SmartPtr<VideoBuffer> &output, 568 int idx, int idx_next, int &cur_start_pos) 569 { 570 const VideoBufferInfo &in0_info = input0->get_video_info (); 571 const VideoBufferInfo &in1_info = input1->get_video_info (); 572 const VideoBufferInfo &out_info = output->get_video_info (); 573 574 XCAM_ASSERT (in0_info.height == in1_info.height); 575 XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width); 576 577 Rect left_lap = get_image_overlap (idx, 1); 578 Rect right_lap = get_image_overlap (idx_next, 0); 579 580 int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X); 581 int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X); 582 583 int32_t prev_pos; 584 prev_pos = left_lap.pos_x; 585 left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X); 586 left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_CL_BLENDER_ALIGNMENT_X); 587 right_lap.pos_x += left_lap.pos_x - prev_pos; 588 right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X); 589 right_lap.width = left_lap.width; 590 591 Rect area; 592 area.pos_y = left_lap.pos_y; 593 area.height = left_lap.height; 594 area.pos_x = left_img_mid; 595 area.width = left_lap.pos_x + left_lap.width - left_img_mid; 596 _blender[idx]->set_input_valid_area (area, 0); 597 598 area.pos_y = right_lap.pos_y; 599 area.height = right_lap.height; 600 area.pos_x = right_lap.pos_x; 601 area.width = right_img_mid - right_lap.pos_x; 602 _blender[idx]->set_input_valid_area (area, 1); 603 604 Rect out_merge_window; 605 out_merge_window.width = left_lap.width; 606 out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid); 607 out_merge_window.pos_y = 0; 608 out_merge_window.height = out_info.height; 609 _blender[idx]->set_merge_window (out_merge_window); 610 611 _blender[idx]->set_input_merge_area (left_lap, 0); 612 _blender[idx]->set_input_merge_area (right_lap, 1); 613 614 cur_start_pos += left_lap.pos_x - left_img_mid + right_img_mid - right_lap.pos_x; 615 return XCAM_RETURN_NO_ERROR; 616 } 617 618 XCamReturn 619 CLImage360Stitch::prepare_local_scale_blender_parameters ( 620 SmartPtr<VideoBuffer> &input0, SmartPtr<VideoBuffer> &input1, SmartPtr<VideoBuffer> &output, int idx, int idx_next) 621 { 622 const VideoBufferInfo &in0_info = input0->get_video_info (); 623 const VideoBufferInfo &in1_info = input1->get_video_info (); 624 const VideoBufferInfo &out_info = output->get_video_info (); 625 626 XCAM_ASSERT (in0_info.height == in1_info.height); 627 XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width); 628 629 Rect left_lap = get_image_overlap (idx, 1); 630 Rect right_lap = get_image_overlap (idx_next, 0); 631 632 int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X); 633 int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_CL_BLENDER_ALIGNMENT_X); 634 int cur_start_pos = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num * idx, XCAM_CL_BLENDER_ALIGNMENT_X); 635 int merge_std_width = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num, XCAM_CL_BLENDER_ALIGNMENT_X); 636 637 int32_t prev_pos; 638 prev_pos = left_lap.pos_x; 639 left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X); 640 left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_CL_BLENDER_ALIGNMENT_X); 641 right_lap.pos_x += left_lap.pos_x - prev_pos; 642 right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_CL_BLENDER_ALIGNMENT_X); 643 right_lap.width = left_lap.width; 644 645 Rect area; 646 area.pos_y = left_lap.pos_y; 647 area.height = left_lap.height; 648 area.pos_x = left_img_mid; 649 area.width = left_lap.pos_x + left_lap.width - left_img_mid; 650 _blender[idx]->set_input_valid_area (area, 0); 651 652 area.pos_y = right_lap.pos_y; 653 area.height = right_lap.height; 654 area.pos_x = right_lap.pos_x; 655 area.width = right_img_mid - right_lap.pos_x; 656 _blender[idx]->set_input_valid_area (area, 1); 657 658 Rect out_merge_window; 659 int delta_width = merge_std_width - (right_img_mid - right_lap.pos_x) - (left_lap.pos_x - left_img_mid); 660 out_merge_window.width = left_lap.width + delta_width; 661 out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid); 662 out_merge_window.pos_y = 0; 663 out_merge_window.height = out_info.height; 664 _blender[idx]->set_merge_window (out_merge_window); 665 666 _blender[idx]->set_input_merge_area (left_lap, 0); 667 _blender[idx]->set_input_merge_area (right_lap, 1); 668 669 return XCAM_RETURN_NO_ERROR; 670 } 671 672 bool 673 CLImage360Stitch::create_buffer_pool (SmartPtr<BufferPool> &buf_pool, uint32_t width, uint32_t height) 674 { 675 VideoBufferInfo buf_info; 676 width = XCAM_ALIGN_UP (width, 16); 677 buf_info.init (V4L2_PIX_FMT_NV12, width, height, 678 XCAM_ALIGN_UP (width, 16), XCAM_ALIGN_UP (height, 16)); 679 680 buf_pool = new CLVideoBufferPool (); 681 XCAM_ASSERT (buf_pool.ptr ()); 682 buf_pool->set_video_info (buf_info); 683 if (!buf_pool->reserve (6)) { 684 XCAM_LOG_ERROR ("CLImage360Stitch init buffer pool failed"); 685 return false; 686 } 687 688 return true; 689 } 690 691 XCamReturn 692 CLImage360Stitch::reset_buffer_info (SmartPtr<VideoBuffer> &input) 693 { 694 VideoBufferInfo reset_info; 695 const VideoBufferInfo &buf_info = input->get_video_info (); 696 697 uint32_t reset_width = 0; 698 for (int i = 0; i < _fisheye_num; i++) { 699 Rect img_left = get_image_overlap (i, 0); 700 Rect img_right = get_image_overlap (i, 1); 701 702 reset_width += img_right.pos_x - img_left.pos_x; 703 } 704 705 reset_width = XCAM_ALIGN_UP (reset_width, XCAM_CL_BLENDER_ALIGNMENT_X); 706 reset_info.init (buf_info.format, reset_width, buf_info.height, 707 buf_info.aligned_width, buf_info.aligned_height); 708 709 input->set_video_info (reset_info); 710 return XCAM_RETURN_NO_ERROR; 711 } 712 713 XCamReturn 714 CLImage360Stitch::prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output) 715 { 716 XCamReturn ret = XCAM_RETURN_NO_ERROR; 717 if (!_is_stitch_inited) 718 set_stitch_info (get_default_stitch_info (_res_mode)); 719 720 ret = ensure_fisheye_parameters (input, output); 721 STITCH_CHECK (ret, "ensure fisheye parameters failed"); 722 723 update_image_overlap (); 724 if (_scale_mode == CLBlenderScaleLocal) { 725 int idx_next = 1; 726 for (int i = 0; i < _fisheye_num; i++) { 727 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); 728 729 ret = prepare_local_scale_blender_parameters ( 730 _fisheye[i].buf, _fisheye[idx_next].buf, output, i, idx_next); 731 STITCH_CHECK (ret, "prepare local scale blender parameters failed"); 732 733 _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf); 734 ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, output); 735 STITCH_CHECK (ret, "blender: execute ensure_parameters failed"); 736 _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf); 737 } 738 } else { //global scale 739 const VideoBufferInfo &buf_info = output->get_video_info (); 740 if (!_scale_buf_pool.ptr ()) 741 create_buffer_pool (_scale_buf_pool, buf_info.width + XCAM_BLENDER_GLOBAL_SCALE_EXT_WIDTH, buf_info.height); 742 SmartPtr<VideoBuffer> scale_input = _scale_buf_pool->get_buffer (_scale_buf_pool); 743 XCAM_ASSERT (scale_input.ptr ()); 744 745 int idx_next = 1; 746 int cur_start_pos = 0; 747 for (int i = 0; i < _fisheye_num; i++) { 748 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); 749 750 ret = prepare_global_scale_blender_parameters ( 751 _fisheye[i].buf, _fisheye[idx_next].buf, scale_input, i, idx_next, cur_start_pos); 752 STITCH_CHECK (ret, "prepare global scale blender parameters failed"); 753 754 _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf); 755 ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, scale_input); 756 STITCH_CHECK (ret, "blender: execute ensure_parameters failed"); 757 _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf); 758 } 759 760 reset_buffer_info (scale_input); 761 _scale_global_input = scale_input; 762 _scale_global_output = output; 763 } 764 765 return XCAM_RETURN_NO_ERROR; 766 } 767 768 XCamReturn 769 CLImage360Stitch::execute_done (SmartPtr<VideoBuffer> &output) 770 { 771 #if HAVE_OPENCV 772 for (int i = 0; i < _fisheye_num; i++) { 773 if (!_feature_match[i]->is_ocl_path ()) { 774 get_context ()->finish (); 775 break; 776 } 777 } 778 #endif 779 780 _scale_global_input.release (); 781 _scale_global_output.release (); 782 783 return CLMultiImageHandler::execute_done (output); 784 } 785 786 static void 787 convert_to_stitch_rect (Rect xcam_rect, Rect &stitch_rect) 788 { 789 stitch_rect.pos_x = xcam_rect.pos_x; 790 stitch_rect.pos_y = xcam_rect.pos_y + xcam_rect.height / 3; 791 stitch_rect.width = xcam_rect.width; 792 stitch_rect.height = xcam_rect.height / 3; 793 } 794 795 static void 796 convert_to_xcam_rect (Rect stitch_rect, Rect &xcam_rect) 797 { 798 xcam_rect.pos_x = stitch_rect.pos_x; 799 xcam_rect.width = stitch_rect.width; 800 } 801 802 803 XCamReturn 804 CLImage360Stitch::sub_handler_execute_done (SmartPtr<CLImageHandler> &handler) 805 { 806 #if HAVE_OPENCV 807 XCAM_ASSERT (handler.ptr ()); 808 809 if (handler.ptr () == _fisheye[_fisheye_num - 1].handler.ptr ()) { 810 int idx_next = 1; 811 Rect crop_left, crop_right; 812 813 for (int i = 0; i < _fisheye_num; i++) { 814 idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); 815 816 convert_to_stitch_rect (_img_merge_info[i].right, crop_left); 817 convert_to_stitch_rect (_img_merge_info[idx_next].left, crop_right); 818 819 _feature_match[i]->optical_flow_feature_match ( 820 _fisheye[i].buf, _fisheye[idx_next].buf, crop_left, crop_right, _fisheye[i].width); 821 822 convert_to_xcam_rect (crop_left, _img_merge_info[i].right); 823 convert_to_xcam_rect (crop_right, _img_merge_info[idx_next].left); 824 } 825 } 826 #else 827 XCAM_UNUSED (handler); 828 #endif 829 830 return XCAM_RETURN_NO_ERROR; 831 } 832 833 static SmartPtr<CLImageKernel> 834 create_blender_global_scale_kernel ( 835 const SmartPtr<CLContext> &context, 836 SmartPtr<CLImage360Stitch> &stitch, 837 bool is_uv) 838 { 839 char transform_option[1024]; 840 snprintf (transform_option, sizeof(transform_option), "-DPYRAMID_UV=%d", is_uv ? 1 : 0); 841 842 static const XCamKernelInfo &kernel_info = { 843 "kernel_pyramid_scale", 844 #include "kernel_gauss_lap_pyramid.clx" 845 , 0 846 }; 847 848 SmartPtr<CLImageKernel> kernel; 849 kernel = new CLBlenderGlobalScaleKernel (context, stitch, is_uv); 850 XCAM_ASSERT (kernel.ptr ()); 851 XCAM_FAIL_RETURN ( 852 ERROR, 853 kernel->build_kernel (kernel_info, transform_option) == XCAM_RETURN_NO_ERROR, 854 NULL, 855 "load blender global scaling kernel(%s) failed", is_uv ? "UV" : "Y"); 856 857 return kernel; 858 } 859 860 SmartPtr<CLImageHandler> 861 create_image_360_stitch ( 862 const SmartPtr<CLContext> &context, bool need_seam, 863 CLBlenderScaleMode scale_mode, bool fisheye_map, bool need_lsc, SurroundMode surround_mode, 864 StitchResMode res_mode, int fisheye_num, bool all_in_one_img) 865 { 866 const int layer = 2; 867 const bool need_uv = true; 868 SmartPtr<CLFisheyeHandler> fisheye; 869 SmartPtr<CLBlender> blender; 870 SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch ( 871 context, scale_mode, surround_mode, res_mode, fisheye_num, all_in_one_img); 872 XCAM_ASSERT (stitch.ptr ()); 873 874 for (int index = 0; index < fisheye_num; ++index) { 875 fisheye = create_fisheye_handler (context, surround_mode, fisheye_map, need_lsc).dynamic_cast_ptr<CLFisheyeHandler> (); 876 XCAM_FAIL_RETURN (ERROR, fisheye.ptr (), NULL, "image_360_stitch create fisheye handler failed"); 877 fisheye->disable_buf_pool (true); 878 stitch->set_fisheye_handler (fisheye, index); 879 } 880 881 for (int index = 0; index < fisheye_num; ++index) { 882 blender = create_pyramid_blender (context, layer, need_uv, need_seam, scale_mode).dynamic_cast_ptr<CLBlender> (); 883 XCAM_FAIL_RETURN (ERROR, blender.ptr (), NULL, "image_360_stitch create blender failed"); 884 blender->disable_buf_pool (true); 885 stitch->set_blender (blender, index); 886 } 887 888 if (scale_mode == CLBlenderScaleGlobal) { 889 int max_plane = need_uv ? 2 : 1; 890 bool uv_status[2] = {false, true}; 891 for (int plane = 0; plane < max_plane; ++plane) { 892 SmartPtr<CLImageKernel> kernel = create_blender_global_scale_kernel (context, stitch, uv_status[plane]); 893 XCAM_FAIL_RETURN (ERROR, kernel.ptr (), NULL, "create blender global scaling kernel failed"); 894 stitch->add_kernel (kernel); 895 } 896 } 897 898 return stitch; 899 } 900 901 } 902 903
