1 /* 2 * soft_stitcher.cpp - soft stitcher implementation 3 * 4 * Copyright (c) 2017 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Wind Yuan <feng.yuan (at) intel.com> 19 */ 20 21 #include "soft_stitcher.h" 22 #include "soft_blender.h" 23 #include "soft_geo_mapper.h" 24 #include "soft_video_buf_allocator.h" 25 #include "interface/feature_match.h" 26 #include "surview_fisheye_dewarp.h" 27 #include "soft_copy_task.h" 28 #include "xcam_utils.h" 29 #include <map> 30 31 #define ENABLE_FEATURE_MATCH HAVE_OPENCV 32 33 #if ENABLE_FEATURE_MATCH 34 #include "cv_capi_feature_match.h" 35 #ifndef ANDROID 36 #include <opencv2/core/ocl.hpp> 37 #endif 38 #endif 39 40 #define SOFT_STITCHER_ALIGNMENT_X 8 41 #define SOFT_STITCHER_ALIGNMENT_Y 4 42 43 #define MAP_FACTOR_X 16 44 #define MAP_FACTOR_Y 16 45 46 #define DUMP_STITCHER 0 47 48 namespace XCam { 49 50 #if DUMP_STITCHER 51 static void 52 stitcher_dump_buf (const SmartPtr<VideoBuffer> buf, uint32_t idx, const char *prefix) 53 { 54 XCAM_ASSERT (prefix); 55 char name[256]; 56 snprintf (name, 256, "%s-%d", prefix, idx); 57 dump_buf_perfix_path (buf, name); 58 } 59 #else 60 static void stitcher_dump_buf (const SmartPtr<VideoBuffer> buf, ...) { 61 XCAM_UNUSED (buf); 62 } 63 #endif 64 65 66 namespace SoftSitcherPriv { 67 68 DECLARE_HANDLER_CALLBACK (CbGeoMap, SoftStitcher, dewarp_done); 69 DECLARE_HANDLER_CALLBACK (CbBlender, SoftStitcher, blender_done); 70 DECLARE_WORK_CALLBACK (CbCopyTask, SoftStitcher, copy_task_done); 71 72 struct BlenderParam 73 : SoftBlender::BlenderParam 74 { 75 SmartPtr<SoftStitcher::StitcherParam> stitch_param; 76 uint32_t idx; 77 78 BlenderParam ( 79 uint32_t i, 80 const SmartPtr<VideoBuffer> &in0, 81 const SmartPtr<VideoBuffer> &in1, 82 const SmartPtr<VideoBuffer> &out) 83 : SoftBlender::BlenderParam (in0, in1, out) 84 , idx (i) 85 {} 86 }; 87 88 typedef std::map<void*, SmartPtr<BlenderParam>> BlenderParams; 89 typedef std::map<void*, int32_t> BlendCopyTaskNums; 90 91 struct HandlerParam 92 : ImageHandler::Parameters 93 { 94 SmartPtr<SoftStitcher::StitcherParam> stitch_param; 95 uint32_t idx; 96 97 HandlerParam (uint32_t i) 98 : idx (i) 99 {} 100 }; 101 102 struct StitcherCopyArgs 103 : XCamSoftTasks::CopyTask::Args 104 { 105 uint32_t idx; 106 107 StitcherCopyArgs ( 108 uint32_t i, 109 const SmartPtr<ImageHandler::Parameters> ¶m) 110 : XCamSoftTasks::CopyTask::Args (param) 111 , idx (i) 112 {} 113 }; 114 115 struct Factor { 116 float x, y; 117 118 Factor () : x (1.0f), y (1.0f) {} 119 void reset () { 120 x = 1.0f; 121 y = 1.0f; 122 } 123 }; 124 125 struct Overlap { 126 SmartPtr<FeatureMatch> matcher; 127 SmartPtr<SoftBlender> blender; 128 BlenderParams param_map; 129 130 SmartPtr<BlenderParam> find_blender_param_in_map ( 131 const SmartPtr<SoftStitcher::StitcherParam> &key, 132 const uint32_t idx); 133 }; 134 135 struct FisheyeDewarp { 136 SmartPtr<SoftGeoMapper> dewarp; 137 SmartPtr<BufferPool> buf_pool; 138 Factor left_match_factor, right_match_factor; 139 140 bool set_dewarp_factor (); 141 XCamReturn set_dewarp_geo_table ( 142 SmartPtr<SoftGeoMapper> mapper, 143 const CameraInfo &cam_info, 144 const Stitcher::RoundViewSlice &view_slice, 145 const BowlDataConfig &bowl); 146 }; 147 148 struct Copier { 149 SmartPtr<XCamSoftTasks::CopyTask> copy_task; 150 Stitcher::CopyArea copy_area; 151 152 XCamReturn start_copy_task ( 153 const SmartPtr<ImageHandler::Parameters> ¶m, 154 const uint32_t idx, const SmartPtr<VideoBuffer> &buf); 155 }; 156 typedef std::vector<Copier> Copiers; 157 158 class StitcherImpl { 159 friend class XCam::SoftStitcher; 160 161 public: 162 StitcherImpl (SoftStitcher *handler) 163 : _stitcher (handler) 164 {} 165 166 XCamReturn init_config (uint32_t count); 167 168 bool remove_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m); 169 int32_t dec_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m); 170 171 XCamReturn start_dewarp_works (const SmartPtr<SoftStitcher::StitcherParam> ¶m); 172 XCamReturn start_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m); 173 XCamReturn start_overlap_tasks ( 174 const SmartPtr<SoftStitcher::StitcherParam> ¶m, 175 const uint32_t idx, const SmartPtr<VideoBuffer> &buf); 176 XCamReturn start_copy_tasks ( 177 const SmartPtr<SoftStitcher::StitcherParam> ¶m, 178 const uint32_t idx, const SmartPtr<VideoBuffer> &buf); 179 180 XCamReturn start_single_blender (const uint32_t idx, const SmartPtr<BlenderParam> ¶m); 181 XCamReturn stop (); 182 183 XCamReturn fisheye_dewarp_to_table (); 184 XCamReturn feature_match ( 185 const SmartPtr<VideoBuffer> &left_buf, 186 const SmartPtr<VideoBuffer> &right_buf, 187 const uint32_t idx); 188 189 bool get_and_reset_feature_match_factors (uint32_t idx, Factor &left, Factor &right); 190 191 private: 192 XCamReturn init_fisheye (uint32_t idx); 193 bool init_dewarp_factors (uint32_t idx); 194 XCamReturn create_copier (Stitcher::CopyArea area); 195 196 private: 197 FisheyeDewarp _fisheye [XCAM_STITCH_MAX_CAMERAS]; 198 Overlap _overlaps [XCAM_STITCH_MAX_CAMERAS]; 199 Copiers _copiers; 200 SmartPtr<BufferPool> _dewarp_pool; 201 202 Mutex _map_mutex; 203 BlendCopyTaskNums _task_counts; 204 205 SoftStitcher *_stitcher; 206 }; 207 208 bool 209 StitcherImpl::init_dewarp_factors (uint32_t idx) 210 { 211 XCAM_FAIL_RETURN ( 212 ERROR, _fisheye[idx].dewarp.ptr (), false, 213 "FisheyeDewarp dewarp handler empty"); 214 215 Factor match_left_factor, match_right_factor; 216 get_and_reset_feature_match_factors (idx, match_left_factor, match_right_factor); 217 218 Factor unify_factor, last_left_factor, last_right_factor; 219 _fisheye[idx].dewarp->get_factors (unify_factor.x, unify_factor.y); 220 last_left_factor = last_right_factor = unify_factor; 221 if (XCAM_DOUBLE_EQUAL_AROUND (unify_factor.x, 0.0f) || 222 XCAM_DOUBLE_EQUAL_AROUND (unify_factor.y, 0.0f)) { // not started. 223 return true; 224 } 225 226 Factor cur_left, cur_right; 227 cur_left.x = last_left_factor.x * match_left_factor.x; 228 cur_left.y = last_left_factor.y * match_left_factor.y; 229 cur_right.x = last_right_factor.x * match_right_factor.x; 230 cur_right.y = last_right_factor.y * match_right_factor.y; 231 232 unify_factor.x = (cur_left.x + cur_right.x) / 2.0f; 233 unify_factor.y = (cur_left.y + cur_right.y) / 2.0f; 234 _fisheye[idx].dewarp->set_factors (unify_factor.x, unify_factor.y); 235 236 return true; 237 } 238 239 XCamReturn 240 FisheyeDewarp::set_dewarp_geo_table ( 241 SmartPtr<SoftGeoMapper> mapper, 242 const CameraInfo &cam_info, 243 const Stitcher::RoundViewSlice &view_slice, 244 const BowlDataConfig &bowl) 245 { 246 PolyFisheyeDewarp fd; 247 fd.set_intrinsic_param (cam_info.calibration.intrinsic); 248 fd.set_extrinsic_param (cam_info.calibration.extrinsic); 249 250 uint32_t table_width, table_height; 251 table_width = view_slice.width / MAP_FACTOR_X; 252 table_width = XCAM_ALIGN_UP (table_width, 4); 253 table_height = view_slice.height / MAP_FACTOR_Y; 254 table_height = XCAM_ALIGN_UP (table_height, 2); 255 SurViewFisheyeDewarp::MapTable map_table(table_width * table_height); 256 fd.fisheye_dewarp ( 257 map_table, table_width, table_height, 258 view_slice.width, view_slice.height, bowl); 259 260 XCAM_FAIL_RETURN ( 261 ERROR, mapper->set_lookup_table (map_table.data (), table_width, table_height), 262 XCAM_RETURN_ERROR_UNKNOWN, "set fisheye dewarp lookup table failed"); 263 return XCAM_RETURN_NO_ERROR; 264 } 265 266 bool 267 StitcherImpl::get_and_reset_feature_match_factors (uint32_t idx, Factor &left, Factor &right) 268 { 269 uint32_t cam_num = _stitcher->get_camera_num (); 270 XCAM_FAIL_RETURN ( 271 ERROR, idx < cam_num, false, 272 "get dewarp factor failed, idx(%d) > camera_num(%d)", idx, cam_num); 273 274 SmartLock locker (_map_mutex); 275 left = _fisheye[idx].left_match_factor; 276 right = _fisheye[idx].right_match_factor; 277 278 _fisheye[idx].left_match_factor.reset (); 279 _fisheye[idx].right_match_factor.reset (); 280 return true; 281 } 282 283 XCamReturn 284 StitcherImpl::init_fisheye (uint32_t idx) 285 { 286 FisheyeDewarp &fisheye = _fisheye[idx]; 287 SmartPtr<ImageHandler::Callback> dewarp_cb = new CbGeoMap (_stitcher); 288 fisheye.dewarp = new SoftGeoMapper ("sitcher_remapper"); 289 XCAM_ASSERT (fisheye.dewarp.ptr ()); 290 fisheye.dewarp->set_callback (dewarp_cb); 291 292 Stitcher::RoundViewSlice view_slice = 293 _stitcher->get_round_view_slice (idx); 294 295 VideoBufferInfo buf_info; 296 buf_info.init ( 297 V4L2_PIX_FMT_NV12, view_slice.width, view_slice.height, 298 XCAM_ALIGN_UP (view_slice.width, SOFT_STITCHER_ALIGNMENT_X), 299 XCAM_ALIGN_UP (view_slice.height, SOFT_STITCHER_ALIGNMENT_Y)); 300 301 fisheye.buf_pool = new SoftVideoBufAllocator (buf_info); 302 XCAM_ASSERT (fisheye.buf_pool.ptr ()); 303 XCAM_FAIL_RETURN ( 304 ERROR, fisheye.buf_pool->reserve (2), XCAM_RETURN_ERROR_MEM, 305 "stitcher:%s reserve dewarp buffer pool(w:%d,h:%d) failed", 306 XCAM_STR (_stitcher->get_name ()), buf_info.width, buf_info.height); 307 return XCAM_RETURN_NO_ERROR; 308 } 309 310 XCamReturn 311 StitcherImpl::create_copier (Stitcher::CopyArea area) 312 { 313 XCAM_FAIL_RETURN ( 314 ERROR, 315 area.in_idx != INVALID_INDEX && 316 area.in_area.width == area.out_area.width && area.in_area.height == area.out_area.height, 317 XCAM_RETURN_ERROR_PARAM, 318 "stitcher: copy area (idx:%d) is invalid", area.in_idx); 319 320 SmartPtr<Worker::Callback> copy_cb = new CbCopyTask (_stitcher); 321 XCAM_ASSERT (copy_cb.ptr ()); 322 323 Copier copier; 324 copier.copy_task = new XCamSoftTasks::CopyTask (copy_cb); 325 XCAM_ASSERT (copier.copy_task.ptr ()); 326 copier.copy_area = area; 327 _copiers.push_back (copier); 328 329 return XCAM_RETURN_NO_ERROR; 330 } 331 332 XCamReturn 333 StitcherImpl::init_config (uint32_t count) 334 { 335 XCamReturn ret = XCAM_RETURN_NO_ERROR; 336 337 SmartPtr<ImageHandler::Callback> blender_cb = new CbBlender (_stitcher); 338 for (uint32_t i = 0; i < count; ++i) { 339 ret = init_fisheye (i); 340 XCAM_FAIL_RETURN ( 341 ERROR, xcam_ret_is_ok (ret), ret, 342 "stitcher:%s init fisheye failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); 343 344 #if ENABLE_FEATURE_MATCH 345 _overlaps[i].matcher = new CVCapiFeatureMatch; 346 347 CVFMConfig config; 348 config.sitch_min_width = 136; 349 config.min_corners = 4; 350 config.offset_factor = 0.8f; 351 config.delta_mean_offset = 120.0f; 352 config.recur_offset_error = 8.0f; 353 config.max_adjusted_offset = 24.0f; 354 config.max_valid_offset_y = 20.0f; 355 #ifndef ANDROID 356 config.max_track_error = 28.0f; 357 #else 358 config.max_track_error = 3600.0f; 359 #endif 360 _overlaps[i].matcher->set_config (config); 361 _overlaps[i].matcher->set_fm_index (i); 362 #endif 363 364 _overlaps[i].blender = create_soft_blender ().dynamic_cast_ptr<SoftBlender>(); 365 XCAM_ASSERT (_overlaps[i].blender.ptr ()); 366 _overlaps[i].blender->set_callback (blender_cb); 367 _overlaps[i].param_map.clear (); 368 } 369 370 Stitcher::CopyAreaArray areas = _stitcher->get_copy_area (); 371 uint32_t size = areas.size (); 372 for (uint32_t i = 0; i < size; ++i) { 373 XCAM_LOG_DEBUG ("soft-stitcher:copy area (idx:%d) input area(%d, %d, %d, %d) output area(%d, %d, %d, %d)", 374 areas[i].in_idx, 375 areas[i].in_area.pos_x, areas[i].in_area.pos_y, areas[i].in_area.width, areas[i].in_area.height, 376 areas[i].out_area.pos_x, areas[i].out_area.pos_y, areas[i].out_area.width, areas[i].out_area.height); 377 378 XCAM_ASSERT (areas[i].in_idx < size); 379 ret = create_copier (areas[i]); 380 XCAM_FAIL_RETURN ( 381 ERROR, xcam_ret_is_ok (ret), ret, 382 "soft-stitcher::%s init copier failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); 383 } 384 385 return XCAM_RETURN_NO_ERROR; 386 } 387 388 bool 389 StitcherImpl::remove_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m) 390 { 391 XCAM_ASSERT (param.ptr ()); 392 SmartLock locker (_map_mutex); 393 BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ()); 394 if (i == _task_counts.end ()) 395 return false; 396 397 _task_counts.erase (i); 398 return true; 399 } 400 401 int32_t 402 StitcherImpl::dec_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m) 403 { 404 XCAM_ASSERT (param.ptr ()); 405 SmartLock locker (_map_mutex); 406 BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ()); 407 if (i == _task_counts.end ()) 408 return -1; 409 410 int32_t &count = i->second; 411 --count; 412 if (count > 0) 413 return count; 414 415 XCAM_ASSERT (count == 0); 416 _task_counts.erase (i); 417 return 0; 418 } 419 420 XCamReturn 421 StitcherImpl::fisheye_dewarp_to_table () 422 { 423 uint32_t camera_num = _stitcher->get_camera_num (); 424 for (uint32_t i = 0; i < camera_num; ++i) { 425 CameraInfo cam_info; 426 _stitcher->get_camera_info (i, cam_info); 427 Stitcher::RoundViewSlice view_slice = _stitcher->get_round_view_slice (i); 428 429 BowlDataConfig bowl = _stitcher->get_bowl_config (); 430 bowl.angle_start = view_slice.hori_angle_start; 431 bowl.angle_end = format_angle (view_slice.hori_angle_start + view_slice.hori_angle_range); 432 433 uint32_t out_width, out_height; 434 _stitcher->get_output_size (out_width, out_height); 435 436 _fisheye[i].dewarp->set_output_size (view_slice.width, view_slice.height); 437 if (bowl.angle_end < bowl.angle_start) 438 bowl.angle_start -= 360.0f; 439 XCAM_LOG_INFO ( 440 "soft-stitcher:%s camera(idx:%d) info (angle start:%.2f, range:%.2f), bowl info (angle start%.2f, end:%.2f)", 441 XCAM_STR (_stitcher->get_name ()), i, 442 view_slice.hori_angle_start, view_slice.hori_angle_range, 443 bowl.angle_start, bowl.angle_end); 444 XCamReturn ret = _fisheye[i].set_dewarp_geo_table (_fisheye[i].dewarp, cam_info, view_slice, bowl); 445 XCAM_FAIL_RETURN ( 446 ERROR, xcam_ret_is_ok (ret), ret, 447 "stitcher:%s set dewarp geo table failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i); 448 449 } 450 451 return XCAM_RETURN_NO_ERROR; 452 } 453 454 XCamReturn 455 StitcherImpl::start_dewarp_works (const SmartPtr<SoftStitcher::StitcherParam> ¶m) 456 { 457 uint32_t camera_num = _stitcher->get_camera_num (); 458 Factor cur_left, cur_right; 459 460 for (uint32_t i = 0; i < camera_num; ++i) { 461 SmartPtr<VideoBuffer> out_buf = _fisheye[i].buf_pool->get_buffer (); 462 SmartPtr<HandlerParam> dewarp_params = new HandlerParam (i); 463 dewarp_params->in_buf = param->in_bufs[i]; 464 dewarp_params->out_buf = out_buf; 465 dewarp_params->stitch_param = param; 466 467 init_dewarp_factors (i); 468 XCamReturn ret = _fisheye[i].dewarp->execute_buffer (dewarp_params, false); 469 XCAM_FAIL_RETURN ( 470 ERROR, xcam_ret_is_ok (ret), ret, 471 "soft-stitcher:%s fisheye dewarp buffer failed", XCAM_STR (_stitcher->get_name ())); 472 } 473 return XCAM_RETURN_NO_ERROR; 474 } 475 476 SmartPtr<BlenderParam> 477 Overlap::find_blender_param_in_map ( 478 const SmartPtr<SoftStitcher::StitcherParam> &key, 479 const uint32_t idx) 480 { 481 SmartPtr<BlenderParam> param; 482 BlenderParams::iterator i = param_map.find (key.ptr ()); 483 if (i == param_map.end ()) { 484 param = new BlenderParam (idx, NULL, NULL, NULL); 485 XCAM_ASSERT (param.ptr ()); 486 param->stitch_param = key; 487 param_map.insert (std::make_pair ((void*)key.ptr (), param)); 488 } else { 489 param = (*i).second; 490 } 491 492 return param; 493 } 494 495 XCamReturn 496 StitcherImpl::feature_match ( 497 const SmartPtr<VideoBuffer> &left_buf, 498 const SmartPtr<VideoBuffer> &right_buf, 499 const uint32_t idx) 500 { 501 const Stitcher::ImageOverlapInfo overlap_info = _stitcher->get_overlap (idx); 502 Rect left_ovlap = overlap_info.left; 503 Rect right_ovlap = overlap_info.right; 504 const VideoBufferInfo left_buf_info = left_buf->get_video_info (); 505 506 left_ovlap.pos_y = left_ovlap.height / 5; 507 left_ovlap.height = left_ovlap.height / 2; 508 right_ovlap.pos_y = right_ovlap.height / 5; 509 right_ovlap.height = right_ovlap.height / 2; 510 511 _overlaps[idx].matcher->reset_offsets (); 512 _overlaps[idx].matcher->optical_flow_feature_match ( 513 left_buf, right_buf, left_ovlap, right_ovlap, left_buf_info.width); 514 float left_offsetx = _overlaps[idx].matcher->get_current_left_offset_x (); 515 Factor left_factor, right_factor; 516 517 uint32_t left_idx = idx; 518 float center_x = (float) _stitcher->get_center (left_idx).slice_center_x; 519 float feature_center_x = (float)left_ovlap.pos_x + (left_ovlap.width / 2.0f); 520 float range = feature_center_x - center_x; 521 XCAM_ASSERT (range > 1.0f); 522 right_factor.x = (range + left_offsetx / 2.0f) / range; 523 right_factor.y = 1.0; 524 XCAM_ASSERT (right_factor.x > 0.0f && right_factor.x < 2.0f); 525 526 uint32_t right_idx = (idx + 1) % _stitcher->get_camera_num (); 527 center_x = (float) _stitcher->get_center (right_idx).slice_center_x; 528 feature_center_x = (float)right_ovlap.pos_x + (right_ovlap.width / 2.0f); 529 range = center_x - feature_center_x; 530 XCAM_ASSERT (range > 1.0f); 531 left_factor.x = (range + left_offsetx / 2.0f) / range; 532 left_factor.y = 1.0; 533 XCAM_ASSERT (left_factor.x > 0.0f && left_factor.x < 2.0f); 534 535 { 536 SmartLock locker (_map_mutex); 537 _fisheye[left_idx].right_match_factor = right_factor; 538 _fisheye[right_idx].left_match_factor = left_factor; 539 } 540 541 return XCAM_RETURN_NO_ERROR; 542 } 543 544 XCamReturn 545 StitcherImpl::start_single_blender ( 546 const uint32_t idx, 547 const SmartPtr<BlenderParam> ¶m) 548 { 549 SmartPtr<SoftBlender> blender = _overlaps[idx].blender; 550 const Stitcher::ImageOverlapInfo &overlap_info = _stitcher->get_overlap (idx); 551 uint32_t out_width, out_height; 552 _stitcher->get_output_size (out_width, out_height); 553 554 blender->set_output_size (out_width, out_height); 555 blender->set_merge_window (overlap_info.out_area); 556 blender->set_input_valid_area (overlap_info.left, 0); 557 blender->set_input_valid_area (overlap_info.right, 1); 558 blender->set_input_merge_area (overlap_info.left, 0); 559 blender->set_input_merge_area (overlap_info.right, 1); 560 return blender->execute_buffer (param, false); 561 } 562 563 XCamReturn 564 StitcherImpl::start_overlap_tasks ( 565 const SmartPtr<SoftStitcher::StitcherParam> ¶m, 566 const uint32_t idx, const SmartPtr<VideoBuffer> &buf) 567 { 568 SmartPtr<BlenderParam> cur_param, prev_param; 569 const uint32_t camera_num = _stitcher->get_camera_num (); 570 uint32_t pre_idx = (idx + camera_num - 1) % camera_num; 571 XCamReturn ret = XCAM_RETURN_NO_ERROR; 572 { 573 SmartPtr<BlenderParam> param_b; 574 575 SmartLock locker (_map_mutex); 576 param_b = _overlaps[idx].find_blender_param_in_map (param, idx); 577 param_b->in_buf = buf; 578 if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) { 579 cur_param = param_b; 580 _overlaps[idx].param_map.erase (param.ptr ()); 581 } 582 583 param_b = _overlaps[pre_idx].find_blender_param_in_map (param, pre_idx); 584 param_b->in1_buf = buf; 585 if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) { 586 prev_param = param_b; 587 _overlaps[pre_idx].param_map.erase (param.ptr ()); 588 } 589 } 590 591 if (cur_param.ptr ()) { 592 cur_param->out_buf = param->out_buf; 593 ret = start_single_blender (idx, cur_param); 594 XCAM_FAIL_RETURN ( 595 ERROR, xcam_ret_is_ok (ret), ret, 596 "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx); 597 } 598 599 if (prev_param.ptr ()) { 600 prev_param->out_buf = param->out_buf; 601 ret = start_single_blender (pre_idx, prev_param); 602 XCAM_FAIL_RETURN ( 603 ERROR, xcam_ret_is_ok (ret), ret, 604 "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx); 605 } 606 607 #if ENABLE_FEATURE_MATCH 608 //start feature match 609 if (cur_param.ptr ()) { 610 ret = feature_match (cur_param->in_buf, cur_param->in1_buf, idx); 611 XCAM_FAIL_RETURN ( 612 ERROR, xcam_ret_is_ok (ret), ret, 613 "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx); 614 } 615 616 if (prev_param.ptr ()) { 617 ret = feature_match (prev_param->in_buf, prev_param->in1_buf, pre_idx); 618 XCAM_FAIL_RETURN ( 619 ERROR, xcam_ret_is_ok (ret), ret, 620 "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx); 621 } 622 #endif 623 return XCAM_RETURN_NO_ERROR; 624 } 625 626 XCamReturn 627 Copier::start_copy_task ( 628 const SmartPtr<ImageHandler::Parameters> ¶m, 629 const uint32_t idx, const SmartPtr<VideoBuffer> &buf) 630 { 631 XCAM_ASSERT (copy_task.ptr ()); 632 633 SmartPtr<VideoBuffer> in_buf = buf, out_buf = param->out_buf; 634 const VideoBufferInfo &in_info = in_buf->get_video_info (); 635 const VideoBufferInfo &out_info = out_buf->get_video_info (); 636 637 SmartPtr<StitcherCopyArgs> args = new StitcherCopyArgs (idx, param); 638 args->in_luma = new UcharImage ( 639 in_buf, copy_area.in_area.width, copy_area.in_area.height, in_info.strides[0], 640 in_info.offsets[0] + copy_area.in_area.pos_x + copy_area.in_area.pos_y * in_info.strides[0]); 641 args->in_uv = new Uchar2Image ( 642 in_buf, copy_area.in_area.width / 2, copy_area.in_area.height / 2, in_info.strides[0], 643 in_info.offsets[1] + copy_area.in_area.pos_x + copy_area.in_area.pos_y / 2 * in_info.strides[1]); 644 645 args->out_luma = new UcharImage ( 646 out_buf, copy_area.out_area.width, copy_area.out_area.height, out_info.strides[0], 647 out_info.offsets[0] + copy_area.out_area.pos_x + copy_area.out_area.pos_y * out_info.strides[0]); 648 args->out_uv = new Uchar2Image ( 649 out_buf, copy_area.out_area.width / 2, copy_area.out_area.height / 2, out_info.strides[0], 650 out_info.offsets[1] + copy_area.out_area.pos_x + copy_area.out_area.pos_y / 2 * out_info.strides[1]); 651 652 uint32_t thread_x = 1, thread_y = 4; 653 WorkSize global_size (1, xcam_ceil (copy_area.in_area.height, 2) / 2); 654 WorkSize local_size ( 655 xcam_ceil (global_size.value[0], thread_x) / thread_x, 656 xcam_ceil (global_size.value[1], thread_y) / thread_y); 657 658 copy_task->set_local_size (local_size); 659 copy_task->set_global_size (global_size); 660 661 return copy_task->work (args); 662 } 663 664 XCamReturn 665 StitcherImpl::start_copy_tasks ( 666 const SmartPtr<SoftStitcher::StitcherParam> ¶m, 667 const uint32_t idx, const SmartPtr<VideoBuffer> &buf) 668 { 669 uint32_t size = _stitcher->get_copy_area ().size (); 670 for (uint32_t i = 0; i < size; ++i) { 671 if(_copiers[i].copy_area.in_idx == idx) { 672 XCamReturn ret = _copiers[i].start_copy_task (param, idx, buf); 673 XCAM_FAIL_RETURN ( 674 ERROR, xcam_ret_is_ok (ret), ret, 675 "soft-stitcher:%s start copy task failed, idx:%d", XCAM_STR (_stitcher->get_name ()), idx); 676 } 677 } 678 679 return XCAM_RETURN_NO_ERROR; 680 } 681 682 XCamReturn 683 StitcherImpl::stop () 684 { 685 uint32_t cam_num = _stitcher->get_camera_num (); 686 for (uint32_t i = 0; i < cam_num; ++i) { 687 if (_fisheye[i].dewarp.ptr ()) { 688 _fisheye[i].dewarp->terminate (); 689 _fisheye[i].dewarp.release (); 690 } 691 if (_fisheye[i].buf_pool.ptr ()) { 692 _fisheye[i].buf_pool->stop (); 693 } 694 695 if (_overlaps[i].blender.ptr ()) { 696 _overlaps[i].blender->terminate (); 697 _overlaps[i].blender.release (); 698 } 699 } 700 701 for (Copiers::iterator i_copy = _copiers.begin (); i_copy != _copiers.end (); ++i_copy) { 702 Copier © = *i_copy; 703 if (copy.copy_task.ptr ()) { 704 copy.copy_task->stop (); 705 copy.copy_task.release (); 706 } 707 } 708 709 if (_dewarp_pool.ptr ()) { 710 _dewarp_pool->stop (); 711 } 712 return XCAM_RETURN_NO_ERROR; 713 } 714 715 }; 716 717 SoftStitcher::SoftStitcher (const char *name) 718 : SoftHandler (name) 719 , Stitcher (SOFT_STITCHER_ALIGNMENT_X, SOFT_STITCHER_ALIGNMENT_Y) 720 { 721 _impl = new SoftSitcherPriv::StitcherImpl (this); 722 XCAM_ASSERT (_impl.ptr ()); 723 #if ENABLE_FEATURE_MATCH 724 #ifndef ANDROID 725 cv::ocl::setUseOpenCL (false); 726 #endif 727 #endif 728 } 729 730 SoftStitcher::~SoftStitcher () 731 { 732 } 733 734 XCamReturn 735 SoftStitcher::stitch_buffers (const VideoBufferList &in_bufs, SmartPtr<VideoBuffer> &out_buf) 736 { 737 XCAM_FAIL_RETURN ( 738 ERROR, !in_bufs.empty (), XCAM_RETURN_ERROR_PARAM, 739 "soft-stitcher:%s stitch buffer failed, in_bufs is empty", XCAM_STR (get_name ())); 740 741 SmartPtr<StitcherParam> param = new StitcherParam; 742 param->out_buf = out_buf; 743 uint32_t count = 0; 744 for (VideoBufferList::const_iterator i = in_bufs.begin(); i != in_bufs.end (); ++i) { 745 SmartPtr<VideoBuffer> buf = *i; 746 XCAM_ASSERT (buf.ptr ()); 747 param->in_bufs[count++] = buf; 748 } 749 param->in_buf_num = count; 750 XCamReturn ret = execute_buffer (param, true); 751 if (!out_buf.ptr () && xcam_ret_is_ok (ret)) { 752 out_buf = param->out_buf; 753 } 754 return ret; 755 } 756 757 XCamReturn 758 SoftStitcher::terminate () 759 { 760 _impl->stop (); 761 return SoftHandler::terminate (); 762 } 763 764 XCamReturn 765 SoftStitcher::start_task_count (const SmartPtr<SoftStitcher::StitcherParam> ¶m) 766 { 767 XCAM_ASSERT (param.ptr ()); 768 XCAM_ASSERT (_impl.ptr ()); 769 770 SmartLock locker (_impl->_map_mutex); 771 772 XCAM_FAIL_RETURN ( 773 ERROR, check_work_continue (param, XCAM_RETURN_NO_ERROR), XCAM_RETURN_ERROR_PARAM, 774 "soft-stitcher:%s start task count failed in work check", XCAM_STR (get_name ())); 775 776 if (_impl->_task_counts.find (param.ptr ()) != _impl->_task_counts.end ()) { 777 XCAM_LOG_ERROR ("tasks already started, this should never happen."); 778 return XCAM_RETURN_ERROR_UNKNOWN; 779 } 780 781 int32_t count = get_camera_num (); 782 count += get_copy_area ().size (); 783 784 XCAM_LOG_DEBUG ("stitcher :%s start task count :%d", XCAM_STR(get_name ()), count); 785 _impl->_task_counts.insert (std::make_pair((void*)param.ptr(), count)); 786 return XCAM_RETURN_NO_ERROR; 787 } 788 789 void 790 SoftStitcher::dewarp_done ( 791 const SmartPtr<ImageHandler> &handler, 792 const SmartPtr<ImageHandler::Parameters> &base, 793 const XCamReturn error) 794 { 795 SmartPtr<SoftSitcherPriv::HandlerParam> dewarp_param = base.dynamic_cast_ptr<SoftSitcherPriv::HandlerParam> (); 796 XCAM_ASSERT (dewarp_param.ptr ()); 797 SmartPtr<SoftStitcher::StitcherParam> param = dewarp_param->stitch_param; 798 XCAM_ASSERT (param.ptr ()); 799 XCAM_UNUSED (handler); 800 801 if (!check_work_continue (param, error)) 802 return; 803 804 XCAM_LOG_INFO ("soft-stitcher:%s camera(idx:%d) dewarp done", XCAM_STR (get_name ()), dewarp_param->idx); 805 stitcher_dump_buf (dewarp_param->out_buf, dewarp_param->idx, "stitcher-dewarp"); 806 807 //start both blender and feature match 808 XCamReturn ret = _impl->start_overlap_tasks (param, dewarp_param->idx, dewarp_param->out_buf); 809 if (!xcam_ret_is_ok (ret)) { 810 work_broken (param, ret); 811 } 812 813 ret = _impl->start_copy_tasks (param, dewarp_param->idx, dewarp_param->out_buf); 814 if (!xcam_ret_is_ok (ret)) { 815 work_broken (param, ret); 816 } 817 } 818 819 void 820 SoftStitcher::blender_done ( 821 const SmartPtr<ImageHandler> &handler, 822 const SmartPtr<ImageHandler::Parameters> &base, 823 const XCamReturn error) 824 { 825 SmartPtr<SoftSitcherPriv::BlenderParam> blender_param = base.dynamic_cast_ptr<SoftSitcherPriv::BlenderParam> (); 826 XCAM_ASSERT (blender_param.ptr ()); 827 SmartPtr<SoftStitcher::StitcherParam> param = blender_param->stitch_param; 828 XCAM_ASSERT (param.ptr ()); 829 XCAM_UNUSED (handler); 830 831 if (!check_work_continue (param, error)) { 832 _impl->remove_task_count (param); 833 return; 834 } 835 836 stitcher_dump_buf (blender_param->out_buf, blender_param->idx, "stitcher-blend"); 837 XCAM_LOG_INFO ("blender:(%s) overlap:%d done", XCAM_STR (handler->get_name ()), blender_param->idx); 838 839 if (_impl->dec_task_count (param) == 0) { 840 work_well_done (param, error); 841 } 842 } 843 844 void 845 SoftStitcher::copy_task_done ( 846 const SmartPtr<Worker> &worker, 847 const SmartPtr<Worker::Arguments> &base, 848 const XCamReturn error) 849 { 850 XCAM_UNUSED (worker); 851 XCAM_ASSERT (worker.ptr ()); 852 SmartPtr<SoftSitcherPriv::StitcherCopyArgs> args = base.dynamic_cast_ptr<SoftSitcherPriv::StitcherCopyArgs> (); 853 XCAM_ASSERT (args.ptr ()); 854 const SmartPtr<SoftStitcher::StitcherParam> param = 855 args->get_param ().dynamic_cast_ptr<SoftStitcher::StitcherParam> (); 856 XCAM_ASSERT (param.ptr ()); 857 858 if (!check_work_continue (param, error)) { 859 _impl->remove_task_count (param); 860 return; 861 } 862 XCAM_LOG_INFO ("soft-stitcher:%s camera(idx:%d) copy done", XCAM_STR (get_name ()), args->idx); 863 864 if (_impl->dec_task_count (param) == 0) { 865 work_well_done (param, error); 866 } 867 } 868 869 XCamReturn 870 SoftStitcher::configure_resource (const SmartPtr<Parameters> ¶m) 871 { 872 XCAM_UNUSED (param); 873 XCAM_ASSERT (_impl.ptr ()); 874 875 XCamReturn ret = estimate_round_slices (); 876 XCAM_FAIL_RETURN ( 877 ERROR, xcam_ret_is_ok (ret), ret, 878 "soft-stitcher:%s estimate round view slices failed", XCAM_STR (get_name ())); 879 880 ret = estimate_coarse_crops (); 881 XCAM_FAIL_RETURN ( 882 ERROR, xcam_ret_is_ok (ret), ret, 883 "soft-stitcher:%s estimate coarse crops failed", XCAM_STR (get_name ())); 884 885 ret = mark_centers (); 886 XCAM_FAIL_RETURN ( 887 ERROR, xcam_ret_is_ok (ret), ret, 888 "soft-stitcher:%s mark centers failed", XCAM_STR (get_name ())); 889 890 ret = estimate_overlap (); 891 XCAM_FAIL_RETURN ( 892 ERROR, xcam_ret_is_ok (ret), ret, 893 "soft-stitcher:%s estimake coarse overlap failed", XCAM_STR (get_name ())); 894 895 ret = update_copy_areas (); 896 XCAM_FAIL_RETURN ( 897 ERROR, xcam_ret_is_ok (ret), ret, 898 "soft-stitcher:%s update copy areas failed", XCAM_STR (get_name ())); 899 900 uint32_t camera_count = get_camera_num (); 901 ret = _impl->init_config (camera_count); 902 XCAM_FAIL_RETURN ( 903 ERROR, xcam_ret_is_ok (ret), ret, 904 "soft-stitcher:%s initialize private config failed", XCAM_STR (get_name ())); 905 906 ret = _impl->fisheye_dewarp_to_table (); 907 XCAM_FAIL_RETURN ( 908 ERROR, xcam_ret_is_ok (ret), ret, 909 "soft-stitcher:%s fisheye_dewarp_to_table failed", XCAM_STR (get_name ())); 910 911 VideoBufferInfo out_info; 912 uint32_t out_width, out_height; 913 get_output_size (out_width, out_height); 914 XCAM_FAIL_RETURN ( 915 ERROR, out_width && out_height, XCAM_RETURN_ERROR_PARAM, 916 "soft-stitcher:%s output size was not set", XCAM_STR(get_name ())); 917 918 out_info.init ( 919 V4L2_PIX_FMT_NV12, out_width, out_height, 920 XCAM_ALIGN_UP (out_width, SOFT_STITCHER_ALIGNMENT_X), 921 XCAM_ALIGN_UP (out_height, SOFT_STITCHER_ALIGNMENT_Y)); 922 set_out_video_info (out_info); 923 924 return ret; 925 } 926 927 XCamReturn 928 SoftStitcher::start_work (const SmartPtr<Parameters> &base) 929 { 930 SmartPtr<StitcherParam> param = base.dynamic_cast_ptr<StitcherParam> (); 931 932 XCAM_FAIL_RETURN ( 933 ERROR, param.ptr () && param->in_buf_num > 0 && param->in_bufs[0].ptr (), XCAM_RETURN_ERROR_PARAM, 934 "soft_stitcher:%s start_work failed, params(in_buf_num) in_bufs are set", 935 XCAM_STR (get_name ())); 936 937 XCamReturn ret = start_task_count (param); 938 XCAM_FAIL_RETURN ( 939 ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM, 940 "soft_stitcher:%s start blender count failed", XCAM_STR (get_name ())); 941 942 ret = _impl->start_dewarp_works (param); 943 XCAM_FAIL_RETURN ( 944 ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM, 945 "soft_stitcher:%s start dewarp works failed", XCAM_STR (get_name ())); 946 947 //for (uint32_t i = 0; i < param->in_buf_num; ++i) { 948 // param->in_bufs[i].release (); 949 //} 950 951 return ret; 952 } 953 954 SmartPtr<Stitcher> 955 Stitcher::create_soft_stitcher () 956 { 957 return new SoftStitcher; 958 } 959 960 } 961 962
