1 /* 2 * gstxcamfilter.cpp -gst xcamfilter plugin 3 * 4 * Copyright (c) 2016 Intel Corporation 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Yinhang Liu <yinhangx.liu (at) intel.com> 19 */ 20 21 #include "gstxcamfilter.h" 22 #include "gstxcambuffermeta.h" 23 24 #include <gst/gstmeta.h> 25 #include <gst/allocators/gstdmabuf.h> 26 27 using namespace XCam; 28 using namespace GstXCam; 29 30 #define DEFAULT_SMART_ANALYSIS_LIB_DIR "/usr/lib/xcam/plugins/smart" 31 #define DEFAULT_DELAY_BUFFER_NUM 2 32 33 #define DEFAULT_PROP_BUFFERCOUNT 8 34 #define DEFAULT_PROP_COPY_MODE COPY_MODE_CPU 35 #define DEFAULT_PROP_DEFOG_MODE DEFOG_NONE 36 #define DEFAULT_PROP_WAVELET_MODE NONE_WAVELET 37 #define DEFAULT_PROP_3D_DENOISE_MODE DENOISE_3D_NONE 38 #define DEFAULT_PROP_ENABLE_WIREFRAME FALSE 39 #define DEFAULT_PROP_ENABLE_IMAGE_WARP FALSE 40 #define DEFAULT_PROP_ENABLE_IMAGE_STITCH FALSE 41 #define DEFAULT_PROP_STITCH_ENABLE_SEAM FALSE 42 #define DEFAULT_PROP_STITCH_SCALE_MODE CLBlenderScaleLocal 43 #define DEFAULT_PROP_STITCH_FISHEYE_MAP FALSE 44 #define DEFAULT_PROP_STITCH_LSC FALSE 45 #define DEFAULT_PROP_STITCH_FM_OCL FALSE 46 #define DEFAULT_PROP_STITCH_RES_MODE StitchRes1080P 47 48 XCAM_BEGIN_DECLARE 49 50 enum { 51 PROP_0, 52 PROP_BUFFERCOUNT, 53 PROP_COPY_MODE, 54 PROP_DEFOG_MODE, 55 PROP_WAVELET_MODE, 56 PROP_DENOISE_3D_MODE, 57 PROP_ENABLE_WIREFRAME, 58 PROP_ENABLE_IMAGE_WARP, 59 PROP_ENABLE_IMAGE_STITCH, 60 PROP_STITCH_ENABLE_SEAM, 61 PROP_STITCH_SCALE_MODE, 62 PROP_STITCH_FISHEYE_MAP, 63 PROP_STITCH_LSC, 64 PROP_STITCH_FM_OCL, 65 PROP_STITCH_RES_MODE 66 }; 67 68 #define GST_TYPE_XCAM_FILTER_COPY_MODE (gst_xcam_filter_copy_mode_get_type ()) 69 static GType 70 gst_xcam_filter_copy_mode_get_type (void) 71 { 72 static GType g_type = 0; 73 static const GEnumValue copy_mode_types[] = { 74 {COPY_MODE_CPU, "Copy buffer with CPU", "cpu"}, 75 {COPY_MODE_DMA, "Copy buffer with DMA", "dma"}, 76 {0, NULL, NULL} 77 }; 78 79 if (g_once_init_enter (&g_type)) { 80 const GType type = 81 g_enum_register_static ("GstXCamFilterCopyModeType", copy_mode_types); 82 g_once_init_leave (&g_type, type); 83 } 84 85 return g_type; 86 } 87 88 #define GST_TYPE_XCAM_FILTER_DEFOG_MODE (gst_xcam_filter_defog_mode_get_type ()) 89 static GType 90 gst_xcam_filter_defog_mode_get_type (void) 91 { 92 static GType g_type = 0; 93 static const GEnumValue defog_mode_types [] = { 94 {DEFOG_NONE, "Defog disabled", "none"}, 95 {DEFOG_RETINEX, "Defog retinex", "retinex"}, 96 {DEFOG_DCP, "Defog dark channel prior", "dcp"}, 97 {0, NULL, NULL} 98 }; 99 100 if (g_once_init_enter (&g_type)) { 101 const GType type = 102 g_enum_register_static ("GstXCamFilterDefogModeType", defog_mode_types); 103 g_once_init_leave (&g_type, type); 104 } 105 106 return g_type; 107 } 108 109 #define GST_TYPE_XCAM_FILTER_WAVELET_MODE (gst_xcam_filter_wavelet_mode_get_type ()) 110 static GType 111 gst_xcam_filter_wavelet_mode_get_type (void) 112 { 113 static GType g_type = 0; 114 static const GEnumValue wavelet_mode_types[] = { 115 {NONE_WAVELET, "Wavelet disabled", "none"}, 116 {HAT_WAVELET_Y, "Hat wavelet Y", "hat Y"}, 117 {HAT_WAVELET_UV, "Hat wavelet UV", "hat UV"}, 118 {HARR_WAVELET_Y, "Haar wavelet Y", "haar Y"}, 119 {HARR_WAVELET_UV, "Haar wavelet UV", "haar UV"}, 120 {HARR_WAVELET_YUV, "Haar wavelet YUV", "haar YUV"}, 121 {HARR_WAVELET_BAYES, "Haar wavelet bayes shrink", "haar Bayes"}, 122 {0, NULL, NULL}, 123 }; 124 125 if (g_once_init_enter (&g_type)) { 126 const GType type = 127 g_enum_register_static ("GstXCamFilterWaveletModeType", wavelet_mode_types); 128 g_once_init_leave (&g_type, type); 129 } 130 131 return g_type; 132 } 133 134 #define GST_TYPE_XCAM_FILTER_3D_DENOISE_MODE (gst_xcam_filter_3d_denoise_mode_get_type ()) 135 static GType 136 gst_xcam_filter_3d_denoise_mode_get_type (void) 137 { 138 static GType g_type = 0; 139 static const GEnumValue denoise_3d_mode_types [] = { 140 {DENOISE_3D_NONE, "3D Denoise disabled", "none"}, 141 {DENOISE_3D_YUV, "3D Denoise yuv", "yuv"}, 142 {DENOISE_3D_UV, "3D Denoise uv", "uv"}, 143 {0, NULL, NULL} 144 }; 145 146 if (g_once_init_enter (&g_type)) { 147 const GType type = 148 g_enum_register_static ("GstXCamFilter3DDenoiseModeType", denoise_3d_mode_types); 149 g_once_init_leave (&g_type, type); 150 } 151 152 return g_type; 153 } 154 155 #define GST_TYPE_XCAM_FILTER_STITCH_SCALE_MODE (gst_xcam_filter_stitch_scale_mode_get_type ()) 156 static GType 157 gst_xcam_filter_stitch_scale_mode_get_type (void) 158 { 159 static GType g_type = 0; 160 static const GEnumValue stitch_scale_mode_types [] = { 161 {CLBlenderScaleLocal, "Image stitch local scale", "local"}, 162 {CLBlenderScaleGlobal, "Image stitch glocal scale", "global"}, 163 {0, NULL, NULL} 164 }; 165 166 if (g_once_init_enter (&g_type)) { 167 const GType type = 168 g_enum_register_static ("GstXCamFilterStitchScaleModeType", stitch_scale_mode_types); 169 g_once_init_leave (&g_type, type); 170 } 171 172 return g_type; 173 } 174 175 #define GST_TYPE_XCAM_FILTER_STITCH_RES_MODE (gst_xcam_filter_stitch_res_mode_get_type ()) 176 static GType 177 gst_xcam_filter_stitch_res_mode_get_type (void) 178 { 179 static GType g_type = 0; 180 static const GEnumValue stitch_res_mode_types [] = { 181 {StitchRes1080P, "Image stitch 1080P mode", "1080p"}, 182 {StitchRes4K, "Image stitch 4K mode", "4k"}, 183 {0, NULL, NULL} 184 }; 185 186 if (g_once_init_enter (&g_type)) { 187 const GType type = 188 g_enum_register_static ("GstXCamFilterStitchResModeType", stitch_res_mode_types); 189 g_once_init_leave (&g_type, type); 190 } 191 192 return g_type; 193 } 194 195 static GstStaticPadTemplate gst_xcam_sink_factory = 196 GST_STATIC_PAD_TEMPLATE ("sink", 197 GST_PAD_SINK, 198 GST_PAD_ALWAYS, 199 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12 }"))); 200 201 static GstStaticPadTemplate gst_xcam_src_factory = 202 GST_STATIC_PAD_TEMPLATE ("src", 203 GST_PAD_SRC, 204 GST_PAD_ALWAYS, 205 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12 }"))); 206 207 GST_DEBUG_CATEGORY (gst_xcam_filter_debug); 208 #define GST_CAT_DEFAULT gst_xcam_filter_debug 209 210 #define gst_xcam_filter_parent_class parent_class 211 G_DEFINE_TYPE (GstXCamFilter, gst_xcam_filter, GST_TYPE_BASE_TRANSFORM); 212 213 static void gst_xcam_filter_finalize (GObject * object); 214 static void gst_xcam_filter_set_property (GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); 215 static void gst_xcam_filter_get_property (GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); 216 static gboolean gst_xcam_filter_start (GstBaseTransform *trans); 217 static GstCaps *gst_xcam_filter_transform_caps ( 218 GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter); 219 static gboolean gst_xcam_filter_set_caps (GstBaseTransform *trans, GstCaps *incaps, GstCaps *outcaps); 220 static gboolean gst_xcam_filter_stop (GstBaseTransform *trans); 221 static void gst_xcam_filter_before_transform (GstBaseTransform *trans, GstBuffer *buffer); 222 static GstFlowReturn gst_xcam_filter_prepare_output_buffer (GstBaseTransform * trans, GstBuffer *input, GstBuffer **outbuf); 223 static GstFlowReturn gst_xcam_filter_transform (GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf); 224 225 XCAM_END_DECLARE 226 227 static void 228 gst_xcam_filter_class_init (GstXCamFilterClass *class_self) 229 { 230 GObjectClass *gobject_class; 231 GstElementClass *element_class; 232 GstBaseTransformClass *basetrans_class; 233 234 gobject_class = (GObjectClass *) class_self; 235 element_class = (GstElementClass *) class_self; 236 basetrans_class = (GstBaseTransformClass *) class_self; 237 238 GST_DEBUG_CATEGORY_INIT (gst_xcam_filter_debug, "xcamfilter", 0, "LibXCam filter plugin"); 239 240 gobject_class->finalize = gst_xcam_filter_finalize; 241 gobject_class->set_property = gst_xcam_filter_set_property; 242 gobject_class->get_property = gst_xcam_filter_get_property; 243 244 g_object_class_install_property ( 245 gobject_class, PROP_BUFFERCOUNT, 246 g_param_spec_int ("buffercount", "buffer count", "Buffer count", 247 0, G_MAXINT, DEFAULT_PROP_BUFFERCOUNT, 248 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 249 250 g_object_class_install_property ( 251 gobject_class, PROP_COPY_MODE, 252 g_param_spec_enum ("copy-mode", "copy mode", "Copy Mode", 253 GST_TYPE_XCAM_FILTER_COPY_MODE, DEFAULT_PROP_COPY_MODE, 254 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 255 256 g_object_class_install_property ( 257 gobject_class, PROP_DEFOG_MODE, 258 g_param_spec_enum ("defog-mode", "defog mode", "Defog mode", 259 GST_TYPE_XCAM_FILTER_DEFOG_MODE, DEFAULT_PROP_DEFOG_MODE, 260 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 261 262 g_object_class_install_property ( 263 gobject_class, PROP_WAVELET_MODE, 264 g_param_spec_enum ("wavelet-mode", "wavelet mode", "Wavelet Mode", 265 GST_TYPE_XCAM_FILTER_WAVELET_MODE, DEFAULT_PROP_WAVELET_MODE, 266 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 267 268 g_object_class_install_property ( 269 gobject_class, PROP_DENOISE_3D_MODE, 270 g_param_spec_enum ("denoise-3d", "3D Denoise mode", "3D Denoise mode", 271 GST_TYPE_XCAM_FILTER_3D_DENOISE_MODE, DEFAULT_PROP_3D_DENOISE_MODE, 272 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 273 274 g_object_class_install_property ( 275 gobject_class, PROP_ENABLE_WIREFRAME, 276 g_param_spec_boolean ("enable-wireframe", "enable wire frame", "Enable wire frame", 277 DEFAULT_PROP_ENABLE_WIREFRAME, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 278 279 g_object_class_install_property ( 280 gobject_class, PROP_ENABLE_IMAGE_WARP, 281 g_param_spec_boolean ("enable-warp", "enable image warp", "Enable Image Warp", 282 DEFAULT_PROP_ENABLE_IMAGE_WARP, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 283 284 g_object_class_install_property ( 285 gobject_class, PROP_ENABLE_IMAGE_STITCH, 286 g_param_spec_boolean ("enable-stitch", "enable image stitch", "Enable Image Stitch", 287 DEFAULT_PROP_ENABLE_IMAGE_STITCH, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 288 289 g_object_class_install_property ( 290 gobject_class, PROP_STITCH_ENABLE_SEAM, 291 g_param_spec_boolean ("stitch-seam", "enable seam just for stitch", "Enable Seam Just For Stitch", 292 DEFAULT_PROP_STITCH_ENABLE_SEAM, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 293 294 g_object_class_install_property ( 295 gobject_class, PROP_STITCH_SCALE_MODE, 296 g_param_spec_enum ("stitch-scale", "stitch scale mode", "Stitch Scale Mode", 297 GST_TYPE_XCAM_FILTER_STITCH_SCALE_MODE, DEFAULT_PROP_STITCH_SCALE_MODE, 298 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 299 300 g_object_class_install_property ( 301 gobject_class, PROP_STITCH_FISHEYE_MAP, 302 g_param_spec_boolean ("stitch-fisheye-map", "stitch fisheye map", "Enable fisheye map for stitch", 303 DEFAULT_PROP_STITCH_FISHEYE_MAP, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 304 305 g_object_class_install_property ( 306 gobject_class, PROP_STITCH_LSC, 307 g_param_spec_boolean ("stitch-lsc", "stitch enable lens shading correction", "Enable Lens Shading Correction", 308 DEFAULT_PROP_STITCH_LSC, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 309 310 #if HAVE_OPENCV 311 g_object_class_install_property ( 312 gobject_class, PROP_STITCH_FM_OCL, 313 g_param_spec_boolean ("stitch-fm-ocl", "stitch enable ocl for feature match", "Enable ocl for feature match", 314 DEFAULT_PROP_STITCH_FM_OCL, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 315 #endif 316 317 g_object_class_install_property ( 318 gobject_class, PROP_STITCH_RES_MODE, 319 g_param_spec_enum ("stitch-res-mode", "stitch resolution mode", "Stitch Resolution Mode", 320 GST_TYPE_XCAM_FILTER_STITCH_RES_MODE, DEFAULT_PROP_STITCH_RES_MODE, 321 (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); 322 323 gst_element_class_set_details_simple (element_class, 324 "Libxcam Filter", 325 "Filter/Effect/Video", 326 "Process NV12 stream using xcam library", 327 "Wind Yuan <feng.yuan (at) intel.com> & Yinhang Liu <yinhangx.liu (at) intel.com>"); 328 329 gst_element_class_add_pad_template (element_class, 330 gst_static_pad_template_get (&gst_xcam_src_factory)); 331 gst_element_class_add_pad_template (element_class, 332 gst_static_pad_template_get (&gst_xcam_sink_factory)); 333 334 basetrans_class->start = GST_DEBUG_FUNCPTR (gst_xcam_filter_start); 335 basetrans_class->stop = GST_DEBUG_FUNCPTR (gst_xcam_filter_stop); 336 basetrans_class->transform_caps = GST_DEBUG_FUNCPTR (gst_xcam_filter_transform_caps); 337 basetrans_class->set_caps = GST_DEBUG_FUNCPTR (gst_xcam_filter_set_caps); 338 basetrans_class->before_transform = GST_DEBUG_FUNCPTR (gst_xcam_filter_before_transform); 339 basetrans_class->prepare_output_buffer = GST_DEBUG_FUNCPTR (gst_xcam_filter_prepare_output_buffer); 340 basetrans_class->transform = GST_DEBUG_FUNCPTR (gst_xcam_filter_transform); 341 } 342 343 static void 344 gst_xcam_filter_init (GstXCamFilter *xcamfilter) 345 { 346 xcamfilter->buf_count = DEFAULT_PROP_BUFFERCOUNT; 347 xcamfilter->copy_mode = DEFAULT_PROP_COPY_MODE; 348 xcamfilter->defog_mode = DEFAULT_PROP_DEFOG_MODE; 349 xcamfilter->wavelet_mode = DEFAULT_PROP_WAVELET_MODE; 350 xcamfilter->denoise_3d_mode = DEFAULT_PROP_3D_DENOISE_MODE; 351 xcamfilter->denoise_3d_ref_count = 2; 352 xcamfilter->enable_wireframe = DEFAULT_PROP_ENABLE_WIREFRAME; 353 xcamfilter->enable_image_warp = DEFAULT_PROP_ENABLE_IMAGE_WARP; 354 xcamfilter->enable_stitch = DEFAULT_PROP_ENABLE_IMAGE_STITCH; 355 xcamfilter->stitch_enable_seam = DEFAULT_PROP_STITCH_ENABLE_SEAM; 356 xcamfilter->stitch_fisheye_map = DEFAULT_PROP_STITCH_FISHEYE_MAP; 357 xcamfilter->stitch_lsc = DEFAULT_PROP_STITCH_LSC; 358 xcamfilter->stitch_fm_ocl = DEFAULT_PROP_STITCH_FM_OCL; 359 xcamfilter->stitch_scale_mode = DEFAULT_PROP_STITCH_SCALE_MODE; 360 xcamfilter->stitch_res_mode = DEFAULT_PROP_STITCH_RES_MODE; 361 362 xcamfilter->delay_buf_num = DEFAULT_DELAY_BUFFER_NUM; 363 xcamfilter->cached_buf_num = 0; 364 365 XCAM_CONSTRUCTOR (xcamfilter->pipe_manager, SmartPtr<MainPipeManager>); 366 xcamfilter->pipe_manager = new MainPipeManager; 367 XCAM_ASSERT (xcamfilter->pipe_manager.ptr ()); 368 } 369 370 static void 371 gst_xcam_filter_finalize (GObject *object) 372 { 373 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (object); 374 375 if (xcamfilter->allocator) 376 gst_object_unref (xcamfilter->allocator); 377 378 xcamfilter->pipe_manager.release (); 379 XCAM_DESTRUCTOR (xcamfilter->pipe_manager, SmartPtr<MainPipeManager>); 380 381 G_OBJECT_CLASS (parent_class)->finalize (object); 382 } 383 384 static void 385 gst_xcam_filter_set_property (GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) 386 { 387 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (object); 388 389 switch (prop_id) { 390 case PROP_BUFFERCOUNT: 391 xcamfilter->buf_count = g_value_get_int (value); 392 break; 393 case PROP_COPY_MODE: 394 xcamfilter->copy_mode = (CopyMode) g_value_get_enum (value); 395 break; 396 case PROP_DEFOG_MODE: 397 xcamfilter->defog_mode = (DefogModeType) g_value_get_enum (value); 398 break; 399 case PROP_WAVELET_MODE: 400 xcamfilter->wavelet_mode = (WaveletModeType) g_value_get_enum (value); 401 break; 402 case PROP_DENOISE_3D_MODE: 403 xcamfilter->denoise_3d_mode = (Denoise3DModeType) g_value_get_enum (value); 404 break; 405 case PROP_ENABLE_WIREFRAME: 406 xcamfilter->enable_wireframe = g_value_get_boolean (value); 407 break; 408 case PROP_ENABLE_IMAGE_WARP: 409 xcamfilter->enable_image_warp = g_value_get_boolean (value); 410 break; 411 case PROP_ENABLE_IMAGE_STITCH: 412 xcamfilter->enable_stitch = g_value_get_boolean (value); 413 break; 414 case PROP_STITCH_ENABLE_SEAM: 415 xcamfilter->stitch_enable_seam = g_value_get_boolean (value); 416 break; 417 case PROP_STITCH_SCALE_MODE: 418 xcamfilter->stitch_scale_mode = (CLBlenderScaleMode) g_value_get_enum (value); 419 break; 420 case PROP_STITCH_FISHEYE_MAP: 421 xcamfilter->stitch_fisheye_map = g_value_get_boolean (value); 422 break; 423 case PROP_STITCH_LSC: 424 xcamfilter->stitch_lsc = g_value_get_boolean (value); 425 break; 426 #if HAVE_OPENCV 427 case PROP_STITCH_FM_OCL: 428 xcamfilter->stitch_fm_ocl = g_value_get_boolean (value); 429 break; 430 #endif 431 case PROP_STITCH_RES_MODE: 432 xcamfilter->stitch_res_mode = (StitchResMode) g_value_get_enum (value); 433 break; 434 default: 435 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 436 break; 437 } 438 } 439 440 static void 441 gst_xcam_filter_get_property (GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) 442 { 443 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (object); 444 445 switch (prop_id) { 446 case PROP_BUFFERCOUNT: 447 g_value_set_int (value, xcamfilter->buf_count); 448 break; 449 case PROP_COPY_MODE: 450 g_value_set_enum (value, xcamfilter->copy_mode); 451 break; 452 case PROP_DEFOG_MODE: 453 g_value_set_enum (value, xcamfilter->defog_mode); 454 break; 455 case PROP_WAVELET_MODE: 456 g_value_set_enum (value, xcamfilter->wavelet_mode); 457 break; 458 case PROP_DENOISE_3D_MODE: 459 g_value_set_enum (value, xcamfilter->denoise_3d_mode); 460 break; 461 case PROP_ENABLE_WIREFRAME: 462 g_value_set_boolean (value, xcamfilter->enable_wireframe); 463 break; 464 case PROP_ENABLE_IMAGE_WARP: 465 g_value_set_boolean (value, xcamfilter->enable_image_warp); 466 break; 467 case PROP_ENABLE_IMAGE_STITCH: 468 g_value_set_boolean (value, xcamfilter->enable_stitch); 469 break; 470 case PROP_STITCH_ENABLE_SEAM: 471 g_value_set_boolean (value, xcamfilter->stitch_enable_seam); 472 break; 473 case PROP_STITCH_SCALE_MODE: 474 g_value_set_enum (value, xcamfilter->stitch_scale_mode); 475 break; 476 case PROP_STITCH_FISHEYE_MAP: 477 g_value_set_boolean (value, xcamfilter->stitch_fisheye_map); 478 break; 479 case PROP_STITCH_LSC: 480 g_value_set_boolean (value, xcamfilter->stitch_lsc); 481 break; 482 #if HAVE_OPENCV 483 case PROP_STITCH_FM_OCL: 484 g_value_set_boolean (value, xcamfilter->stitch_fm_ocl); 485 break; 486 #endif 487 case PROP_STITCH_RES_MODE: 488 g_value_set_enum (value, xcamfilter->stitch_res_mode); 489 break; 490 default: 491 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); 492 break; 493 } 494 } 495 496 static gboolean 497 gst_xcam_filter_start (GstBaseTransform *trans) 498 { 499 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 500 501 if (xcamfilter->buf_count <= xcamfilter->delay_buf_num) { 502 XCAM_LOG_ERROR ( 503 "buffer count (%d) should be greater than delayed buffer number (%d)", 504 xcamfilter->buf_count, 505 xcamfilter->delay_buf_num); 506 return false; 507 } 508 509 SmartPtr<MainPipeManager> pipe_manager = xcamfilter->pipe_manager; 510 SmartPtr<SmartAnalyzer> smart_analyzer; 511 SmartPtr<CLPostImageProcessor> image_processor; 512 513 SmartHandlerList smart_handlers = SmartAnalyzerLoader::load_smart_handlers (DEFAULT_SMART_ANALYSIS_LIB_DIR); 514 if (!smart_handlers.empty ()) { 515 smart_analyzer = new SmartAnalyzer (); 516 if (smart_analyzer.ptr ()) { 517 SmartHandlerList::iterator i_handler = smart_handlers.begin (); 518 for (; i_handler != smart_handlers.end (); ++i_handler) 519 { 520 XCAM_ASSERT ((*i_handler).ptr ()); 521 smart_analyzer->add_handler (*i_handler); 522 } 523 if (smart_analyzer->prepare_handlers () != XCAM_RETURN_NO_ERROR) { 524 XCAM_LOG_WARNING ("analyzer(%s) prepare handlers failed", smart_analyzer->get_name ()); 525 return false; 526 } 527 pipe_manager->set_smart_analyzer (smart_analyzer); 528 } else { 529 XCAM_LOG_WARNING ("load smart analyzer(%s) failed, please check.", DEFAULT_SMART_ANALYSIS_LIB_DIR); 530 } 531 } 532 533 image_processor = new CLPostImageProcessor (); 534 XCAM_ASSERT (image_processor.ptr ()); 535 image_processor->set_stats_callback (pipe_manager); 536 image_processor->set_defog_mode ((CLPostImageProcessor::CLDefogMode) xcamfilter->defog_mode); 537 538 if (NONE_WAVELET != xcamfilter->wavelet_mode) { 539 if (HAT_WAVELET_Y == xcamfilter->wavelet_mode) { 540 image_processor->set_wavelet (CL_WAVELET_HAT, CL_IMAGE_CHANNEL_Y, false); 541 } else if (HAT_WAVELET_UV == xcamfilter->wavelet_mode) { 542 image_processor->set_wavelet (CL_WAVELET_HAT, CL_IMAGE_CHANNEL_UV, false); 543 } else if (HARR_WAVELET_Y == xcamfilter->wavelet_mode) { 544 image_processor->set_wavelet (CL_WAVELET_HAAR, CL_IMAGE_CHANNEL_Y, false); 545 } else if (HARR_WAVELET_UV == xcamfilter->wavelet_mode) { 546 image_processor->set_wavelet (CL_WAVELET_HAAR, CL_IMAGE_CHANNEL_UV, false); 547 } else if (HARR_WAVELET_YUV == xcamfilter->wavelet_mode) { 548 image_processor->set_wavelet (CL_WAVELET_HAAR, CL_IMAGE_CHANNEL_UV | CL_IMAGE_CHANNEL_Y, false); 549 } else if (HARR_WAVELET_BAYES == xcamfilter->wavelet_mode) { 550 image_processor->set_wavelet (CL_WAVELET_HAAR, CL_IMAGE_CHANNEL_UV | CL_IMAGE_CHANNEL_Y, true); 551 } else { 552 image_processor->set_wavelet (CL_WAVELET_DISABLED, CL_IMAGE_CHANNEL_UV, false); 553 } 554 } 555 556 image_processor->set_3ddenoise_mode ( 557 (CLPostImageProcessor::CL3DDenoiseMode) xcamfilter->denoise_3d_mode, xcamfilter->denoise_3d_ref_count); 558 559 image_processor->set_wireframe (xcamfilter->enable_wireframe); 560 image_processor->set_image_warp (xcamfilter->enable_image_warp); 561 if (smart_analyzer.ptr ()) { 562 if (xcamfilter->enable_wireframe) 563 image_processor->set_scaler (true); 564 565 if (xcamfilter->enable_image_warp) { 566 image_processor->set_scaler (true); 567 xcamfilter->delay_buf_num = DEFAULT_DELAY_BUFFER_NUM + 16; 568 } 569 } 570 571 pipe_manager->add_image_processor (image_processor); 572 pipe_manager->set_image_processor (image_processor); 573 574 xcamfilter->buf_pool = new CLVideoBufferPool (); 575 XCAM_ASSERT (xcamfilter->buf_pool.ptr ()); 576 if (xcamfilter->copy_mode == COPY_MODE_DMA) { 577 XCAM_LOG_WARNING ("CLVideoBuffer doesn't support DMA copy mode, switch to CPU copy mode"); 578 xcamfilter->copy_mode = COPY_MODE_CPU; 579 } 580 581 if (xcamfilter->copy_mode == COPY_MODE_DMA) { 582 xcamfilter->allocator = gst_dmabuf_allocator_new (); 583 if (!xcamfilter->allocator) { 584 GST_WARNING ("xcamfilter get allocator failed"); 585 return false; 586 } 587 } 588 589 return true; 590 } 591 592 static gboolean 593 gst_xcam_filter_stop (GstBaseTransform *trans) 594 { 595 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 596 597 SmartPtr<BufferPool> buf_pool = xcamfilter->buf_pool; 598 if (buf_pool.ptr ()) 599 buf_pool->stop (); 600 601 SmartPtr<MainPipeManager> pipe_manager = xcamfilter->pipe_manager; 602 if (pipe_manager.ptr ()) 603 pipe_manager->stop (); 604 605 return true; 606 } 607 608 static GstCaps * 609 gst_xcam_filter_transform_caps ( 610 GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter) 611 { 612 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 613 614 GstCaps *src_caps, *peer_caps, *intersect_caps; 615 GstStructure *sink_struct, *src_struct; 616 GstPad *peer_pad; 617 gint sink_width, sink_height, src_width, src_height; 618 619 gboolean is_sink_width = false; 620 gboolean is_sink_height = false; 621 622 src_caps = gst_pad_get_pad_template_caps (trans->srcpad); 623 624 if (direction == GST_PAD_SRC || !gst_caps_is_fixed (caps)) 625 goto filtering; 626 627 sink_struct = gst_caps_get_structure (caps, 0); 628 if (!gst_structure_get_int (sink_struct, "width", &sink_width) || 629 !gst_structure_get_int (sink_struct, "height", &sink_height)) 630 goto filtering; 631 632 peer_pad = gst_pad_get_peer (trans->srcpad); 633 peer_caps = gst_pad_query_caps (peer_pad, src_caps); 634 if (!peer_pad || gst_caps_is_empty (peer_caps)) { 635 if (xcamfilter->enable_stitch) { 636 src_height = XCAM_ALIGN_UP (sink_width / 2, 16); 637 if (src_height * 2 != sink_width) { 638 gst_caps_unref (src_caps); 639 gst_caps_unref (peer_caps); 640 XCAM_LOG_ERROR ("xcamfilter stitch incorrect size, sink-width(%d) / 2 should be aligned with 16", 641 sink_width); 642 return NULL; 643 } 644 src_width = sink_width; 645 646 gst_caps_unref (src_caps); 647 src_caps = gst_caps_copy (caps); 648 src_struct = gst_caps_get_structure (src_caps, 0); 649 650 gst_structure_set (src_struct, "width", G_TYPE_INT, src_width, 651 "height", G_TYPE_INT, src_height, NULL); 652 } 653 654 gst_caps_unref (peer_caps); 655 goto filtering; 656 } 657 658 intersect_caps = gst_caps_intersect_full (peer_caps, src_caps, GST_CAPS_INTERSECT_FIRST); 659 gst_caps_unref (src_caps); 660 src_caps = intersect_caps; 661 662 src_struct = gst_caps_get_structure (src_caps, 0); 663 if (!gst_structure_get_int (src_struct, "width", &src_width)) { 664 is_sink_width = true; 665 src_width = sink_width; 666 } 667 if (!gst_structure_get_int (src_struct, "height", &src_height)) { 668 is_sink_height = true; 669 src_height = sink_height; 670 } 671 672 if (xcamfilter->enable_stitch) { 673 if (is_sink_width && is_sink_height) 674 src_height = XCAM_ALIGN_UP (src_width / 2, 16); 675 676 if (src_width != src_height * 2) { 677 XCAM_LOG_ERROR ("xcamfilter incorrect stitch size width:%d height:%d", src_width, src_height); 678 gst_caps_unref (src_caps); 679 return NULL; 680 } 681 } 682 683 gint fps_n, fps_d; 684 if (!gst_structure_get_fraction (src_struct, "framerate", &fps_n, &fps_d) && 685 !gst_structure_get_fraction (sink_struct, "framerate", &fps_n, &fps_d)) { 686 fps_n = 25; 687 fps_d = 1; 688 } 689 690 gst_structure_set (src_struct, "width", G_TYPE_INT, src_width, 691 "height", G_TYPE_INT, src_height, 692 "framerate", GST_TYPE_FRACTION, fps_n, fps_d, NULL); 693 694 filtering: 695 if (filter) { 696 intersect_caps = gst_caps_intersect_full (filter, src_caps, GST_CAPS_INTERSECT_FIRST); 697 gst_caps_unref (src_caps); 698 src_caps = intersect_caps; 699 } 700 701 return src_caps; 702 } 703 704 static gboolean 705 gst_xcam_filter_set_caps (GstBaseTransform *trans, GstCaps *incaps, GstCaps *outcaps) 706 { 707 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 708 GstVideoInfo in_info, out_info; 709 710 if (!gst_video_info_from_caps (&in_info, incaps) || 711 !gst_video_info_from_caps (&out_info, outcaps)) { 712 XCAM_LOG_WARNING ("fail to parse incaps or outcaps"); 713 return false; 714 } 715 716 XCAM_FAIL_RETURN ( 717 ERROR, 718 GST_VIDEO_INFO_FORMAT (&in_info) == GST_VIDEO_FORMAT_NV12 || 719 GST_VIDEO_INFO_FORMAT (&out_info) == GST_VIDEO_FORMAT_NV12, 720 false, 721 "xcamfilter only support NV12 stream"); 722 xcamfilter->gst_sink_video_info = in_info; 723 xcamfilter->gst_src_video_info = out_info; 724 725 SmartPtr<MainPipeManager> pipe_manager = xcamfilter->pipe_manager; 726 SmartPtr<CLPostImageProcessor> processor = pipe_manager->get_image_processor(); 727 XCAM_ASSERT (pipe_manager.ptr () && processor.ptr ()); 728 if (!processor->set_output_format (V4L2_PIX_FMT_NV12)) 729 return false; 730 731 if (processor->is_scaled ()) 732 processor->set_scaler_factor (640.0 / GST_VIDEO_INFO_WIDTH (&in_info)); 733 //processor->set_scaler_factor (0.5f); 734 735 if (xcamfilter->enable_stitch) { 736 processor->set_image_stitch ( 737 xcamfilter->enable_stitch, xcamfilter->stitch_enable_seam, xcamfilter->stitch_scale_mode, 738 xcamfilter->stitch_fisheye_map, xcamfilter->stitch_lsc, xcamfilter->stitch_fm_ocl, 739 GST_VIDEO_INFO_WIDTH (&out_info), GST_VIDEO_INFO_HEIGHT (&out_info), (uint32_t) xcamfilter->stitch_res_mode); 740 XCAM_LOG_INFO ("xcamfilter stitch output size width:%d height:%d", 741 GST_VIDEO_INFO_WIDTH (&out_info), GST_VIDEO_INFO_HEIGHT (&out_info)); 742 } 743 744 if (pipe_manager->start () != XCAM_RETURN_NO_ERROR) { 745 XCAM_LOG_ERROR ("pipe manager start failed"); 746 return false; 747 } 748 749 VideoBufferInfo buf_info; 750 buf_info.init ( 751 V4L2_PIX_FMT_NV12, 752 GST_VIDEO_INFO_WIDTH (&in_info), 753 GST_VIDEO_INFO_HEIGHT (&in_info), 754 XCAM_ALIGN_UP (GST_VIDEO_INFO_WIDTH (&in_info), 16), 755 XCAM_ALIGN_UP (GST_VIDEO_INFO_HEIGHT (&in_info), 16)); 756 757 SmartPtr<BufferPool> buf_pool = xcamfilter->buf_pool; 758 XCAM_ASSERT (buf_pool.ptr ()); 759 if (!buf_pool->set_video_info (buf_info) || 760 !buf_pool->reserve (xcamfilter->buf_count)) { 761 XCAM_LOG_ERROR ("init buffer pool failed"); 762 return false; 763 } 764 765 return true; 766 } 767 768 static GstFlowReturn 769 copy_gstbuf_to_xcambuf (GstVideoInfo gstinfo, GstBuffer *gstbuf, SmartPtr<VideoBuffer> xcambuf) 770 { 771 GstMapInfo mapinfo; 772 VideoBufferPlanarInfo planar; 773 const VideoBufferInfo xcaminfo = xcambuf->get_video_info (); 774 775 uint8_t *memory = xcambuf->map (); 776 gboolean ret = gst_buffer_map (gstbuf, &mapinfo, GST_MAP_READ); 777 if (!memory || !ret) { 778 XCAM_LOG_WARNING ("xcamfilter map buffer failed"); 779 return GST_FLOW_ERROR; 780 } 781 782 uint8_t *src = NULL; 783 uint8_t *dest = NULL; 784 for (uint32_t index = 0; index < xcaminfo.components; index++) { 785 xcaminfo.get_planar_info (planar, index); 786 787 src = mapinfo.data + GST_VIDEO_INFO_PLANE_OFFSET (&gstinfo, index); 788 dest = memory + xcaminfo.offsets [index]; 789 for (uint32_t i = 0; i < planar.height; i++) { 790 memcpy (dest, src, GST_VIDEO_INFO_WIDTH (&gstinfo)); 791 src += GST_VIDEO_INFO_PLANE_STRIDE (&gstinfo, index); 792 dest += xcaminfo.strides [index]; 793 } 794 } 795 796 gst_buffer_unmap (gstbuf, &mapinfo); 797 xcambuf->unmap (); 798 799 return GST_FLOW_OK; 800 } 801 802 static GstFlowReturn 803 copy_xcambuf_to_gstbuf (GstVideoInfo gstinfo, SmartPtr<VideoBuffer> xcambuf, GstBuffer **gstbuf) 804 { 805 GstMapInfo mapinfo; 806 VideoBufferPlanarInfo planar; 807 const VideoBufferInfo xcaminfo = xcambuf->get_video_info (); 808 809 GstBuffer *tmpbuf = gst_buffer_new_allocate (NULL, GST_VIDEO_INFO_SIZE (&gstinfo), NULL); 810 if (!tmpbuf) { 811 XCAM_LOG_ERROR ("xcamfilter allocate buffer failed"); 812 return GST_FLOW_ERROR; 813 } 814 815 uint8_t *memory = xcambuf->map (); 816 gboolean ret = gst_buffer_map (tmpbuf, &mapinfo, GST_MAP_WRITE); 817 if (!memory || !ret) { 818 XCAM_LOG_WARNING ("xcamfilter map buffer failed"); 819 return GST_FLOW_ERROR; 820 } 821 822 uint8_t *src = NULL; 823 uint8_t *dest = NULL; 824 for (uint32_t index = 0; index < GST_VIDEO_INFO_N_PLANES (&gstinfo); index++) { 825 xcaminfo.get_planar_info (planar, index); 826 827 src = memory + xcaminfo.offsets [index]; 828 dest = mapinfo.data + GST_VIDEO_INFO_PLANE_OFFSET (&gstinfo, index); 829 for (uint32_t i = 0; i < planar.height; i++) { 830 memcpy (dest, src, planar.width); 831 src += xcaminfo.strides [index]; 832 dest += GST_VIDEO_INFO_PLANE_STRIDE (&gstinfo, index); 833 } 834 } 835 836 gst_buffer_unmap (tmpbuf, &mapinfo); 837 xcambuf->unmap (); 838 839 *gstbuf = tmpbuf; 840 841 return GST_FLOW_OK; 842 } 843 844 static GstFlowReturn 845 append_xcambuf_to_gstbuf (GstAllocator *allocator, SmartPtr<VideoBuffer> xcambuf, GstBuffer **gstbuf) 846 { 847 gsize offsets [XCAM_VIDEO_MAX_COMPONENTS]; 848 849 VideoBufferInfo xcaminfo = xcambuf->get_video_info (); 850 for (int i = 0; i < XCAM_VIDEO_MAX_COMPONENTS; i++) { 851 offsets [i] = xcaminfo.offsets [i]; 852 } 853 854 GstBuffer *tmpbuf = gst_buffer_new (); 855 GstMemory *mem = gst_dmabuf_allocator_alloc (allocator, dup (xcambuf->get_fd ()), xcambuf->get_size ()); 856 XCAM_ASSERT (mem); 857 858 gst_buffer_append_memory (tmpbuf, mem); 859 860 gst_buffer_add_video_meta_full ( 861 tmpbuf, 862 GST_VIDEO_FRAME_FLAG_NONE, 863 GST_VIDEO_FORMAT_NV12, 864 xcaminfo.width, 865 xcaminfo.height, 866 xcaminfo.components, 867 offsets, 868 (gint *) (xcaminfo.strides)); 869 870 *gstbuf = tmpbuf; 871 872 return GST_FLOW_OK; 873 } 874 875 static gint 876 get_dmabuf_fd (GstBuffer *buffer) 877 { 878 GstMemory *mem = gst_buffer_peek_memory (buffer, 0); 879 if (!gst_is_dmabuf_memory (mem)) { 880 return -1; 881 } 882 883 return gst_dmabuf_memory_get_fd (mem); 884 } 885 886 static void 887 gst_xcam_filter_before_transform (GstBaseTransform *trans, GstBuffer *buffer) 888 { 889 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 890 891 SmartPtr<BufferPool> buf_pool = xcamfilter->buf_pool; 892 SmartPtr<MainPipeManager> pipe_manager = xcamfilter->pipe_manager; 893 XCAM_ASSERT (buf_pool.ptr () && pipe_manager.ptr ()); 894 895 if (xcamfilter->cached_buf_num > xcamfilter->delay_buf_num) 896 return; 897 898 SmartPtr<VideoBuffer> video_buf; 899 gint dma_fd = get_dmabuf_fd (buffer); 900 if (dma_fd >= 0) { 901 #if HAVE_LIBDRM 902 SmartPtr<DrmBoBufferPool> bo_buf_pool = buf_pool.dynamic_cast_ptr<DrmBoBufferPool> (); 903 SmartPtr<DrmDisplay> display = bo_buf_pool->get_drm_display (); 904 VideoBufferInfo info = bo_buf_pool->get_video_info (); 905 906 SmartPtr<VideoBuffer> dma_buf = new DmaGstBuffer (info, dma_fd, buffer); 907 video_buf = display->convert_to_drm_bo_buf (display, dma_buf); 908 #endif 909 if (!video_buf.ptr ()) { 910 XCAM_LOG_ERROR ("xcamfilter convert to drm bo buffer failed"); 911 return; 912 } 913 } else { 914 video_buf = buf_pool->get_buffer (buf_pool); 915 if (!buf_pool.ptr ()) { 916 XCAM_LOG_ERROR ("xcamfilter sink-pad get buffer failed"); 917 return; 918 } 919 920 copy_gstbuf_to_xcambuf (xcamfilter->gst_sink_video_info, buffer, video_buf); 921 } 922 923 if (pipe_manager->push_buffer (video_buf) != XCAM_RETURN_NO_ERROR) { 924 XCAM_LOG_ERROR ("xcamfilter push buffer failed"); 925 return; 926 } 927 928 xcamfilter->cached_buf_num++; 929 } 930 931 static GstFlowReturn 932 gst_xcam_filter_prepare_output_buffer (GstBaseTransform *trans, GstBuffer *input, GstBuffer **outbuf) 933 { 934 GstXCamFilter *xcamfilter = GST_XCAM_FILTER (trans); 935 GstFlowReturn ret = GST_FLOW_OK; 936 937 SmartPtr<MainPipeManager> pipe_manager = xcamfilter->pipe_manager; 938 SmartPtr<VideoBuffer> video_buf; 939 940 if (xcamfilter->cached_buf_num > xcamfilter->buf_count) 941 return GST_FLOW_ERROR; 942 943 int32_t timeout = -1; 944 if (xcamfilter->cached_buf_num <= xcamfilter->delay_buf_num) 945 timeout = 0; 946 947 video_buf = pipe_manager->dequeue_buffer (timeout); 948 if (!video_buf.ptr ()) { 949 XCAM_LOG_WARNING ("xcamfilter dequeue buffer failed"); 950 *outbuf = NULL; 951 return GST_FLOW_OK; 952 } 953 954 if (xcamfilter->copy_mode == COPY_MODE_CPU) { 955 ret = copy_xcambuf_to_gstbuf (xcamfilter->gst_src_video_info, video_buf, outbuf); 956 } else if (xcamfilter->copy_mode == COPY_MODE_DMA) { 957 GstAllocator *allocator = xcamfilter->allocator; 958 ret = append_xcambuf_to_gstbuf (allocator, video_buf, outbuf); 959 } 960 961 if (ret == GST_FLOW_OK) { 962 xcamfilter->cached_buf_num--; 963 GST_BUFFER_TIMESTAMP (*outbuf) = GST_BUFFER_TIMESTAMP (input); 964 } 965 966 return ret; 967 } 968 969 static GstFlowReturn 970 gst_xcam_filter_transform (GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf) 971 { 972 XCAM_UNUSED (trans); 973 XCAM_UNUSED (inbuf); 974 975 if (!outbuf) { 976 XCAM_LOG_ERROR ("transform failed with null outbufer"); 977 return GST_FLOW_ERROR; 978 } 979 980 XCAM_STATIC_FPS_CALCULATION (gstxcamfilter, XCAM_OBJ_DUR_FRAME_NUM); 981 return GST_FLOW_OK; 982 } 983 984 static gboolean 985 gst_xcam_filter_plugin_init (GstPlugin *xcamfilter) 986 { 987 return gst_element_register (xcamfilter, "xcamfilter", GST_RANK_NONE, 988 GST_TYPE_XCAM_FILTER); 989 } 990 991 #ifndef PACKAGE 992 #define PACKAGE "libxam" 993 #endif 994 995 GST_PLUGIN_DEFINE ( 996 GST_VERSION_MAJOR, 997 GST_VERSION_MINOR, 998 xcamfilter, 999 "Libxcam filter plugin", 1000 gst_xcam_filter_plugin_init, 1001 VERSION, 1002 GST_LICENSE_UNKNOWN, 1003 "libxcamfilter", 1004 "https://github.com/01org/libxcam" 1005 ) 1006
