Home | History | Annotate | Download | only in ocl 
      1 /*
      2  * cl_video_stabilizer.h -  Digital Video Stabilization using IMU (Gyroscope, Accelerometer)
      3  *
      4  *  Copyright (c) 2017 Intel Corporation
      5  *
      6  * Licensed under the Apache License, Version 2.0 (the "License");
      7  * you may not use this file except in compliance with the License.
      8  * You may obtain a copy of the License at
      9  *
     10  *      http://www.apache.org/licenses/LICENSE-2.0
     11  *
     12  * Unless required by applicable law or agreed to in writing, software
     13  * distributed under the License is distributed on an "AS IS" BASIS,
     14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     15  * See the License for the specific language governing permissions and
     16  * limitations under the License.
     17  *
     18  * Author: Zong Wei <wei.zong (at) intel.com>
     19  */
     20 
     21 #ifndef XCAM_CL_VIDEO_STABILIZER_H
     22 #define XCAM_CL_VIDEO_STABILIZER_H
     23 
     24 #include <xcam_std.h>
     25 #include <meta_data.h>
     26 #include <vec_mat.h>
     27 #include <image_projector.h>
     28 #include <ocl/cl_image_warp_handler.h>
     29 
     30 namespace XCam {
     31 
     32 class MotionFilter;
     33 class ImageProjector;
     34 class CLVideoStabilizer;
     35 class CLImageWarpKernel;
     36 class CLImageWarpHandler;
     37 
     38 class CLVideoStabilizerKernel
     39     : public CLImageWarpKernel
     40 {
     41 public:
     42     explicit CLVideoStabilizerKernel (
     43         const SmartPtr<CLContext> &context,
     44         const char *name,
     45         uint32_t channel,
     46         SmartPtr<CLImageHandler> &handler);
     47 
     48 private:
     49     XCAM_DEAD_COPY (CLVideoStabilizerKernel);
     50 
     51     SmartPtr<CLVideoStabilizer> _handler;
     52 };
     53 
     54 class CLVideoStabilizer
     55     : public CLImageWarpHandler
     56 {
     57     typedef std::list<SmartPtr<VideoBuffer>> CLImageBufferList;
     58 
     59 public:
     60     explicit CLVideoStabilizer (
     61         const SmartPtr<CLContext> &context,
     62         const char *name = "CLVideoStabilizer");
     63 
     64     virtual ~CLVideoStabilizer () {
     65         _input_buf_list.clear ();
     66     }
     67 
     68     virtual SmartPtr<VideoBuffer> get_warp_input_buf ();
     69 
     70     virtual bool is_ready ();
     71 
     72     void reset_counter ();
     73 
     74     XCamReturn set_sensor_calibration (CalibrationParams &params);
     75     XCamReturn set_camera_intrinsics (
     76         double focal_x,
     77         double focal_y,
     78         double offset_x,
     79         double offset_y,
     80         double skew);
     81 
     82     XCamReturn align_coordinate_system (
     83         CoordinateSystemConv& world_to_device,
     84         CoordinateSystemConv& device_to_image);
     85 
     86     XCamReturn set_motion_filter (uint32_t radius, float stdev);
     87     uint32_t filter_radius () const {
     88         return _filter_radius;
     89     };
     90 
     91     Mat3d analyze_motion (
     92         int64_t frame0_ts,
     93         DevicePoseList pose0_list,
     94         int64_t frame1_ts,
     95         DevicePoseList pose1_list);
     96 
     97     Mat3d stabilize_motion (int32_t stab_frame_id, std::list<Mat3d> &motions);
     98 
     99 protected:
    100     virtual XCamReturn prepare_parameters (SmartPtr<VideoBuffer> &input, SmartPtr<VideoBuffer> &output);
    101     virtual XCamReturn execute_done (SmartPtr<VideoBuffer> &output);
    102 
    103 private:
    104     XCAM_DEAD_COPY (CLVideoStabilizer);
    105 
    106 private:
    107     Mat3d                    _intrinsics;
    108     CalibrationParams        _calib_params;
    109     SmartPtr<ImageProjector> _projector;
    110     SmartPtr<MotionFilter>   _motion_filter;
    111     CoordinateSystemConv     _world_to_device;
    112     CoordinateSystemConv     _device_to_image;
    113     int64_t                  _input_frame_id;
    114     int64_t                  _frame_ts[2];
    115     int64_t                  _stabilized_frame_id;
    116     DevicePoseList           _device_pose[2];
    117     std::list<Mat3d>         _motions; //motions[i] calculated from frame i to i+1
    118     uint32_t                 _filter_radius;
    119     CLImageBufferList        _input_buf_list;
    120 };
    121 
    122 SmartPtr<CLImageHandler>
    123 create_cl_video_stab_handler (const SmartPtr<CLContext> &context);
    124 
    125 
    126 class MotionFilter
    127 {
    128 public:
    129     MotionFilter (uint32_t radius = 15, float stdev = 10);
    130     virtual ~MotionFilter ();
    131 
    132     void set_filters (uint32_t radius, float stdev);
    133 
    134     uint32_t radius () const {
    135         return _radius;
    136     };
    137     float stdev () const {
    138         return _stdev;
    139     };
    140 
    141     Mat3d stabilize (int32_t index,
    142                      std::list<Mat3d> &motions,
    143                      int32_t max);
    144 
    145 protected:
    146     Mat3d cumulate_motion (uint32_t index, uint32_t from, std::list<Mat3d> &motions);
    147 
    148 private:
    149     XCAM_DEAD_COPY (MotionFilter);
    150 
    151 private:
    152     int32_t            _radius;
    153     float              _stdev;
    154     std::vector<float> _weight;
    155 };
    156 
    157 }
    158 #endif
    159