1 // libjingle 2 // Copyright 2010 Google Inc. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // 1. Redistributions of source code must retain the above copyright notice, 8 // this list of conditions and the following disclaimer. 9 // 2. Redistributions in binary form must reproduce the above copyright notice, 10 // this list of conditions and the following disclaimer in the documentation 11 // and/or other materials provided with the distribution. 12 // 3. The name of the author may not be used to endorse or promote products 13 // derived from this software without specific prior written permission. 14 // 15 // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 16 // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 17 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 18 // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 24 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 // 26 // Declaration of abstract class VideoCapturer 27 28 #ifndef TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 29 #define TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 30 31 #include <string> 32 #include <vector> 33 34 #include "talk/base/basictypes.h" 35 #include "talk/base/criticalsection.h" 36 #include "talk/base/messagehandler.h" 37 #include "talk/base/rollingaccumulator.h" 38 #include "talk/base/scoped_ptr.h" 39 #include "talk/base/sigslot.h" 40 #include "talk/base/thread.h" 41 #include "talk/base/timing.h" 42 #include "talk/media/base/mediachannel.h" 43 #include "talk/media/base/videoadapter.h" 44 #include "talk/media/base/videocommon.h" 45 #include "talk/media/devices/devicemanager.h" 46 47 48 namespace cricket { 49 50 class VideoProcessor; 51 52 // Current state of the capturer. 53 // TODO(hellner): CS_NO_DEVICE is an error code not a capture state. Separate 54 // error codes and states. 55 enum CaptureState { 56 CS_STOPPED, // The capturer has been stopped or hasn't started yet. 57 CS_STARTING, // The capturer is in the process of starting. Note, it may 58 // still fail to start. 59 CS_RUNNING, // The capturer has been started successfully and is now 60 // capturing. 61 CS_PAUSED, // The capturer has been paused. 62 CS_FAILED, // The capturer failed to start. 63 CS_NO_DEVICE, // The capturer has no device and consequently failed to start. 64 }; 65 66 class VideoFrame; 67 68 struct CapturedFrame { 69 static const uint32 kFrameHeaderSize = 40; // Size from width to data_size. 70 static const uint32 kUnknownDataSize = 0xFFFFFFFF; 71 72 CapturedFrame(); 73 74 // Get the number of bytes of the frame data. If data_size is known, return 75 // it directly. Otherwise, calculate the size based on width, height, and 76 // fourcc. Return true if succeeded. 77 bool GetDataSize(uint32* size) const; 78 79 // The width and height of the captured frame could be different from those 80 // of VideoFormat. Once the first frame is captured, the width, height, 81 // fourcc, pixel_width, and pixel_height should keep the same over frames. 82 int width; // in number of pixels 83 int height; // in number of pixels 84 uint32 fourcc; // compression 85 uint32 pixel_width; // width of a pixel, default is 1 86 uint32 pixel_height; // height of a pixel, default is 1 87 int64 elapsed_time; // elapsed time since the creation of the frame 88 // source (that is, the camera), in nanoseconds. 89 int64 time_stamp; // timestamp of when the frame was captured, in unix 90 // time with nanosecond units. 91 uint32 data_size; // number of bytes of the frame data 92 int rotation; // rotation in degrees of the frame (0, 90, 180, 270) 93 void* data; // pointer to the frame data. This object allocates the 94 // memory or points to an existing memory. 95 96 private: 97 DISALLOW_COPY_AND_ASSIGN(CapturedFrame); 98 }; 99 100 // VideoCapturer is an abstract class that defines the interfaces for video 101 // capturing. The subclasses implement the video capturer for various types of 102 // capturers and various platforms. 103 // 104 // The captured frames may need to be adapted (for example, cropping). 105 // Video adaptation is built into and enabled by default. After a frame has 106 // been captured from the device, it is sent to the video adapter, then video 107 // processors, then out to the encoder. 108 // 109 // Programming model: 110 // Create an object of a subclass of VideoCapturer 111 // Initialize 112 // SignalStateChange.connect() 113 // SignalFrameCaptured.connect() 114 // Find the capture format for Start() by either calling GetSupportedFormats() 115 // and selecting one of the supported or calling GetBestCaptureFormat(). 116 // video_adapter()->OnOutputFormatRequest(desired_encoding_format) 117 // Start() 118 // GetCaptureFormat() optionally 119 // Stop() 120 // 121 // Assumption: 122 // The Start() and Stop() methods are called by a single thread (E.g., the 123 // media engine thread). Hence, the VideoCapture subclasses dont need to be 124 // thread safe. 125 // 126 class VideoCapturer 127 : public sigslot::has_slots<>, 128 public talk_base::MessageHandler { 129 public: 130 typedef std::vector<VideoProcessor*> VideoProcessors; 131 132 // All signals are marshalled to |thread| or the creating thread if 133 // none is provided. 134 VideoCapturer(); 135 explicit VideoCapturer(talk_base::Thread* thread); 136 virtual ~VideoCapturer() {} 137 138 // Gets the id of the underlying device, which is available after the capturer 139 // is initialized. Can be used to determine if two capturers reference the 140 // same device. 141 const std::string& GetId() const { return id_; } 142 143 // Get the capture formats supported by the video capturer. The supported 144 // formats are non empty after the device has been opened successfully. 145 const std::vector<VideoFormat>* GetSupportedFormats() const; 146 147 // Get the best capture format for the desired format. The best format is the 148 // same as one of the supported formats except that the frame interval may be 149 // different. If the application asks for 16x9 and the camera does not support 150 // 16x9 HD or the application asks for 16x10, we find the closest 4x3 and then 151 // crop; Otherwise, we find what the application asks for. Note that we assume 152 // that for HD, the desired format is always 16x9. The subclasses can override 153 // the default implementation. 154 // Parameters 155 // desired: the input desired format. If desired.fourcc is not kAnyFourcc, 156 // the best capture format has the exactly same fourcc. Otherwise, 157 // the best capture format uses a fourcc in GetPreferredFourccs(). 158 // best_format: the output of the best capture format. 159 // Return false if there is no such a best format, that is, the desired format 160 // is not supported. 161 virtual bool GetBestCaptureFormat(const VideoFormat& desired, 162 VideoFormat* best_format); 163 164 // TODO(hellner): deprecate (make private) the Start API in favor of this one. 165 // Also remove CS_STARTING as it is implied by the return 166 // value of StartCapturing(). 167 bool StartCapturing(const VideoFormat& capture_format); 168 // Start the video capturer with the specified capture format. 169 // Parameter 170 // capture_format: The caller got this parameter by either calling 171 // GetSupportedFormats() and selecting one of the supported 172 // or calling GetBestCaptureFormat(). 173 // Return 174 // CS_STARTING: The capturer is trying to start. Success or failure will 175 // be notified via the |SignalStateChange| callback. 176 // CS_RUNNING: if the capturer is started and capturing. 177 // CS_PAUSED: Will never be returned. 178 // CS_FAILED: if the capturer failes to start.. 179 // CS_NO_DEVICE: if the capturer has no device and fails to start. 180 virtual CaptureState Start(const VideoFormat& capture_format) = 0; 181 // Sets the desired aspect ratio. If the capturer is capturing at another 182 // aspect ratio it will crop the width or the height so that asked for 183 // aspect ratio is acheived. Note that ratio_w and ratio_h do not need to be 184 // relatively prime. 185 void UpdateAspectRatio(int ratio_w, int ratio_h); 186 void ClearAspectRatio(); 187 188 // Get the current capture format, which is set by the Start() call. 189 // Note that the width and height of the captured frames may differ from the 190 // capture format. For example, the capture format is HD but the captured 191 // frames may be smaller than HD. 192 const VideoFormat* GetCaptureFormat() const { 193 return capture_format_.get(); 194 } 195 196 // Pause the video capturer. 197 virtual bool Pause(bool paused); 198 // Stop the video capturer. 199 virtual void Stop() = 0; 200 // Check if the video capturer is running. 201 virtual bool IsRunning() = 0; 202 // Restart the video capturer with the new |capture_format|. 203 // Default implementation stops and starts the capturer. 204 virtual bool Restart(const VideoFormat& capture_format); 205 // TODO(thorcarpenter): This behavior of keeping the camera open just to emit 206 // black frames is a total hack and should be fixed. 207 // When muting, produce black frames then pause the camera. 208 // When unmuting, start the camera. Camera starts unmuted. 209 virtual bool MuteToBlackThenPause(bool muted); 210 virtual bool IsMuted() const { 211 return muted_; 212 } 213 CaptureState capture_state() const { 214 return capture_state_; 215 } 216 217 // Adds a video processor that will be applied on VideoFrames returned by 218 // |SignalVideoFrame|. Multiple video processors can be added. The video 219 // processors will be applied in the order they were added. 220 void AddVideoProcessor(VideoProcessor* video_processor); 221 // Removes the |video_processor| from the list of video processors or 222 // returns false. 223 bool RemoveVideoProcessor(VideoProcessor* video_processor); 224 225 // Returns true if the capturer is screencasting. This can be used to 226 // implement screencast specific behavior. 227 virtual bool IsScreencast() const = 0; 228 229 // Caps the VideoCapturer's format according to max_format. It can e.g. be 230 // used to prevent cameras from capturing at a resolution or framerate that 231 // the capturer is capable of but not performing satisfactorily at. 232 // The capping is an upper bound for each component of the capturing format. 233 // The fourcc component is ignored. 234 void ConstrainSupportedFormats(const VideoFormat& max_format); 235 236 void set_enable_camera_list(bool enable_camera_list) { 237 enable_camera_list_ = enable_camera_list; 238 } 239 bool enable_camera_list() { 240 return enable_camera_list_; 241 } 242 243 // Enable scaling to ensure square pixels. 244 void set_square_pixel_aspect_ratio(bool square_pixel_aspect_ratio) { 245 square_pixel_aspect_ratio_ = square_pixel_aspect_ratio; 246 } 247 bool square_pixel_aspect_ratio() { 248 return square_pixel_aspect_ratio_; 249 } 250 251 // Signal all capture state changes that are not a direct result of calling 252 // Start(). 253 sigslot::signal2<VideoCapturer*, CaptureState> SignalStateChange; 254 // Frame callbacks are multithreaded to allow disconnect and connect to be 255 // called concurrently. It also ensures that it is safe to call disconnect 256 // at any time which is needed since the signal may be called from an 257 // unmarshalled thread owned by the VideoCapturer. 258 // Signal the captured frame to downstream. 259 sigslot::signal2<VideoCapturer*, const CapturedFrame*, 260 sigslot::multi_threaded_local> SignalFrameCaptured; 261 // Signal the captured and possibly adapted frame to downstream consumers 262 // such as the encoder. 263 sigslot::signal2<VideoCapturer*, const VideoFrame*, 264 sigslot::multi_threaded_local> SignalVideoFrame; 265 266 const VideoProcessors& video_processors() const { return video_processors_; } 267 268 // If 'screencast_max_pixels' is set greater than zero, screencasts will be 269 // scaled to be no larger than this value. 270 // If set to zero, the max pixels will be limited to 271 // Retina MacBookPro 15" resolution of 2880 x 1800. 272 // For high fps, maximum pixels limit is set based on common 24" monitor 273 // resolution of 2048 x 1280. 274 int screencast_max_pixels() const { return screencast_max_pixels_; } 275 void set_screencast_max_pixels(int p) { 276 screencast_max_pixels_ = talk_base::_max(0, p); 277 } 278 279 // If true, run video adaptation. By default, video adaptation is enabled 280 // and users must call video_adapter()->OnOutputFormatRequest() 281 // to receive frames. 282 bool enable_video_adapter() const { return enable_video_adapter_; } 283 void set_enable_video_adapter(bool enable_video_adapter) { 284 enable_video_adapter_ = enable_video_adapter; 285 } 286 287 CoordinatedVideoAdapter* video_adapter() { return &video_adapter_; } 288 const CoordinatedVideoAdapter* video_adapter() const { 289 return &video_adapter_; 290 } 291 292 // Gets statistics for tracked variables recorded since the last call to 293 // GetStats. Note that calling GetStats resets any gathered data so it 294 // should be called only periodically to log statistics. 295 void GetStats(VariableInfo<int>* adapt_drop_stats, 296 VariableInfo<int>* effect_drop_stats, 297 VariableInfo<double>* frame_time_stats, 298 VideoFormat* last_captured_frame_format); 299 300 protected: 301 // Callback attached to SignalFrameCaptured where SignalVideoFrames is called. 302 void OnFrameCaptured(VideoCapturer* video_capturer, 303 const CapturedFrame* captured_frame); 304 void SetCaptureState(CaptureState state); 305 306 // Marshals SignalStateChange onto thread_. 307 void OnMessage(talk_base::Message* message); 308 309 // subclasses override this virtual method to provide a vector of fourccs, in 310 // order of preference, that are expected by the media engine. 311 virtual bool GetPreferredFourccs(std::vector<uint32>* fourccs) = 0; 312 313 // mutators to set private attributes 314 void SetId(const std::string& id) { 315 id_ = id; 316 } 317 318 void SetCaptureFormat(const VideoFormat* format) { 319 capture_format_.reset(format ? new VideoFormat(*format) : NULL); 320 if (capture_format_) { 321 ASSERT(capture_format_->interval > 0 && 322 "Capture format expected to have positive interval."); 323 // Video adapter really only cares about capture format interval. 324 video_adapter_.SetInputFormat(*capture_format_); 325 } 326 } 327 328 void SetSupportedFormats(const std::vector<VideoFormat>& formats); 329 330 private: 331 void Construct(); 332 // Get the distance between the desired format and the supported format. 333 // Return the max distance if they mismatch. See the implementation for 334 // details. 335 int64 GetFormatDistance(const VideoFormat& desired, 336 const VideoFormat& supported); 337 338 // Convert captured frame to readable string for LOG messages. 339 std::string ToString(const CapturedFrame* frame) const; 340 341 // Applies all registered processors. If any of the processors signal that 342 // the frame should be dropped the return value will be false. Note that 343 // this frame should be dropped as it has not applied all processors. 344 bool ApplyProcessors(VideoFrame* video_frame); 345 346 // Updates filtered_supported_formats_ so that it contains the formats in 347 // supported_formats_ that fulfill all applied restrictions. 348 void UpdateFilteredSupportedFormats(); 349 // Returns true if format doesn't fulfill all applied restrictions. 350 bool ShouldFilterFormat(const VideoFormat& format) const; 351 352 void UpdateStats(const CapturedFrame* captured_frame); 353 354 // Helper function to save statistics on the current data from a 355 // RollingAccumulator into stats. 356 template<class T> 357 static void GetVariableSnapshot( 358 const talk_base::RollingAccumulator<T>& data, 359 VariableInfo<T>* stats); 360 361 talk_base::Thread* thread_; 362 std::string id_; 363 CaptureState capture_state_; 364 talk_base::scoped_ptr<VideoFormat> capture_format_; 365 std::vector<VideoFormat> supported_formats_; 366 talk_base::scoped_ptr<VideoFormat> max_format_; 367 std::vector<VideoFormat> filtered_supported_formats_; 368 369 int ratio_w_; // View resolution. e.g. 1280 x 720. 370 int ratio_h_; 371 bool enable_camera_list_; 372 bool square_pixel_aspect_ratio_; // Enable scaling to square pixels. 373 int scaled_width_; // Current output size from ComputeScale. 374 int scaled_height_; 375 int screencast_max_pixels_; // Downscale screencasts further if requested. 376 bool muted_; 377 int black_frame_count_down_; 378 379 bool enable_video_adapter_; 380 CoordinatedVideoAdapter video_adapter_; 381 382 talk_base::Timing frame_length_time_reporter_; 383 talk_base::CriticalSection frame_stats_crit_; 384 385 int adapt_frame_drops_; 386 talk_base::RollingAccumulator<int> adapt_frame_drops_data_; 387 int effect_frame_drops_; 388 talk_base::RollingAccumulator<int> effect_frame_drops_data_; 389 double previous_frame_time_; 390 talk_base::RollingAccumulator<double> frame_time_data_; 391 // The captured frame format before potential adapation. 392 VideoFormat last_captured_frame_format_; 393 394 talk_base::CriticalSection crit_; 395 VideoProcessors video_processors_; 396 397 DISALLOW_COPY_AND_ASSIGN(VideoCapturer); 398 }; 399 400 } // namespace cricket 401 402 #endif // TALK_MEDIA_BASE_VIDEOCAPTURER_H_ 403