1 /* 2 * Copyright (C) 2007 OpenedHand 3 * Copyright (C) 2007 Alp Toker <alp (at) atoker.com> 4 * 5 * This library is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU Lesser General Public 7 * License as published by the Free Software Foundation; either 8 * version 2 of the License, or (at your option) any later version. 9 * 10 * This library is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * Lesser General Public License for more details. 14 * 15 * You should have received a copy of the GNU Lesser General Public 16 * License along with this library; if not, write to the Free Software 17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 */ 19 20 /** 21 * SECTION:webkit-video-sink 22 * @short_description: GStreamer video sink 23 * 24 * #WebKitVideoSink is a GStreamer sink element that triggers 25 * repaints in the WebKit GStreamer media player for the 26 * current video buffer. 27 */ 28 29 #include "config.h" 30 #include "VideoSinkGStreamer.h" 31 #if USE(GSTREAMER) 32 33 #include <glib.h> 34 #include <gst/gst.h> 35 #include <gst/video/video.h> 36 37 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE("sink", 38 GST_PAD_SINK, GST_PAD_ALWAYS, 39 // CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant. 40 #if G_BYTE_ORDER == G_LITTLE_ENDIAN 41 GST_STATIC_CAPS(GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGRA) 42 #else 43 GST_STATIC_CAPS(GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_ARGB) 44 #endif 45 ); 46 47 GST_DEBUG_CATEGORY_STATIC(webkit_video_sink_debug); 48 #define GST_CAT_DEFAULT webkit_video_sink_debug 49 50 enum { 51 REPAINT_REQUESTED, 52 LAST_SIGNAL 53 }; 54 55 enum { 56 PROP_0 57 }; 58 59 static guint webkit_video_sink_signals[LAST_SIGNAL] = { 0, }; 60 61 struct _WebKitVideoSinkPrivate { 62 GstBuffer* buffer; 63 guint timeout_id; 64 GMutex* buffer_mutex; 65 GCond* data_cond; 66 67 // If this is TRUE all processing should finish ASAP 68 // This is necessary because there could be a race between 69 // unlock() and render(), where unlock() wins, signals the 70 // GCond, then render() tries to render a frame although 71 // everything else isn't running anymore. This will lead 72 // to deadlocks because render() holds the stream lock. 73 // 74 // Protected by the buffer mutex 75 gboolean unlocked; 76 }; 77 78 #define _do_init(bla) \ 79 GST_DEBUG_CATEGORY_INIT(webkit_video_sink_debug, \ 80 "webkitsink", \ 81 0, \ 82 "webkit video sink") 83 84 GST_BOILERPLATE_FULL(WebKitVideoSink, 85 webkit_video_sink, 86 GstVideoSink, 87 GST_TYPE_VIDEO_SINK, 88 _do_init); 89 90 static void 91 webkit_video_sink_base_init(gpointer g_class) 92 { 93 GstElementClass* element_class = GST_ELEMENT_CLASS(g_class); 94 95 gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&sinktemplate)); 96 gst_element_class_set_details_simple(element_class, "WebKit video sink", 97 "Sink/Video", "Sends video data from a GStreamer pipeline to a Cairo surface", 98 "Alp Toker <alp (at) atoker.com>"); 99 } 100 101 static void 102 webkit_video_sink_init(WebKitVideoSink* sink, WebKitVideoSinkClass* klass) 103 { 104 WebKitVideoSinkPrivate* priv; 105 106 sink->priv = priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate); 107 priv->data_cond = g_cond_new(); 108 priv->buffer_mutex = g_mutex_new(); 109 } 110 111 static gboolean 112 webkit_video_sink_timeout_func(gpointer data) 113 { 114 WebKitVideoSink* sink = reinterpret_cast<WebKitVideoSink*>(data); 115 WebKitVideoSinkPrivate* priv = sink->priv; 116 GstBuffer* buffer; 117 118 g_mutex_lock(priv->buffer_mutex); 119 buffer = priv->buffer; 120 priv->buffer = 0; 121 priv->timeout_id = 0; 122 123 if (!buffer || priv->unlocked || G_UNLIKELY(!GST_IS_BUFFER(buffer))) { 124 g_cond_signal(priv->data_cond); 125 g_mutex_unlock(priv->buffer_mutex); 126 return FALSE; 127 } 128 129 g_signal_emit(sink, webkit_video_sink_signals[REPAINT_REQUESTED], 0, buffer); 130 gst_buffer_unref(buffer); 131 g_cond_signal(priv->data_cond); 132 g_mutex_unlock(priv->buffer_mutex); 133 134 return FALSE; 135 } 136 137 static GstFlowReturn 138 webkit_video_sink_render(GstBaseSink* bsink, GstBuffer* buffer) 139 { 140 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(bsink); 141 WebKitVideoSinkPrivate* priv = sink->priv; 142 143 g_mutex_lock(priv->buffer_mutex); 144 145 if (priv->unlocked) { 146 g_mutex_unlock(priv->buffer_mutex); 147 return GST_FLOW_OK; 148 } 149 150 priv->buffer = gst_buffer_ref(buffer); 151 152 // For the unlikely case where the buffer has no caps, the caps 153 // are implicitely the caps of the pad. This shouldn't happen. 154 if (G_UNLIKELY(!GST_BUFFER_CAPS(buffer))) { 155 buffer = priv->buffer = gst_buffer_make_metadata_writable(priv->buffer); 156 gst_buffer_set_caps(priv->buffer, GST_PAD_CAPS(GST_BASE_SINK_PAD(bsink))); 157 } 158 159 GstCaps *caps = GST_BUFFER_CAPS(buffer); 160 GstVideoFormat format; 161 int width, height; 162 if (G_UNLIKELY(!gst_video_format_parse_caps(caps, &format, &width, &height))) { 163 gst_buffer_unref(buffer); 164 g_mutex_unlock(priv->buffer_mutex); 165 return GST_FLOW_ERROR; 166 } 167 168 // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't. 169 // Here we convert to Cairo's ARGB. 170 if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) { 171 // Because GstBaseSink::render() only owns the buffer reference in the 172 // method scope we can't use gst_buffer_make_writable() here. Also 173 // The buffer content should not be changed here because the same buffer 174 // could be passed multiple times to this method (in theory) 175 GstBuffer *newBuffer = gst_buffer_try_new_and_alloc(GST_BUFFER_SIZE(buffer)); 176 177 // Check if allocation failed 178 if (G_UNLIKELY(!newBuffer)) { 179 gst_buffer_unref(buffer); 180 g_mutex_unlock(priv->buffer_mutex); 181 return GST_FLOW_ERROR; 182 } 183 184 gst_buffer_copy_metadata(newBuffer, buffer, (GstBufferCopyFlags) GST_BUFFER_COPY_ALL); 185 186 // We don't use Color::premultipliedARGBFromColor() here because 187 // one function call per video pixel is just too expensive: 188 // For 720p/PAL for example this means 1280*720*25=23040000 189 // function calls per second! 190 unsigned short alpha; 191 const guint8 *source = GST_BUFFER_DATA(buffer); 192 guint8 *destination = GST_BUFFER_DATA(newBuffer); 193 194 for (int x = 0; x < height; x++) { 195 for (int y = 0; y < width; y++) { 196 #if G_BYTE_ORDER == G_LITTLE_ENDIAN 197 alpha = source[3]; 198 destination[0] = (source[0] * alpha + 128) / 255; 199 destination[1] = (source[1] * alpha + 128) / 255; 200 destination[2] = (source[2] * alpha + 128) / 255; 201 destination[3] = alpha; 202 #else 203 alpha = source[0]; 204 destination[0] = alpha; 205 destination[1] = (source[1] * alpha + 128) / 255; 206 destination[2] = (source[2] * alpha + 128) / 255; 207 destination[3] = (source[3] * alpha + 128) / 255; 208 #endif 209 source += 4; 210 destination += 4; 211 } 212 } 213 gst_buffer_unref(buffer); 214 buffer = priv->buffer = newBuffer; 215 } 216 217 // This should likely use a lower priority, but glib currently starves 218 // lower priority sources. 219 // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830. 220 priv->timeout_id = g_timeout_add_full(G_PRIORITY_DEFAULT, 0, 221 webkit_video_sink_timeout_func, 222 gst_object_ref(sink), 223 (GDestroyNotify)gst_object_unref); 224 225 g_cond_wait(priv->data_cond, priv->buffer_mutex); 226 g_mutex_unlock(priv->buffer_mutex); 227 return GST_FLOW_OK; 228 } 229 230 static void 231 webkit_video_sink_dispose(GObject* object) 232 { 233 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object); 234 WebKitVideoSinkPrivate* priv = sink->priv; 235 236 if (priv->data_cond) { 237 g_cond_free(priv->data_cond); 238 priv->data_cond = 0; 239 } 240 241 if (priv->buffer_mutex) { 242 g_mutex_free(priv->buffer_mutex); 243 priv->buffer_mutex = 0; 244 } 245 246 G_OBJECT_CLASS(parent_class)->dispose(object); 247 } 248 249 static void 250 unlock_buffer_mutex(WebKitVideoSinkPrivate* priv) 251 { 252 g_mutex_lock(priv->buffer_mutex); 253 254 if (priv->buffer) { 255 gst_buffer_unref(priv->buffer); 256 priv->buffer = 0; 257 } 258 259 priv->unlocked = TRUE; 260 261 g_cond_signal(priv->data_cond); 262 g_mutex_unlock(priv->buffer_mutex); 263 } 264 265 static gboolean 266 webkit_video_sink_unlock(GstBaseSink* object) 267 { 268 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object); 269 270 unlock_buffer_mutex(sink->priv); 271 272 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, 273 (object), TRUE); 274 } 275 276 static gboolean 277 webkit_video_sink_unlock_stop(GstBaseSink* object) 278 { 279 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object); 280 WebKitVideoSinkPrivate* priv = sink->priv; 281 282 g_mutex_lock(priv->buffer_mutex); 283 priv->unlocked = FALSE; 284 g_mutex_unlock(priv->buffer_mutex); 285 286 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, 287 (object), TRUE); 288 } 289 290 static gboolean 291 webkit_video_sink_stop(GstBaseSink* base_sink) 292 { 293 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv; 294 295 unlock_buffer_mutex(priv); 296 return TRUE; 297 } 298 299 static gboolean 300 webkit_video_sink_start(GstBaseSink* base_sink) 301 { 302 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv; 303 304 g_mutex_lock(priv->buffer_mutex); 305 priv->unlocked = FALSE; 306 g_mutex_unlock(priv->buffer_mutex); 307 return TRUE; 308 } 309 310 static void 311 marshal_VOID__MINIOBJECT(GClosure * closure, GValue * return_value, 312 guint n_param_values, const GValue * param_values, 313 gpointer invocation_hint, gpointer marshal_data) 314 { 315 typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2); 316 marshalfunc_VOID__MINIOBJECT callback; 317 GCClosure *cc = (GCClosure *) closure; 318 gpointer data1, data2; 319 320 g_return_if_fail(n_param_values == 2); 321 322 if (G_CCLOSURE_SWAP_DATA(closure)) { 323 data1 = closure->data; 324 data2 = g_value_peek_pointer(param_values + 0); 325 } else { 326 data1 = g_value_peek_pointer(param_values + 0); 327 data2 = closure->data; 328 } 329 callback = (marshalfunc_VOID__MINIOBJECT) (marshal_data ? marshal_data : cc->callback); 330 331 callback(data1, gst_value_get_mini_object(param_values + 1), data2); 332 } 333 334 static void 335 webkit_video_sink_class_init(WebKitVideoSinkClass* klass) 336 { 337 GObjectClass* gobject_class = G_OBJECT_CLASS(klass); 338 GstBaseSinkClass* gstbase_sink_class = GST_BASE_SINK_CLASS(klass); 339 340 g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate)); 341 342 gobject_class->dispose = webkit_video_sink_dispose; 343 344 gstbase_sink_class->unlock = webkit_video_sink_unlock; 345 gstbase_sink_class->unlock_stop = webkit_video_sink_unlock_stop; 346 gstbase_sink_class->render = webkit_video_sink_render; 347 gstbase_sink_class->preroll = webkit_video_sink_render; 348 gstbase_sink_class->stop = webkit_video_sink_stop; 349 gstbase_sink_class->start = webkit_video_sink_start; 350 351 webkit_video_sink_signals[REPAINT_REQUESTED] = g_signal_new("repaint-requested", 352 G_TYPE_FROM_CLASS(klass), 353 (GSignalFlags)(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION), 354 0, 355 0, 356 0, 357 marshal_VOID__MINIOBJECT, 358 G_TYPE_NONE, 1, GST_TYPE_BUFFER); 359 } 360 361 /** 362 * webkit_video_sink_new: 363 * 364 * Creates a new GStreamer video sink. 365 * 366 * Return value: a #GstElement for the newly created video sink 367 */ 368 GstElement* 369 webkit_video_sink_new(void) 370 { 371 return (GstElement*)g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0); 372 } 373 374 #endif // USE(GSTREAMER) 375
