Home | History | Annotate | Download | only in media_stream_video 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include <GLES2/gl2.h>
      6 #include <GLES2/gl2ext.h>
      7 #include <string.h>
      8 
      9 #include <vector>
     10 
     11 #include "ppapi/c/pp_errors.h"
     12 #include "ppapi/c/ppb_opengles2.h"
     13 #include "ppapi/cpp/completion_callback.h"
     14 #include "ppapi/cpp/graphics_3d.h"
     15 #include "ppapi/cpp/graphics_3d_client.h"
     16 #include "ppapi/cpp/instance.h"
     17 #include "ppapi/cpp/media_stream_video_track.h"
     18 #include "ppapi/cpp/module.h"
     19 #include "ppapi/cpp/rect.h"
     20 #include "ppapi/cpp/var.h"
     21 #include "ppapi/cpp/var_dictionary.h"
     22 #include "ppapi/cpp/video_frame.h"
     23 #include "ppapi/lib/gl/gles2/gl2ext_ppapi.h"
     24 #include "ppapi/utility/completion_callback_factory.h"
     25 
     26 // When compiling natively on Windows, PostMessage can be #define-d to
     27 // something else.
     28 #ifdef PostMessage
     29 #undef PostMessage
     30 #endif
     31 
     32 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
     33 // function to preserve line number information in the failure message.
     34 #define AssertNoGLError() \
     35   PP_DCHECK(!glGetError());
     36 
     37 namespace {
     38 
     39 // This object is the global object representing this plugin library as long
     40 // as it is loaded.
     41 class MediaStreamVideoModule : public pp::Module {
     42  public:
     43   MediaStreamVideoModule() : pp::Module() {}
     44   virtual ~MediaStreamVideoModule() {}
     45 
     46   virtual pp::Instance* CreateInstance(PP_Instance instance);
     47 };
     48 
     49 class MediaStreamVideoDemoInstance : public pp::Instance,
     50                         public pp::Graphics3DClient {
     51  public:
     52   MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module);
     53   virtual ~MediaStreamVideoDemoInstance();
     54 
     55   // pp::Instance implementation (see PPP_Instance).
     56   virtual void DidChangeView(const pp::Rect& position,
     57                              const pp::Rect& clip_ignored);
     58   virtual void HandleMessage(const pp::Var& message_data);
     59 
     60   // pp::Graphics3DClient implementation.
     61   virtual void Graphics3DContextLost() {
     62     InitGL();
     63     CreateTextures();
     64     Render();
     65   }
     66 
     67  private:
     68   void DrawYUV();
     69   void DrawRGB();
     70   void Render();
     71 
     72   // GL-related functions.
     73   void InitGL();
     74   GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba);
     75   void CreateGLObjects();
     76   void CreateShader(GLuint program, GLenum type, const char* source);
     77   void PaintFinished(int32_t result);
     78   void CreateTextures();
     79   void ConfigureTrack();
     80 
     81 
     82   // MediaStreamVideoTrack callbacks.
     83   void OnConfigure(int32_t result);
     84   void OnGetFrame(int32_t result, pp::VideoFrame frame);
     85 
     86   pp::Size position_size_;
     87   bool is_painting_;
     88   bool needs_paint_;
     89   bool is_bgra_;
     90   GLuint program_yuv_;
     91   GLuint program_rgb_;
     92   GLuint buffer_;
     93   GLuint texture_y_;
     94   GLuint texture_u_;
     95   GLuint texture_v_;
     96   GLuint texture_rgb_;
     97   pp::MediaStreamVideoTrack video_track_;
     98   pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_;
     99   std::vector<int32_t> attrib_list_;
    100 
    101   // MediaStreamVideoTrack attributes:
    102   bool need_config_;
    103   PP_VideoFrame_Format attrib_format_;
    104   int32_t attrib_width_;
    105   int32_t attrib_height_;
    106 
    107   // Owned data.
    108   pp::Graphics3D* context_;
    109 
    110   pp::Size frame_size_;
    111 };
    112 
    113 MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance(
    114     PP_Instance instance, pp::Module* module)
    115     : pp::Instance(instance),
    116       pp::Graphics3DClient(this),
    117       is_painting_(false),
    118       needs_paint_(false),
    119       is_bgra_(false),
    120       texture_y_(0),
    121       texture_u_(0),
    122       texture_v_(0),
    123       texture_rgb_(0),
    124       callback_factory_(this),
    125       need_config_(false),
    126       attrib_format_(PP_VIDEOFRAME_FORMAT_I420),
    127       attrib_width_(0),
    128       attrib_height_(0),
    129       context_(NULL) {
    130   if (!glInitializePPAPI(pp::Module::Get()->get_browser_interface())) {
    131     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Unable to initialize GL PPAPI!"));
    132     assert(false);
    133   }
    134 }
    135 
    136 MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() {
    137   delete context_;
    138 }
    139 
    140 void MediaStreamVideoDemoInstance::DidChangeView(
    141     const pp::Rect& position, const pp::Rect& clip_ignored) {
    142   if (position.width() == 0 || position.height() == 0)
    143     return;
    144   if (position.size() == position_size_)
    145     return;
    146 
    147   position_size_ = position.size();
    148 
    149   // Initialize graphics.
    150   InitGL();
    151   Render();
    152 }
    153 
    154 void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) {
    155   if (!var_message.is_dictionary()) {
    156     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
    157     return;
    158   }
    159 
    160   pp::VarDictionary var_dictionary_message(var_message);
    161   std::string command = var_dictionary_message.Get("command").AsString();
    162 
    163   if (command == "init") {
    164     pp::Var var_track = var_dictionary_message.Get("track");
    165     if (!var_track.is_resource())
    166       return;
    167     pp::Resource resource_track = var_track.AsResource();
    168     video_track_ = pp::MediaStreamVideoTrack(resource_track);
    169     ConfigureTrack();
    170   } else if (command == "format") {
    171     std::string str_format = var_dictionary_message.Get("format").AsString();
    172     if (str_format == "YV12") {
    173       attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12;
    174     } else if (str_format == "I420") {
    175       attrib_format_ = PP_VIDEOFRAME_FORMAT_I420;
    176     } else if (str_format == "BGRA") {
    177       attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA;
    178     } else {
    179       attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN;
    180     }
    181     need_config_ = true;
    182   } else if (command == "size") {
    183     attrib_width_ = var_dictionary_message.Get("width").AsInt();
    184     attrib_height_ = var_dictionary_message.Get("height").AsInt();
    185     need_config_ = true;
    186   } else {
    187     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
    188   }
    189 }
    190 
    191 void MediaStreamVideoDemoInstance::InitGL() {
    192   PP_DCHECK(position_size_.width() && position_size_.height());
    193   is_painting_ = false;
    194 
    195   delete context_;
    196   int32_t attributes[] = {
    197     PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
    198     PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
    199     PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
    200     PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
    201     PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
    202     PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
    203     PP_GRAPHICS3DATTRIB_SAMPLES, 0,
    204     PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
    205     PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
    206     PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
    207     PP_GRAPHICS3DATTRIB_NONE,
    208   };
    209   context_ = new pp::Graphics3D(this, attributes);
    210   PP_DCHECK(!context_->is_null());
    211 
    212   glSetCurrentContextPPAPI(context_->pp_resource());
    213 
    214   // Set viewport window size and clear color bit.
    215   glClearColor(1, 0, 0, 1);
    216   glClear(GL_COLOR_BUFFER_BIT);
    217   glViewport(0, 0, position_size_.width(), position_size_.height());
    218 
    219   BindGraphics(*context_);
    220   AssertNoGLError();
    221 
    222   CreateGLObjects();
    223 }
    224 
    225 void MediaStreamVideoDemoInstance::DrawYUV() {
    226   static const float kColorMatrix[9] = {
    227     1.1643828125f, 1.1643828125f, 1.1643828125f,
    228     0.0f, -0.39176171875f, 2.017234375f,
    229     1.59602734375f, -0.81296875f, 0.0f
    230   };
    231 
    232   glUseProgram(program_yuv_);
    233   glUniform1i(glGetUniformLocation(program_yuv_, "y_texture"), 0);
    234   glUniform1i(glGetUniformLocation(program_yuv_, "u_texture"), 1);
    235   glUniform1i(glGetUniformLocation(program_yuv_, "v_texture"), 2);
    236   glUniformMatrix3fv(glGetUniformLocation(program_yuv_, "color_matrix"),
    237       1, GL_FALSE, kColorMatrix);
    238   AssertNoGLError();
    239 
    240   GLint pos_location = glGetAttribLocation(program_yuv_, "a_position");
    241   GLint tc_location = glGetAttribLocation(program_yuv_, "a_texCoord");
    242   AssertNoGLError();
    243   glEnableVertexAttribArray(pos_location);
    244   glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
    245   glEnableVertexAttribArray(tc_location);
    246   glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
    247       static_cast<float*>(0) + 16);  // Skip position coordinates.
    248   AssertNoGLError();
    249 
    250   glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
    251   AssertNoGLError();
    252 }
    253 
    254 void MediaStreamVideoDemoInstance::DrawRGB() {
    255   glUseProgram(program_rgb_);
    256   glUniform1i(glGetUniformLocation(program_rgb_, "rgb_texture"), 3);
    257   AssertNoGLError();
    258 
    259   GLint pos_location = glGetAttribLocation(program_rgb_, "a_position");
    260   GLint tc_location = glGetAttribLocation(program_rgb_, "a_texCoord");
    261   AssertNoGLError();
    262   glEnableVertexAttribArray(pos_location);
    263   glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
    264   glEnableVertexAttribArray(tc_location);
    265   glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
    266       static_cast<float*>(0) + 16);  // Skip position coordinates.
    267   AssertNoGLError();
    268 
    269   glDrawArrays(GL_TRIANGLE_STRIP, 4, 4);
    270 }
    271 
    272 void MediaStreamVideoDemoInstance::Render() {
    273   PP_DCHECK(!is_painting_);
    274   is_painting_ = true;
    275   needs_paint_ = false;
    276 
    277   if (texture_y_) {
    278     DrawRGB();
    279     DrawYUV();
    280   } else {
    281     glClear(GL_COLOR_BUFFER_BIT);
    282   }
    283   pp::CompletionCallback cb = callback_factory_.NewCallback(
    284       &MediaStreamVideoDemoInstance::PaintFinished);
    285   context_->SwapBuffers(cb);
    286 }
    287 
    288 void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) {
    289   is_painting_ = false;
    290   if (needs_paint_)
    291     Render();
    292 }
    293 
    294 GLuint MediaStreamVideoDemoInstance::CreateTexture(
    295     int32_t width, int32_t height, int unit, bool rgba) {
    296   GLuint texture_id;
    297   glGenTextures(1, &texture_id);
    298   AssertNoGLError();
    299 
    300   // Assign parameters.
    301   glActiveTexture(GL_TEXTURE0 + unit);
    302   glBindTexture(GL_TEXTURE_2D, texture_id);
    303   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    304   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    305   glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    306   glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    307   // Allocate texture.
    308   glTexImage2D(GL_TEXTURE_2D, 0,
    309                rgba ? GL_BGRA_EXT : GL_LUMINANCE,
    310                width, height, 0,
    311                rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
    312   AssertNoGLError();
    313   return texture_id;
    314 }
    315 
    316 void MediaStreamVideoDemoInstance::CreateGLObjects() {
    317   // Code and constants for shader.
    318   static const char kVertexShader[] =
    319       "varying vec2 v_texCoord;            \n"
    320       "attribute vec4 a_position;          \n"
    321       "attribute vec2 a_texCoord;          \n"
    322       "void main()                         \n"
    323       "{                                   \n"
    324       "    v_texCoord = a_texCoord;        \n"
    325       "    gl_Position = a_position;       \n"
    326       "}";
    327 
    328   static const char kFragmentShaderYUV[] =
    329       "precision mediump float;                                   \n"
    330       "varying vec2 v_texCoord;                                   \n"
    331       "uniform sampler2D y_texture;                               \n"
    332       "uniform sampler2D u_texture;                               \n"
    333       "uniform sampler2D v_texture;                               \n"
    334       "uniform mat3 color_matrix;                                 \n"
    335       "void main()                                                \n"
    336       "{                                                          \n"
    337       "  vec3 yuv;                                                \n"
    338       "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
    339       "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
    340       "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
    341       "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
    342       "  gl_FragColor = vec4(rgb, 1.0);                           \n"
    343       "}";
    344 
    345   static const char kFragmentShaderRGB[] =
    346       "precision mediump float;                                   \n"
    347       "varying vec2 v_texCoord;                                   \n"
    348       "uniform sampler2D rgb_texture;                             \n"
    349       "void main()                                                \n"
    350       "{                                                          \n"
    351       "  gl_FragColor = texture2D(rgb_texture, v_texCoord);       \n"
    352       "}";
    353 
    354   // Create shader programs.
    355   program_yuv_ = glCreateProgram();
    356   CreateShader(program_yuv_, GL_VERTEX_SHADER, kVertexShader);
    357   CreateShader(program_yuv_, GL_FRAGMENT_SHADER, kFragmentShaderYUV);
    358   glLinkProgram(program_yuv_);
    359   AssertNoGLError();
    360 
    361   program_rgb_ = glCreateProgram();
    362   CreateShader(program_rgb_, GL_VERTEX_SHADER, kVertexShader);
    363   CreateShader(program_rgb_, GL_FRAGMENT_SHADER, kFragmentShaderRGB);
    364   glLinkProgram(program_rgb_);
    365   AssertNoGLError();
    366 
    367   // Assign vertex positions and texture coordinates to buffers for use in
    368   // shader program.
    369   static const float kVertices[] = {
    370     -1, 1, -1, -1, 0, 1, 0, -1,  // Position coordinates.
    371     0, 1, 0, -1, 1, 1, 1, -1,  // Position coordinates.
    372     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
    373     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
    374   };
    375 
    376   glGenBuffers(1, &buffer_);
    377   glBindBuffer(GL_ARRAY_BUFFER, buffer_);
    378   glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
    379   AssertNoGLError();
    380 }
    381 
    382 void MediaStreamVideoDemoInstance::CreateShader(
    383     GLuint program, GLenum type, const char* source) {
    384   GLuint shader = glCreateShader(type);
    385   GLint length = strlen(source) + 1;
    386   glShaderSource(shader, 1, &source, &length);
    387   glCompileShader(shader);
    388   glAttachShader(program, shader);
    389   glDeleteShader(shader);
    390 }
    391 
    392 void MediaStreamVideoDemoInstance::CreateTextures() {
    393   int32_t width = frame_size_.width();
    394   int32_t height = frame_size_.height();
    395   if (width == 0 || height == 0)
    396     return;
    397   if (texture_y_)
    398     glDeleteTextures(1, &texture_y_);
    399   if (texture_u_)
    400     glDeleteTextures(1, &texture_u_);
    401   if (texture_v_)
    402     glDeleteTextures(1, &texture_v_);
    403   if (texture_rgb_)
    404     glDeleteTextures(1, &texture_rgb_);
    405   texture_y_ = CreateTexture(width, height, 0, false);
    406 
    407   texture_u_ = CreateTexture(width / 2, height / 2, 1, false);
    408   texture_v_ = CreateTexture(width / 2, height / 2, 2, false);
    409   texture_rgb_ = CreateTexture(width, height, 3, true);
    410 }
    411 
    412 void MediaStreamVideoDemoInstance::ConfigureTrack() {
    413   const int32_t attrib_list[] = {
    414       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_,
    415       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_,
    416       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_,
    417       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE
    418     };
    419   video_track_.Configure(attrib_list, callback_factory_.NewCallback(
    420         &MediaStreamVideoDemoInstance::OnConfigure));
    421 }
    422 
    423 void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) {
    424   video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
    425       &MediaStreamVideoDemoInstance::OnGetFrame));
    426 }
    427 
    428 void MediaStreamVideoDemoInstance::OnGetFrame(
    429     int32_t result, pp::VideoFrame frame) {
    430   if (result != PP_OK)
    431     return;
    432   const char* data = static_cast<const char*>(frame.GetDataBuffer());
    433   pp::Size size;
    434   frame.GetSize(&size);
    435 
    436   if (size != frame_size_) {
    437     frame_size_ = size;
    438     CreateTextures();
    439   }
    440 
    441   is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA);
    442 
    443   int32_t width = frame_size_.width();
    444   int32_t height = frame_size_.height();
    445   if (!is_bgra_) {
    446     glActiveTexture(GL_TEXTURE0);
    447     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
    448                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
    449 
    450     data += width * height;
    451     width /= 2;
    452     height /= 2;
    453 
    454     glActiveTexture(GL_TEXTURE1);
    455     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
    456                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
    457 
    458     data += width * height;
    459     glActiveTexture(GL_TEXTURE2);
    460     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
    461                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
    462   } else {
    463     glActiveTexture(GL_TEXTURE3);
    464     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
    465                     GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
    466   }
    467 
    468   if (is_painting_)
    469     needs_paint_ = true;
    470   else
    471     Render();
    472 
    473   video_track_.RecycleFrame(frame);
    474   if (need_config_) {
    475     ConfigureTrack();
    476     need_config_ = false;
    477   } else {
    478     video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
    479         &MediaStreamVideoDemoInstance::OnGetFrame));
    480   }
    481 }
    482 
    483 pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) {
    484   return new MediaStreamVideoDemoInstance(instance, this);
    485 }
    486 
    487 }  // anonymous namespace
    488 
    489 namespace pp {
    490 // Factory function for your specialization of the Module object.
    491 Module* CreateModule() {
    492   return new MediaStreamVideoModule();
    493 }
    494 }  // namespace pp
    495