Home | History | Annotate | Download | only in video_capture 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include <stdlib.h>
      6 #include <string.h>
      7 
      8 #include <map>
      9 #include <vector>
     10 
     11 #include "ppapi/c/dev/ppb_video_capture_dev.h"
     12 #include "ppapi/c/pp_errors.h"
     13 #include "ppapi/c/ppb_opengles2.h"
     14 #include "ppapi/cpp/dev/buffer_dev.h"
     15 #include "ppapi/cpp/dev/device_ref_dev.h"
     16 #include "ppapi/cpp/dev/video_capture_dev.h"
     17 #include "ppapi/cpp/dev/video_capture_client_dev.h"
     18 #include "ppapi/cpp/completion_callback.h"
     19 #include "ppapi/cpp/graphics_3d_client.h"
     20 #include "ppapi/cpp/graphics_3d.h"
     21 #include "ppapi/cpp/instance.h"
     22 #include "ppapi/cpp/module.h"
     23 #include "ppapi/cpp/rect.h"
     24 #include "ppapi/cpp/var.h"
     25 #include "ppapi/lib/gl/include/GLES2/gl2.h"
     26 #include "ppapi/utility/completion_callback_factory.h"
     27 
     28 // When compiling natively on Windows, PostMessage can be #define-d to
     29 // something else.
     30 #ifdef PostMessage
     31 #undef PostMessage
     32 #endif
     33 
     34 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
     35 // function to preserve line number information in the failure message.
     36 #define AssertNoGLError() \
     37   PP_DCHECK(!gles2_if_->GetError(context_->pp_resource()));
     38 
     39 namespace {
     40 
     41 const char* const kDelimiter = "#__#";
     42 
     43 // This object is the global object representing this plugin library as long
     44 // as it is loaded.
     45 class VCDemoModule : public pp::Module {
     46  public:
     47   VCDemoModule() : pp::Module() {}
     48   virtual ~VCDemoModule() {}
     49 
     50   virtual pp::Instance* CreateInstance(PP_Instance instance);
     51 };
     52 
     53 class VCDemoInstance : public pp::Instance,
     54                        public pp::Graphics3DClient,
     55                        public pp::VideoCaptureClient_Dev {
     56  public:
     57   VCDemoInstance(PP_Instance instance, pp::Module* module);
     58   virtual ~VCDemoInstance();
     59 
     60   // pp::Instance implementation (see PPP_Instance).
     61   virtual void DidChangeView(const pp::Rect& position,
     62                              const pp::Rect& clip_ignored);
     63   virtual void HandleMessage(const pp::Var& message_data);
     64 
     65   // pp::Graphics3DClient implementation.
     66   virtual void Graphics3DContextLost() {
     67     InitGL();
     68     CreateYUVTextures();
     69     Render();
     70   }
     71 
     72   virtual void OnDeviceInfo(PP_Resource resource,
     73                             const PP_VideoCaptureDeviceInfo_Dev& info,
     74                             const std::vector<pp::Buffer_Dev>& buffers) {
     75     capture_info_ = info;
     76     buffers_ = buffers;
     77     CreateYUVTextures();
     78   }
     79 
     80   virtual void OnStatus(PP_Resource resource, uint32_t status) {
     81   }
     82 
     83   virtual void OnError(PP_Resource resource, uint32_t error) {
     84   }
     85 
     86   virtual void OnBufferReady(PP_Resource resource, uint32_t buffer) {
     87     const char* data = static_cast<const char*>(buffers_[buffer].data());
     88     int32_t width = capture_info_.width;
     89     int32_t height = capture_info_.height;
     90     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0);
     91     gles2_if_->TexSubImage2D(
     92         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
     93         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
     94 
     95     data += width * height;
     96     width /= 2;
     97     height /= 2;
     98 
     99     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE1);
    100     gles2_if_->TexSubImage2D(
    101         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
    102         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
    103 
    104     data += width * height;
    105     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE2);
    106     gles2_if_->TexSubImage2D(
    107         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
    108         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
    109 
    110     video_capture_.ReuseBuffer(buffer);
    111     if (is_painting_)
    112       needs_paint_ = true;
    113     else
    114       Render();
    115   }
    116 
    117  private:
    118   void Render();
    119 
    120   // GL-related functions.
    121   void InitGL();
    122   GLuint CreateTexture(int32_t width, int32_t height, int unit);
    123   void CreateGLObjects();
    124   void CreateShader(GLuint program, GLenum type, const char* source, int size);
    125   void PaintFinished(int32_t result);
    126   void CreateYUVTextures();
    127 
    128   void Open(const pp::DeviceRef_Dev& device);
    129   void Stop();
    130   void Start();
    131   void EnumerateDevicesFinished(int32_t result,
    132                                 std::vector<pp::DeviceRef_Dev>& devices);
    133   void OpenFinished(int32_t result);
    134 
    135   static void MonitorDeviceChangeCallback(void* user_data,
    136                                           uint32_t device_count,
    137                                           const PP_Resource devices[]);
    138 
    139   pp::Size position_size_;
    140   bool is_painting_;
    141   bool needs_paint_;
    142   GLuint texture_y_;
    143   GLuint texture_u_;
    144   GLuint texture_v_;
    145   pp::VideoCapture_Dev video_capture_;
    146   PP_VideoCaptureDeviceInfo_Dev capture_info_;
    147   std::vector<pp::Buffer_Dev> buffers_;
    148   pp::CompletionCallbackFactory<VCDemoInstance> callback_factory_;
    149 
    150   // Unowned pointers.
    151   const struct PPB_OpenGLES2* gles2_if_;
    152 
    153   // Owned data.
    154   pp::Graphics3D* context_;
    155 
    156   std::vector<pp::DeviceRef_Dev> enumerate_devices_;
    157   std::vector<pp::DeviceRef_Dev> monitor_devices_;
    158 };
    159 
    160 VCDemoInstance::VCDemoInstance(PP_Instance instance, pp::Module* module)
    161     : pp::Instance(instance),
    162       pp::Graphics3DClient(this),
    163       pp::VideoCaptureClient_Dev(this),
    164       is_painting_(false),
    165       needs_paint_(false),
    166       texture_y_(0),
    167       texture_u_(0),
    168       texture_v_(0),
    169       video_capture_(this),
    170       callback_factory_(this),
    171       context_(NULL) {
    172   gles2_if_ = static_cast<const struct PPB_OpenGLES2*>(
    173       module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
    174   PP_DCHECK(gles2_if_);
    175 
    176   capture_info_.width = 320;
    177   capture_info_.height = 240;
    178   capture_info_.frames_per_second = 30;
    179 }
    180 
    181 VCDemoInstance::~VCDemoInstance() {
    182   video_capture_.MonitorDeviceChange(NULL, NULL);
    183   delete context_;
    184 }
    185 
    186 void VCDemoInstance::DidChangeView(
    187     const pp::Rect& position, const pp::Rect& clip_ignored) {
    188   if (position.width() == 0 || position.height() == 0)
    189     return;
    190   if (position.size() == position_size_)
    191     return;
    192 
    193   position_size_ = position.size();
    194 
    195   // Initialize graphics.
    196   InitGL();
    197 
    198   Render();
    199 }
    200 
    201 void VCDemoInstance::HandleMessage(const pp::Var& message_data) {
    202   if (message_data.is_string()) {
    203     std::string event = message_data.AsString();
    204     if (event == "PageInitialized") {
    205       int32_t result = video_capture_.MonitorDeviceChange(
    206           &VCDemoInstance::MonitorDeviceChangeCallback, this);
    207       if (result != PP_OK)
    208         PostMessage(pp::Var("MonitorDeviceChangeFailed"));
    209 
    210       pp::CompletionCallbackWithOutput<std::vector<pp::DeviceRef_Dev> >
    211           callback = callback_factory_.NewCallbackWithOutput(
    212               &VCDemoInstance::EnumerateDevicesFinished);
    213       result = video_capture_.EnumerateDevices(callback);
    214       if (result != PP_OK_COMPLETIONPENDING)
    215         PostMessage(pp::Var("EnumerationFailed"));
    216     } else if (event == "UseDefault") {
    217       Open(pp::DeviceRef_Dev());
    218     } else if (event == "Stop") {
    219       Stop();
    220     } else if (event == "Start") {
    221       Start();
    222     } else if (event.find("Monitor:") == 0) {
    223       std::string index_str = event.substr(strlen("Monitor:"));
    224       int index = atoi(index_str.c_str());
    225       if (index >= 0 && index < static_cast<int>(monitor_devices_.size()))
    226         Open(monitor_devices_[index]);
    227       else
    228         PP_NOTREACHED();
    229     } else if (event.find("Enumerate:") == 0) {
    230       std::string index_str = event.substr(strlen("Enumerate:"));
    231       int index = atoi(index_str.c_str());
    232       if (index >= 0 && index < static_cast<int>(enumerate_devices_.size()))
    233         Open(enumerate_devices_[index]);
    234       else
    235         PP_NOTREACHED();
    236     }
    237   }
    238 }
    239 
    240 void VCDemoInstance::InitGL() {
    241   PP_DCHECK(position_size_.width() && position_size_.height());
    242   is_painting_ = false;
    243 
    244   delete context_;
    245   int32_t attributes[] = {
    246     PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
    247     PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
    248     PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
    249     PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
    250     PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
    251     PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
    252     PP_GRAPHICS3DATTRIB_SAMPLES, 0,
    253     PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
    254     PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
    255     PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
    256     PP_GRAPHICS3DATTRIB_NONE,
    257   };
    258   context_ = new pp::Graphics3D(this, attributes);
    259   PP_DCHECK(!context_->is_null());
    260 
    261   // Set viewport window size and clear color bit.
    262   gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
    263   gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
    264   gles2_if_->Viewport(context_->pp_resource(), 0, 0,
    265                       position_size_.width(), position_size_.height());
    266 
    267   BindGraphics(*context_);
    268   AssertNoGLError();
    269 
    270   CreateGLObjects();
    271 }
    272 
    273 void VCDemoInstance::Render() {
    274   PP_DCHECK(!is_painting_);
    275   is_painting_ = true;
    276   needs_paint_ = false;
    277   if (texture_y_) {
    278     gles2_if_->DrawArrays(context_->pp_resource(), GL_TRIANGLE_STRIP, 0, 4);
    279   } else {
    280     gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
    281   }
    282   pp::CompletionCallback cb = callback_factory_.NewCallback(
    283       &VCDemoInstance::PaintFinished);
    284   context_->SwapBuffers(cb);
    285 }
    286 
    287 void VCDemoInstance::PaintFinished(int32_t result) {
    288   is_painting_ = false;
    289   if (needs_paint_)
    290     Render();
    291 }
    292 
    293 GLuint VCDemoInstance::CreateTexture(int32_t width, int32_t height, int unit) {
    294   GLuint texture_id;
    295   gles2_if_->GenTextures(context_->pp_resource(), 1, &texture_id);
    296   AssertNoGLError();
    297   // Assign parameters.
    298   gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0 + unit);
    299   gles2_if_->BindTexture(context_->pp_resource(), GL_TEXTURE_2D, texture_id);
    300   gles2_if_->TexParameteri(
    301       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
    302       GL_NEAREST);
    303   gles2_if_->TexParameteri(
    304       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
    305       GL_NEAREST);
    306   gles2_if_->TexParameterf(
    307       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
    308       GL_CLAMP_TO_EDGE);
    309   gles2_if_->TexParameterf(
    310       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
    311       GL_CLAMP_TO_EDGE);
    312 
    313   // Allocate texture.
    314   gles2_if_->TexImage2D(
    315       context_->pp_resource(), GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0,
    316       GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
    317   AssertNoGLError();
    318   return texture_id;
    319 }
    320 
    321 void VCDemoInstance::CreateGLObjects() {
    322   // Code and constants for shader.
    323   static const char kVertexShader[] =
    324       "varying vec2 v_texCoord;            \n"
    325       "attribute vec4 a_position;          \n"
    326       "attribute vec2 a_texCoord;          \n"
    327       "void main()                         \n"
    328       "{                                   \n"
    329       "    v_texCoord = a_texCoord;        \n"
    330       "    gl_Position = a_position;       \n"
    331       "}";
    332 
    333   static const char kFragmentShader[] =
    334       "precision mediump float;                                   \n"
    335       "varying vec2 v_texCoord;                                   \n"
    336       "uniform sampler2D y_texture;                               \n"
    337       "uniform sampler2D u_texture;                               \n"
    338       "uniform sampler2D v_texture;                               \n"
    339       "uniform mat3 color_matrix;                                 \n"
    340       "void main()                                                \n"
    341       "{                                                          \n"
    342       "  vec3 yuv;                                                \n"
    343       "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
    344       "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
    345       "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
    346       "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
    347       "  gl_FragColor = vec4(rgb, 1.0);                           \n"
    348       "}";
    349 
    350   static const float kColorMatrix[9] = {
    351     1.1643828125f, 1.1643828125f, 1.1643828125f,
    352     0.0f, -0.39176171875f, 2.017234375f,
    353     1.59602734375f, -0.81296875f, 0.0f
    354   };
    355 
    356   PP_Resource context = context_->pp_resource();
    357 
    358   // Create shader program.
    359   GLuint program = gles2_if_->CreateProgram(context);
    360   CreateShader(program, GL_VERTEX_SHADER, kVertexShader, sizeof(kVertexShader));
    361   CreateShader(
    362       program, GL_FRAGMENT_SHADER, kFragmentShader, sizeof(kFragmentShader));
    363   gles2_if_->LinkProgram(context, program);
    364   gles2_if_->UseProgram(context, program);
    365   gles2_if_->DeleteProgram(context, program);
    366   gles2_if_->Uniform1i(
    367       context, gles2_if_->GetUniformLocation(context, program, "y_texture"), 0);
    368   gles2_if_->Uniform1i(
    369       context, gles2_if_->GetUniformLocation(context, program, "u_texture"), 1);
    370   gles2_if_->Uniform1i(
    371       context, gles2_if_->GetUniformLocation(context, program, "v_texture"), 2);
    372   gles2_if_->UniformMatrix3fv(
    373       context,
    374       gles2_if_->GetUniformLocation(context, program, "color_matrix"),
    375       1, GL_FALSE, kColorMatrix);
    376   AssertNoGLError();
    377 
    378   // Assign vertex positions and texture coordinates to buffers for use in
    379   // shader program.
    380   static const float kVertices[] = {
    381     -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
    382     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
    383   };
    384 
    385   GLuint buffer;
    386   gles2_if_->GenBuffers(context, 1, &buffer);
    387   gles2_if_->BindBuffer(context, GL_ARRAY_BUFFER, buffer);
    388   gles2_if_->BufferData(context, GL_ARRAY_BUFFER,
    389                         sizeof(kVertices), kVertices, GL_STATIC_DRAW);
    390   AssertNoGLError();
    391   GLint pos_location = gles2_if_->GetAttribLocation(
    392       context, program, "a_position");
    393   GLint tc_location = gles2_if_->GetAttribLocation(
    394       context, program, "a_texCoord");
    395   AssertNoGLError();
    396   gles2_if_->EnableVertexAttribArray(context, pos_location);
    397   gles2_if_->VertexAttribPointer(context, pos_location, 2,
    398                                  GL_FLOAT, GL_FALSE, 0, 0);
    399   gles2_if_->EnableVertexAttribArray(context, tc_location);
    400   gles2_if_->VertexAttribPointer(
    401       context, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
    402       static_cast<float*>(0) + 8);  // Skip position coordinates.
    403   AssertNoGLError();
    404 }
    405 
    406 void VCDemoInstance::CreateShader(
    407     GLuint program, GLenum type, const char* source, int size) {
    408   PP_Resource context = context_->pp_resource();
    409   GLuint shader = gles2_if_->CreateShader(context, type);
    410   gles2_if_->ShaderSource(context, shader, 1, &source, &size);
    411   gles2_if_->CompileShader(context, shader);
    412   gles2_if_->AttachShader(context, program, shader);
    413   gles2_if_->DeleteShader(context, shader);
    414 }
    415 
    416 void VCDemoInstance::CreateYUVTextures() {
    417   int32_t width = capture_info_.width;
    418   int32_t height = capture_info_.height;
    419   texture_y_ = CreateTexture(width, height, 0);
    420 
    421   width /= 2;
    422   height /= 2;
    423   texture_u_ = CreateTexture(width, height, 1);
    424   texture_v_ = CreateTexture(width, height, 2);
    425 }
    426 
    427 void VCDemoInstance::Open(const pp::DeviceRef_Dev& device) {
    428   pp::CompletionCallback callback = callback_factory_.NewCallback(
    429       &VCDemoInstance::OpenFinished);
    430   int32_t result = video_capture_.Open(device, capture_info_, 4, callback);
    431   if (result != PP_OK_COMPLETIONPENDING)
    432     PostMessage(pp::Var("OpenFailed"));
    433 }
    434 
    435 void VCDemoInstance::Stop() {
    436   if (video_capture_.StopCapture() != PP_OK)
    437     PostMessage(pp::Var("StopFailed"));
    438 }
    439 
    440 void VCDemoInstance::Start() {
    441   if (video_capture_.StartCapture() != PP_OK)
    442     PostMessage(pp::Var("StartFailed"));
    443 }
    444 
    445 void VCDemoInstance::EnumerateDevicesFinished(
    446     int32_t result,
    447   std::vector<pp::DeviceRef_Dev>& devices) {
    448   if (result == PP_OK) {
    449     enumerate_devices_.swap(devices);
    450     std::string device_names = "Enumerate:";
    451     for (size_t index = 0; index < enumerate_devices_.size(); ++index) {
    452       pp::Var name = enumerate_devices_[index].GetName();
    453       PP_DCHECK(name.is_string());
    454 
    455       if (index != 0)
    456         device_names += kDelimiter;
    457       device_names += name.AsString();
    458     }
    459     PostMessage(pp::Var(device_names));
    460   } else {
    461     PostMessage(pp::Var("EnumerationFailed"));
    462   }
    463 }
    464 
    465 void VCDemoInstance::OpenFinished(int32_t result) {
    466   if (result == PP_OK)
    467     Start();
    468   else
    469     PostMessage(pp::Var("OpenFailed"));
    470 }
    471 
    472 // static
    473 void VCDemoInstance::MonitorDeviceChangeCallback(void* user_data,
    474                                                  uint32_t device_count,
    475                                                  const PP_Resource devices[]) {
    476   VCDemoInstance* thiz = static_cast<VCDemoInstance*>(user_data);
    477 
    478   std::string device_names = "Monitor:";
    479   thiz->monitor_devices_.clear();
    480   thiz->monitor_devices_.reserve(device_count);
    481   for (size_t index = 0; index < device_count; ++index) {
    482     thiz->monitor_devices_.push_back(pp::DeviceRef_Dev(devices[index]));
    483     pp::Var name = thiz->monitor_devices_.back().GetName();
    484     PP_DCHECK(name.is_string());
    485 
    486     if (index != 0)
    487       device_names += kDelimiter;
    488     device_names += name.AsString();
    489   }
    490   thiz->PostMessage(pp::Var(device_names));
    491 }
    492 
    493 pp::Instance* VCDemoModule::CreateInstance(PP_Instance instance) {
    494   return new VCDemoInstance(instance, this);
    495 }
    496 
    497 }  // anonymous namespace
    498 
    499 namespace pp {
    500 // Factory function for your specialization of the Module object.
    501 Module* CreateModule() {
    502   return new VCDemoModule();
    503 }
    504 }  // namespace pp
    505