Home | History | Annotate | Download | only in objc 
      1 /*
      2  * libjingle
      3  * Copyright 2014, Google Inc.
      4  *
      5  * Redistribution and use in source and binary forms, with or without
      6  * modification, are permitted provided that the following conditions are met:
      7  *
      8  *  1. Redistributions of source code must retain the above copyright notice,
      9  *     this list of conditions and the following disclaimer.
     10  *  2. Redistributions in binary form must reproduce the above copyright notice,
     11  *     this list of conditions and the following disclaimer in the documentation
     12  *     and/or other materials provided with the distribution.
     13  *  3. The name of the author may not be used to endorse or promote products
     14  *     derived from this software without specific prior written permission.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     17  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     18  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
     19  * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #if !defined(__has_feature) || !__has_feature(objc_arc)
     29 #error "This file requires ARC support."
     30 #endif
     31 
     32 #import "RTCOpenGLVideoRenderer.h"
     33 
     34 #if TARGET_OS_IPHONE
     35 #import <OpenGLES/ES2/gl.h>
     36 #else
     37 #import <OpenGL/gl3.h>
     38 #endif
     39 
     40 #import "RTCI420Frame.h"
     41 
     42 // TODO(tkchin): check and log openGL errors. Methods here return BOOLs in
     43 // anticipation of that happening in the future.
     44 
     45 #if TARGET_OS_IPHONE
     46 #define RTC_PIXEL_FORMAT GL_LUMINANCE
     47 #define SHADER_VERSION
     48 #define VERTEX_SHADER_IN "attribute"
     49 #define VERTEX_SHADER_OUT "varying"
     50 #define FRAGMENT_SHADER_IN "varying"
     51 #define FRAGMENT_SHADER_OUT
     52 #define FRAGMENT_SHADER_COLOR "gl_FragColor"
     53 #define FRAGMENT_SHADER_TEXTURE "texture2D"
     54 #else
     55 #define RTC_PIXEL_FORMAT GL_RED
     56 #define SHADER_VERSION "#version 150\n"
     57 #define VERTEX_SHADER_IN "in"
     58 #define VERTEX_SHADER_OUT "out"
     59 #define FRAGMENT_SHADER_IN "in"
     60 #define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n"
     61 #define FRAGMENT_SHADER_COLOR "fragColor"
     62 #define FRAGMENT_SHADER_TEXTURE "texture"
     63 #endif
     64 
     65 // Vertex shader doesn't do anything except pass coordinates through.
     66 static const char kVertexShaderSource[] =
     67   SHADER_VERSION
     68   VERTEX_SHADER_IN " vec2 position;\n"
     69   VERTEX_SHADER_IN " vec2 texcoord;\n"
     70   VERTEX_SHADER_OUT " vec2 v_texcoord;\n"
     71   "void main() {\n"
     72   "    gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n"
     73   "    v_texcoord = texcoord;\n"
     74   "}\n";
     75 
     76 // Fragment shader converts YUV values from input textures into a final RGB
     77 // pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php.
     78 static const char kFragmentShaderSource[] =
     79   SHADER_VERSION
     80   "precision highp float;"
     81   FRAGMENT_SHADER_IN " vec2 v_texcoord;\n"
     82   "uniform lowp sampler2D s_textureY;\n"
     83   "uniform lowp sampler2D s_textureU;\n"
     84   "uniform lowp sampler2D s_textureV;\n"
     85   FRAGMENT_SHADER_OUT
     86   "void main() {\n"
     87   "    float y, u, v, r, g, b;\n"
     88   "    y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n"
     89   "    u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n"
     90   "    v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n"
     91   "    u = u - 0.5;\n"
     92   "    v = v - 0.5;\n"
     93   "    r = y + 1.403 * v;\n"
     94   "    g = y - 0.344 * u - 0.714 * v;\n"
     95   "    b = y + 1.770 * u;\n"
     96   "    " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n"
     97   "  }\n";
     98 
     99 // Compiles a shader of the given |type| with GLSL source |source| and returns
    100 // the shader handle or 0 on error.
    101 GLuint CreateShader(GLenum type, const GLchar* source) {
    102   GLuint shader = glCreateShader(type);
    103   if (!shader) {
    104     return 0;
    105   }
    106   glShaderSource(shader, 1, &source, NULL);
    107   glCompileShader(shader);
    108   GLint compileStatus = GL_FALSE;
    109   glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus);
    110   if (compileStatus == GL_FALSE) {
    111     glDeleteShader(shader);
    112     shader = 0;
    113   }
    114   return shader;
    115 }
    116 
    117 // Links a shader program with the given vertex and fragment shaders and
    118 // returns the program handle or 0 on error.
    119 GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) {
    120   if (vertexShader == 0 || fragmentShader == 0) {
    121     return 0;
    122   }
    123   GLuint program = glCreateProgram();
    124   if (!program) {
    125     return 0;
    126   }
    127   glAttachShader(program, vertexShader);
    128   glAttachShader(program, fragmentShader);
    129   glLinkProgram(program);
    130   GLint linkStatus = GL_FALSE;
    131   glGetProgramiv(program, GL_LINK_STATUS, &linkStatus);
    132   if (linkStatus == GL_FALSE) {
    133     glDeleteProgram(program);
    134     program = 0;
    135   }
    136   return program;
    137 }
    138 
    139 // When modelview and projection matrices are identity (default) the world is
    140 // contained in the square around origin with unit size 2. Drawing to these
    141 // coordinates is equivalent to drawing to the entire screen. The texture is
    142 // stretched over that square using texture coordinates (u, v) that range
    143 // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically
    144 // here because the incoming frame has origin in upper left hand corner but
    145 // OpenGL expects origin in bottom left corner.
    146 const GLfloat gVertices[] = {
    147   // X, Y, U, V.
    148   -1, -1, 0, 1,  // Bottom left.
    149    1, -1, 1, 1,  // Bottom right.
    150    1,  1, 1, 0,  // Top right.
    151   -1,  1, 0, 0,  // Top left.
    152 };
    153 
    154 // |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets
    155 // of 3 textures are used here, one for each of the Y, U and V planes. Having
    156 // two sets alleviates CPU blockage in the event that the GPU is asked to render
    157 // to a texture that is already in use.
    158 static const GLsizei kNumTextureSets = 2;
    159 static const GLsizei kNumTextures = 3 * kNumTextureSets;
    160 
    161 @implementation RTCOpenGLVideoRenderer {
    162 #if TARGET_OS_IPHONE
    163   EAGLContext* _context;
    164 #else
    165   NSOpenGLContext* _context;
    166 #endif
    167   BOOL _isInitialized;
    168   NSUInteger _currentTextureSet;
    169   // Handles for OpenGL constructs.
    170   GLuint _textures[kNumTextures];
    171   GLuint _program;
    172 #if !TARGET_OS_IPHONE
    173   GLuint _vertexArray;
    174 #endif
    175   GLuint _vertexBuffer;
    176   GLint _position;
    177   GLint _texcoord;
    178   GLint _ySampler;
    179   GLint _uSampler;
    180   GLint _vSampler;
    181 }
    182 
    183 + (void)initialize {
    184   // Disable dithering for performance.
    185   glDisable(GL_DITHER);
    186 }
    187 
    188 #if TARGET_OS_IPHONE
    189 - (instancetype)initWithContext:(EAGLContext*)context {
    190 #else
    191 - (instancetype)initWithContext:(NSOpenGLContext*)context {
    192 #endif
    193   NSAssert(context != nil, @"context cannot be nil");
    194   if (self = [super init]) {
    195     _context = context;
    196   }
    197   return self;
    198 }
    199 
    200 - (BOOL)drawFrame:(RTCI420Frame*)frame {
    201   if (!_isInitialized) {
    202     return NO;
    203   }
    204   if (_lastDrawnFrame == frame) {
    205     return NO;
    206   }
    207   [self ensureGLContext];
    208   glClear(GL_COLOR_BUFFER_BIT);
    209   if (frame) {
    210     if (![self updateTextureSizesForFrame:frame] ||
    211         ![self updateTextureDataForFrame:frame]) {
    212       return NO;
    213     }
    214 #if !TARGET_OS_IPHONE
    215     glBindVertexArray(_vertexArray);
    216 #endif
    217     glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
    218     glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
    219   }
    220 #if !TARGET_OS_IPHONE
    221   [_context flushBuffer];
    222 #endif
    223   _lastDrawnFrame = frame;
    224   return YES;
    225 }
    226 
    227 - (void)setupGL {
    228   if (_isInitialized) {
    229     return;
    230   }
    231   [self ensureGLContext];
    232   if (![self setupProgram]) {
    233     return;
    234   }
    235   if (![self setupTextures]) {
    236     return;
    237   }
    238   if (![self setupVertices]) {
    239     return;
    240   }
    241   glUseProgram(_program);
    242   glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
    243   _isInitialized = YES;
    244 }
    245 
    246 - (void)teardownGL {
    247   if (!_isInitialized) {
    248     return;
    249   }
    250   [self ensureGLContext];
    251   glDeleteProgram(_program);
    252   _program = 0;
    253   glDeleteTextures(kNumTextures, _textures);
    254   glDeleteBuffers(1, &_vertexBuffer);
    255   _vertexBuffer = 0;
    256 #if !TARGET_OS_IPHONE
    257   glDeleteVertexArrays(1, &_vertexArray);
    258 #endif
    259   _isInitialized = NO;
    260 }
    261 
    262 #pragma mark - Private
    263 
    264 - (void)ensureGLContext {
    265   NSAssert(_context, @"context shouldn't be nil");
    266 #if TARGET_OS_IPHONE
    267   if ([EAGLContext currentContext] != _context) {
    268     [EAGLContext setCurrentContext:_context];
    269   }
    270 #else
    271   if ([NSOpenGLContext currentContext] != _context) {
    272     [_context makeCurrentContext];
    273   }
    274 #endif
    275 }
    276 
    277 - (BOOL)setupProgram {
    278   NSAssert(!_program, @"program already set up");
    279   GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource);
    280   NSAssert(vertexShader, @"failed to create vertex shader");
    281   GLuint fragmentShader =
    282       CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource);
    283   NSAssert(fragmentShader, @"failed to create fragment shader");
    284   _program = CreateProgram(vertexShader, fragmentShader);
    285   // Shaders are created only to generate program.
    286   if (vertexShader) {
    287     glDeleteShader(vertexShader);
    288   }
    289   if (fragmentShader) {
    290     glDeleteShader(fragmentShader);
    291   }
    292   if (!_program) {
    293     return NO;
    294   }
    295   _position = glGetAttribLocation(_program, "position");
    296   _texcoord = glGetAttribLocation(_program, "texcoord");
    297   _ySampler = glGetUniformLocation(_program, "s_textureY");
    298   _uSampler = glGetUniformLocation(_program, "s_textureU");
    299   _vSampler = glGetUniformLocation(_program, "s_textureV");
    300   if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 ||
    301       _vSampler < 0) {
    302     return NO;
    303   }
    304   return YES;
    305 }
    306 
    307 - (BOOL)setupTextures {
    308   glGenTextures(kNumTextures, _textures);
    309   // Set parameters for each of the textures we created.
    310   for (GLsizei i = 0; i < kNumTextures; i++) {
    311     glActiveTexture(GL_TEXTURE0 + i);
    312     glBindTexture(GL_TEXTURE_2D, _textures[i]);
    313     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    314     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    315     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    316     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    317   }
    318   return YES;
    319 }
    320 
    321 - (BOOL)updateTextureSizesForFrame:(RTCI420Frame*)frame {
    322   if (frame.height == _lastDrawnFrame.height &&
    323       frame.width == _lastDrawnFrame.width &&
    324       frame.chromaWidth == _lastDrawnFrame.chromaWidth &&
    325       frame.chromaHeight == _lastDrawnFrame.chromaHeight) {
    326     return YES;
    327   }
    328   GLsizei lumaWidth = frame.width;
    329   GLsizei lumaHeight = frame.height;
    330   GLsizei chromaWidth = frame.chromaWidth;
    331   GLsizei chromaHeight = frame.chromaHeight;
    332   for (GLint i = 0; i < kNumTextureSets; i++) {
    333     glActiveTexture(GL_TEXTURE0 + i * 3);
    334     glTexImage2D(GL_TEXTURE_2D,
    335                  0,
    336                  RTC_PIXEL_FORMAT,
    337                  lumaWidth,
    338                  lumaHeight,
    339                  0,
    340                  RTC_PIXEL_FORMAT,
    341                  GL_UNSIGNED_BYTE,
    342                  0);
    343     glActiveTexture(GL_TEXTURE0 + i * 3 + 1);
    344     glTexImage2D(GL_TEXTURE_2D,
    345                  0,
    346                  RTC_PIXEL_FORMAT,
    347                  chromaWidth,
    348                  chromaHeight,
    349                  0,
    350                  RTC_PIXEL_FORMAT,
    351                  GL_UNSIGNED_BYTE,
    352                  0);
    353     glActiveTexture(GL_TEXTURE0 + i * 3 + 2);
    354     glTexImage2D(GL_TEXTURE_2D,
    355                  0,
    356                  RTC_PIXEL_FORMAT,
    357                  chromaWidth,
    358                  chromaHeight,
    359                  0,
    360                  RTC_PIXEL_FORMAT,
    361                  GL_UNSIGNED_BYTE,
    362                  0);
    363   }
    364   return YES;
    365 }
    366 
    367 - (BOOL)updateTextureDataForFrame:(RTCI420Frame*)frame {
    368   NSUInteger textureOffset = _currentTextureSet * 3;
    369   NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset");
    370   NSParameterAssert(frame.yPitch == frame.width);
    371   NSParameterAssert(frame.uPitch == frame.chromaWidth);
    372   NSParameterAssert(frame.vPitch == frame.chromaWidth);
    373 
    374   glActiveTexture(GL_TEXTURE0 + textureOffset);
    375   // When setting texture sampler uniforms, the texture index is used not
    376   // the texture handle.
    377   glUniform1i(_ySampler, textureOffset);
    378   glTexImage2D(GL_TEXTURE_2D,
    379                0,
    380                RTC_PIXEL_FORMAT,
    381                frame.width,
    382                frame.height,
    383                0,
    384                RTC_PIXEL_FORMAT,
    385                GL_UNSIGNED_BYTE,
    386                frame.yPlane);
    387 
    388   glActiveTexture(GL_TEXTURE0 + textureOffset + 1);
    389   glUniform1i(_uSampler, textureOffset + 1);
    390   glTexImage2D(GL_TEXTURE_2D,
    391                0,
    392                RTC_PIXEL_FORMAT,
    393                frame.chromaWidth,
    394                frame.chromaHeight,
    395                0,
    396                RTC_PIXEL_FORMAT,
    397                GL_UNSIGNED_BYTE,
    398                frame.uPlane);
    399 
    400   glActiveTexture(GL_TEXTURE0 + textureOffset + 2);
    401   glUniform1i(_vSampler, textureOffset + 2);
    402   glTexImage2D(GL_TEXTURE_2D,
    403                0,
    404                RTC_PIXEL_FORMAT,
    405                frame.chromaWidth,
    406                frame.chromaHeight,
    407                0,
    408                RTC_PIXEL_FORMAT,
    409                GL_UNSIGNED_BYTE,
    410                frame.vPlane);
    411 
    412   _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets;
    413   return YES;
    414 }
    415 
    416 - (BOOL)setupVertices {
    417 #if !TARGET_OS_IPHONE
    418   NSAssert(!_vertexArray, @"vertex array already set up");
    419   glGenVertexArrays(1, &_vertexArray);
    420   if (!_vertexArray) {
    421     return NO;
    422   }
    423   glBindVertexArray(_vertexArray);
    424 #endif
    425   NSAssert(!_vertexBuffer, @"vertex buffer already set up");
    426   glGenBuffers(1, &_vertexBuffer);
    427   if (!_vertexBuffer) {
    428 #if !TARGET_OS_IPHONE
    429     glDeleteVertexArrays(1, &_vertexArray);
    430     _vertexArray = 0;
    431 #endif
    432     return NO;
    433   }
    434   glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
    435   glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW);
    436 
    437   // Read position attribute from |gVertices| with size of 2 and stride of 4
    438   // beginning at the start of the array. The last argument indicates offset
    439   // of data within |gVertices| as supplied to the vertex buffer.
    440   glVertexAttribPointer(
    441       _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void*)0);
    442   glEnableVertexAttribArray(_position);
    443 
    444   // Read texcoord attribute from |gVertices| with size of 2 and stride of 4
    445   // beginning at the first texcoord in the array. The last argument indicates
    446   // offset of data within |gVertices| as supplied to the vertex buffer.
    447   glVertexAttribPointer(_texcoord,
    448                         2,
    449                         GL_FLOAT,
    450                         GL_FALSE,
    451                         4 * sizeof(GLfloat),
    452                         (void*)(2 * sizeof(GLfloat)));
    453   glEnableVertexAttribArray(_texcoord);
    454 
    455   return YES;
    456 }
    457 
    458 @end
    459