1 /* 2 * libjingle 3 * Copyright 2014, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #if !defined(__has_feature) || !__has_feature(objc_arc) 29 #error "This file requires ARC support." 30 #endif 31 32 #import "RTCOpenGLVideoRenderer.h" 33 34 #if TARGET_OS_IPHONE 35 #import <OpenGLES/ES2/gl.h> 36 #else 37 #import <OpenGL/gl3.h> 38 #endif 39 40 #import "RTCI420Frame.h" 41 42 // TODO(tkchin): check and log openGL errors. Methods here return BOOLs in 43 // anticipation of that happening in the future. 44 45 #if TARGET_OS_IPHONE 46 #define RTC_PIXEL_FORMAT GL_LUMINANCE 47 #define SHADER_VERSION 48 #define VERTEX_SHADER_IN "attribute" 49 #define VERTEX_SHADER_OUT "varying" 50 #define FRAGMENT_SHADER_IN "varying" 51 #define FRAGMENT_SHADER_OUT 52 #define FRAGMENT_SHADER_COLOR "gl_FragColor" 53 #define FRAGMENT_SHADER_TEXTURE "texture2D" 54 #else 55 #define RTC_PIXEL_FORMAT GL_RED 56 #define SHADER_VERSION "#version 150\n" 57 #define VERTEX_SHADER_IN "in" 58 #define VERTEX_SHADER_OUT "out" 59 #define FRAGMENT_SHADER_IN "in" 60 #define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" 61 #define FRAGMENT_SHADER_COLOR "fragColor" 62 #define FRAGMENT_SHADER_TEXTURE "texture" 63 #endif 64 65 // Vertex shader doesn't do anything except pass coordinates through. 66 static const char kVertexShaderSource[] = 67 SHADER_VERSION 68 VERTEX_SHADER_IN " vec2 position;\n" 69 VERTEX_SHADER_IN " vec2 texcoord;\n" 70 VERTEX_SHADER_OUT " vec2 v_texcoord;\n" 71 "void main() {\n" 72 " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" 73 " v_texcoord = texcoord;\n" 74 "}\n"; 75 76 // Fragment shader converts YUV values from input textures into a final RGB 77 // pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. 78 static const char kFragmentShaderSource[] = 79 SHADER_VERSION 80 "precision highp float;" 81 FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" 82 "uniform lowp sampler2D s_textureY;\n" 83 "uniform lowp sampler2D s_textureU;\n" 84 "uniform lowp sampler2D s_textureV;\n" 85 FRAGMENT_SHADER_OUT 86 "void main() {\n" 87 " float y, u, v, r, g, b;\n" 88 " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" 89 " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" 90 " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" 91 " u = u - 0.5;\n" 92 " v = v - 0.5;\n" 93 " r = y + 1.403 * v;\n" 94 " g = y - 0.344 * u - 0.714 * v;\n" 95 " b = y + 1.770 * u;\n" 96 " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" 97 " }\n"; 98 99 // Compiles a shader of the given |type| with GLSL source |source| and returns 100 // the shader handle or 0 on error. 101 GLuint CreateShader(GLenum type, const GLchar* source) { 102 GLuint shader = glCreateShader(type); 103 if (!shader) { 104 return 0; 105 } 106 glShaderSource(shader, 1, &source, NULL); 107 glCompileShader(shader); 108 GLint compileStatus = GL_FALSE; 109 glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); 110 if (compileStatus == GL_FALSE) { 111 glDeleteShader(shader); 112 shader = 0; 113 } 114 return shader; 115 } 116 117 // Links a shader program with the given vertex and fragment shaders and 118 // returns the program handle or 0 on error. 119 GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { 120 if (vertexShader == 0 || fragmentShader == 0) { 121 return 0; 122 } 123 GLuint program = glCreateProgram(); 124 if (!program) { 125 return 0; 126 } 127 glAttachShader(program, vertexShader); 128 glAttachShader(program, fragmentShader); 129 glLinkProgram(program); 130 GLint linkStatus = GL_FALSE; 131 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); 132 if (linkStatus == GL_FALSE) { 133 glDeleteProgram(program); 134 program = 0; 135 } 136 return program; 137 } 138 139 // When modelview and projection matrices are identity (default) the world is 140 // contained in the square around origin with unit size 2. Drawing to these 141 // coordinates is equivalent to drawing to the entire screen. The texture is 142 // stretched over that square using texture coordinates (u, v) that range 143 // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically 144 // here because the incoming frame has origin in upper left hand corner but 145 // OpenGL expects origin in bottom left corner. 146 const GLfloat gVertices[] = { 147 // X, Y, U, V. 148 -1, -1, 0, 1, // Bottom left. 149 1, -1, 1, 1, // Bottom right. 150 1, 1, 1, 0, // Top right. 151 -1, 1, 0, 0, // Top left. 152 }; 153 154 // |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets 155 // of 3 textures are used here, one for each of the Y, U and V planes. Having 156 // two sets alleviates CPU blockage in the event that the GPU is asked to render 157 // to a texture that is already in use. 158 static const GLsizei kNumTextureSets = 2; 159 static const GLsizei kNumTextures = 3 * kNumTextureSets; 160 161 @implementation RTCOpenGLVideoRenderer { 162 #if TARGET_OS_IPHONE 163 EAGLContext* _context; 164 #else 165 NSOpenGLContext* _context; 166 #endif 167 BOOL _isInitialized; 168 NSUInteger _currentTextureSet; 169 // Handles for OpenGL constructs. 170 GLuint _textures[kNumTextures]; 171 GLuint _program; 172 #if !TARGET_OS_IPHONE 173 GLuint _vertexArray; 174 #endif 175 GLuint _vertexBuffer; 176 GLint _position; 177 GLint _texcoord; 178 GLint _ySampler; 179 GLint _uSampler; 180 GLint _vSampler; 181 } 182 183 + (void)initialize { 184 // Disable dithering for performance. 185 glDisable(GL_DITHER); 186 } 187 188 #if TARGET_OS_IPHONE 189 - (instancetype)initWithContext:(EAGLContext*)context { 190 #else 191 - (instancetype)initWithContext:(NSOpenGLContext*)context { 192 #endif 193 NSAssert(context != nil, @"context cannot be nil"); 194 if (self = [super init]) { 195 _context = context; 196 } 197 return self; 198 } 199 200 - (BOOL)drawFrame:(RTCI420Frame*)frame { 201 if (!_isInitialized) { 202 return NO; 203 } 204 if (_lastDrawnFrame == frame) { 205 return NO; 206 } 207 [self ensureGLContext]; 208 glClear(GL_COLOR_BUFFER_BIT); 209 if (frame) { 210 if (![self updateTextureSizesForFrame:frame] || 211 ![self updateTextureDataForFrame:frame]) { 212 return NO; 213 } 214 #if !TARGET_OS_IPHONE 215 glBindVertexArray(_vertexArray); 216 #endif 217 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 218 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); 219 } 220 #if !TARGET_OS_IPHONE 221 [_context flushBuffer]; 222 #endif 223 _lastDrawnFrame = frame; 224 return YES; 225 } 226 227 - (void)setupGL { 228 if (_isInitialized) { 229 return; 230 } 231 [self ensureGLContext]; 232 if (![self setupProgram]) { 233 return; 234 } 235 if (![self setupTextures]) { 236 return; 237 } 238 if (![self setupVertices]) { 239 return; 240 } 241 glUseProgram(_program); 242 glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 243 _isInitialized = YES; 244 } 245 246 - (void)teardownGL { 247 if (!_isInitialized) { 248 return; 249 } 250 [self ensureGLContext]; 251 glDeleteProgram(_program); 252 _program = 0; 253 glDeleteTextures(kNumTextures, _textures); 254 glDeleteBuffers(1, &_vertexBuffer); 255 _vertexBuffer = 0; 256 #if !TARGET_OS_IPHONE 257 glDeleteVertexArrays(1, &_vertexArray); 258 #endif 259 _isInitialized = NO; 260 } 261 262 #pragma mark - Private 263 264 - (void)ensureGLContext { 265 NSAssert(_context, @"context shouldn't be nil"); 266 #if TARGET_OS_IPHONE 267 if ([EAGLContext currentContext] != _context) { 268 [EAGLContext setCurrentContext:_context]; 269 } 270 #else 271 if ([NSOpenGLContext currentContext] != _context) { 272 [_context makeCurrentContext]; 273 } 274 #endif 275 } 276 277 - (BOOL)setupProgram { 278 NSAssert(!_program, @"program already set up"); 279 GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); 280 NSAssert(vertexShader, @"failed to create vertex shader"); 281 GLuint fragmentShader = 282 CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); 283 NSAssert(fragmentShader, @"failed to create fragment shader"); 284 _program = CreateProgram(vertexShader, fragmentShader); 285 // Shaders are created only to generate program. 286 if (vertexShader) { 287 glDeleteShader(vertexShader); 288 } 289 if (fragmentShader) { 290 glDeleteShader(fragmentShader); 291 } 292 if (!_program) { 293 return NO; 294 } 295 _position = glGetAttribLocation(_program, "position"); 296 _texcoord = glGetAttribLocation(_program, "texcoord"); 297 _ySampler = glGetUniformLocation(_program, "s_textureY"); 298 _uSampler = glGetUniformLocation(_program, "s_textureU"); 299 _vSampler = glGetUniformLocation(_program, "s_textureV"); 300 if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || 301 _vSampler < 0) { 302 return NO; 303 } 304 return YES; 305 } 306 307 - (BOOL)setupTextures { 308 glGenTextures(kNumTextures, _textures); 309 // Set parameters for each of the textures we created. 310 for (GLsizei i = 0; i < kNumTextures; i++) { 311 glActiveTexture(GL_TEXTURE0 + i); 312 glBindTexture(GL_TEXTURE_2D, _textures[i]); 313 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 314 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 315 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 316 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 317 } 318 return YES; 319 } 320 321 - (BOOL)updateTextureSizesForFrame:(RTCI420Frame*)frame { 322 if (frame.height == _lastDrawnFrame.height && 323 frame.width == _lastDrawnFrame.width && 324 frame.chromaWidth == _lastDrawnFrame.chromaWidth && 325 frame.chromaHeight == _lastDrawnFrame.chromaHeight) { 326 return YES; 327 } 328 GLsizei lumaWidth = frame.width; 329 GLsizei lumaHeight = frame.height; 330 GLsizei chromaWidth = frame.chromaWidth; 331 GLsizei chromaHeight = frame.chromaHeight; 332 for (GLint i = 0; i < kNumTextureSets; i++) { 333 glActiveTexture(GL_TEXTURE0 + i * 3); 334 glTexImage2D(GL_TEXTURE_2D, 335 0, 336 RTC_PIXEL_FORMAT, 337 lumaWidth, 338 lumaHeight, 339 0, 340 RTC_PIXEL_FORMAT, 341 GL_UNSIGNED_BYTE, 342 0); 343 glActiveTexture(GL_TEXTURE0 + i * 3 + 1); 344 glTexImage2D(GL_TEXTURE_2D, 345 0, 346 RTC_PIXEL_FORMAT, 347 chromaWidth, 348 chromaHeight, 349 0, 350 RTC_PIXEL_FORMAT, 351 GL_UNSIGNED_BYTE, 352 0); 353 glActiveTexture(GL_TEXTURE0 + i * 3 + 2); 354 glTexImage2D(GL_TEXTURE_2D, 355 0, 356 RTC_PIXEL_FORMAT, 357 chromaWidth, 358 chromaHeight, 359 0, 360 RTC_PIXEL_FORMAT, 361 GL_UNSIGNED_BYTE, 362 0); 363 } 364 return YES; 365 } 366 367 - (BOOL)updateTextureDataForFrame:(RTCI420Frame*)frame { 368 NSUInteger textureOffset = _currentTextureSet * 3; 369 NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); 370 NSParameterAssert(frame.yPitch == frame.width); 371 NSParameterAssert(frame.uPitch == frame.chromaWidth); 372 NSParameterAssert(frame.vPitch == frame.chromaWidth); 373 374 glActiveTexture(GL_TEXTURE0 + textureOffset); 375 // When setting texture sampler uniforms, the texture index is used not 376 // the texture handle. 377 glUniform1i(_ySampler, textureOffset); 378 glTexImage2D(GL_TEXTURE_2D, 379 0, 380 RTC_PIXEL_FORMAT, 381 frame.width, 382 frame.height, 383 0, 384 RTC_PIXEL_FORMAT, 385 GL_UNSIGNED_BYTE, 386 frame.yPlane); 387 388 glActiveTexture(GL_TEXTURE0 + textureOffset + 1); 389 glUniform1i(_uSampler, textureOffset + 1); 390 glTexImage2D(GL_TEXTURE_2D, 391 0, 392 RTC_PIXEL_FORMAT, 393 frame.chromaWidth, 394 frame.chromaHeight, 395 0, 396 RTC_PIXEL_FORMAT, 397 GL_UNSIGNED_BYTE, 398 frame.uPlane); 399 400 glActiveTexture(GL_TEXTURE0 + textureOffset + 2); 401 glUniform1i(_vSampler, textureOffset + 2); 402 glTexImage2D(GL_TEXTURE_2D, 403 0, 404 RTC_PIXEL_FORMAT, 405 frame.chromaWidth, 406 frame.chromaHeight, 407 0, 408 RTC_PIXEL_FORMAT, 409 GL_UNSIGNED_BYTE, 410 frame.vPlane); 411 412 _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; 413 return YES; 414 } 415 416 - (BOOL)setupVertices { 417 #if !TARGET_OS_IPHONE 418 NSAssert(!_vertexArray, @"vertex array already set up"); 419 glGenVertexArrays(1, &_vertexArray); 420 if (!_vertexArray) { 421 return NO; 422 } 423 glBindVertexArray(_vertexArray); 424 #endif 425 NSAssert(!_vertexBuffer, @"vertex buffer already set up"); 426 glGenBuffers(1, &_vertexBuffer); 427 if (!_vertexBuffer) { 428 #if !TARGET_OS_IPHONE 429 glDeleteVertexArrays(1, &_vertexArray); 430 _vertexArray = 0; 431 #endif 432 return NO; 433 } 434 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); 435 glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); 436 437 // Read position attribute from |gVertices| with size of 2 and stride of 4 438 // beginning at the start of the array. The last argument indicates offset 439 // of data within |gVertices| as supplied to the vertex buffer. 440 glVertexAttribPointer( 441 _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void*)0); 442 glEnableVertexAttribArray(_position); 443 444 // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 445 // beginning at the first texcoord in the array. The last argument indicates 446 // offset of data within |gVertices| as supplied to the vertex buffer. 447 glVertexAttribPointer(_texcoord, 448 2, 449 GL_FLOAT, 450 GL_FALSE, 451 4 * sizeof(GLfloat), 452 (void*)(2 * sizeof(GLfloat))); 453 glEnableVertexAttribArray(_texcoord); 454 455 return YES; 456 } 457 458 @end 459