1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define LOG_TAG "NativeWindowRenderer" 18 #include "NativeWindowRenderer.h" 19 20 #include <GLES2/gl2.h> 21 #include <GLES2/gl2ext.h> 22 #include <cutils/log.h> 23 #include <gui/GLConsumer.h> 24 #include <gui/Surface.h> 25 #include <media/stagefright/MediaBuffer.h> 26 #include <media/stagefright/MetaData.h> 27 #include <media/stagefright/foundation/ADebug.h> 28 #include "VideoEditorTools.h" 29 30 #define CHECK_EGL_ERROR CHECK(EGL_SUCCESS == eglGetError()) 31 #define CHECK_GL_ERROR CHECK(GLenum(GL_NO_ERROR) == glGetError()) 32 33 // 34 // Vertex and fragment programs 35 // 36 37 // The matrix is derived from 38 // frameworks/base/media/libstagefright/colorconversion/ColorConverter.cpp 39 // 40 // R * 255 = 1.164 * (Y - 16) + 1.596 * (V - 128) 41 // G * 255 = 1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128) 42 // B * 255 = 1.164 * (Y - 16) + 2.018 * (U - 128) 43 // 44 // Here we assume YUV are in the range of [0,255], RGB are in the range of 45 // [0, 1] 46 #define RGB2YUV_MATRIX \ 47 "const mat4 rgb2yuv = mat4("\ 48 " 65.52255, -37.79398, 111.98732, 0.00000,"\ 49 " 128.62729, -74.19334, -93.81088, 0.00000,"\ 50 " 24.92233, 111.98732, -18.17644, 0.00000,"\ 51 " 16.00000, 128.00000, 128.00000, 1.00000);\n" 52 53 #define YUV2RGB_MATRIX \ 54 "const mat4 yuv2rgb = mat4("\ 55 " 0.00456, 0.00456, 0.00456, 0.00000,"\ 56 " 0.00000, -0.00153, 0.00791, 0.00000,"\ 57 " 0.00626, -0.00319, 0.00000, 0.00000,"\ 58 " -0.87416, 0.53133, -1.08599, 1.00000);\n" 59 60 static const char vSrcNormal[] = 61 "attribute vec4 vPosition;\n" 62 "attribute vec2 vTexPos;\n" 63 "uniform mat4 texMatrix;\n" 64 "varying vec2 texCoords;\n" 65 "varying float topDown;\n" 66 "void main() {\n" 67 " gl_Position = vPosition;\n" 68 " texCoords = (texMatrix * vec4(vTexPos, 0.0, 1.0)).xy;\n" 69 " topDown = vTexPos.y;\n" 70 "}\n"; 71 72 static const char fSrcNormal[] = 73 "#extension GL_OES_EGL_image_external : require\n" 74 "precision mediump float;\n" 75 "uniform samplerExternalOES texSampler;\n" 76 "varying vec2 texCoords;\n" 77 "void main() {\n" 78 " gl_FragColor = texture2D(texSampler, texCoords);\n" 79 "}\n"; 80 81 static const char fSrcSepia[] = 82 "#extension GL_OES_EGL_image_external : require\n" 83 "precision mediump float;\n" 84 "uniform samplerExternalOES texSampler;\n" 85 "varying vec2 texCoords;\n" 86 RGB2YUV_MATRIX 87 YUV2RGB_MATRIX 88 "void main() {\n" 89 " vec4 rgb = texture2D(texSampler, texCoords);\n" 90 " vec4 yuv = rgb2yuv * rgb;\n" 91 " yuv = vec4(yuv.x, 117.0, 139.0, 1.0);\n" 92 " gl_FragColor = yuv2rgb * yuv;\n" 93 "}\n"; 94 95 static const char fSrcNegative[] = 96 "#extension GL_OES_EGL_image_external : require\n" 97 "precision mediump float;\n" 98 "uniform samplerExternalOES texSampler;\n" 99 "varying vec2 texCoords;\n" 100 RGB2YUV_MATRIX 101 YUV2RGB_MATRIX 102 "void main() {\n" 103 " vec4 rgb = texture2D(texSampler, texCoords);\n" 104 " vec4 yuv = rgb2yuv * rgb;\n" 105 " yuv = vec4(255.0 - yuv.x, yuv.y, yuv.z, 1.0);\n" 106 " gl_FragColor = yuv2rgb * yuv;\n" 107 "}\n"; 108 109 static const char fSrcGradient[] = 110 "#extension GL_OES_EGL_image_external : require\n" 111 "precision mediump float;\n" 112 "uniform samplerExternalOES texSampler;\n" 113 "varying vec2 texCoords;\n" 114 "varying float topDown;\n" 115 RGB2YUV_MATRIX 116 YUV2RGB_MATRIX 117 "void main() {\n" 118 " vec4 rgb = texture2D(texSampler, texCoords);\n" 119 " vec4 yuv = rgb2yuv * rgb;\n" 120 " vec4 mixin = vec4(15.0/31.0, 59.0/63.0, 31.0/31.0, 1.0);\n" 121 " vec4 yuv2 = rgb2yuv * vec4((mixin.xyz * topDown), 1);\n" 122 " yuv = vec4(yuv.x, yuv2.y, yuv2.z, 1);\n" 123 " gl_FragColor = yuv2rgb * yuv;\n" 124 "}\n"; 125 126 namespace android { 127 128 NativeWindowRenderer::NativeWindowRenderer(sp<ANativeWindow> nativeWindow, 129 int width, int height) 130 : mNativeWindow(nativeWindow) 131 , mDstWidth(width) 132 , mDstHeight(height) 133 , mLastVideoEffect(-1) 134 , mNextTextureId(100) 135 , mActiveInputs(0) 136 , mThreadCmd(CMD_IDLE) { 137 createThread(threadStart, this); 138 } 139 140 // The functions below run in the GL thread. 141 // 142 // All GL-related work is done in this thread, and other threads send 143 // requests to this thread using a command code. We expect most of the 144 // time there will only be one thread sending in requests, so we let 145 // other threads wait until the request is finished by GL thread. 146 147 int NativeWindowRenderer::threadStart(void* self) { 148 ALOGD("create thread"); 149 ((NativeWindowRenderer*)self)->glThread(); 150 return 0; 151 } 152 153 void NativeWindowRenderer::glThread() { 154 initializeEGL(); 155 createPrograms(); 156 157 Mutex::Autolock autoLock(mLock); 158 bool quit = false; 159 while (!quit) { 160 switch (mThreadCmd) { 161 case CMD_IDLE: 162 mCond.wait(mLock); 163 continue; 164 case CMD_RENDER_INPUT: 165 render(mThreadRenderInput); 166 break; 167 case CMD_RESERVE_TEXTURE: 168 glBindTexture(GL_TEXTURE_EXTERNAL_OES, mThreadTextureId); 169 CHECK_GL_ERROR; 170 break; 171 case CMD_DELETE_TEXTURE: 172 glDeleteTextures(1, &mThreadTextureId); 173 break; 174 case CMD_QUIT: 175 terminateEGL(); 176 quit = true; 177 break; 178 } 179 // Tell the requester that the command is finished. 180 mThreadCmd = CMD_IDLE; 181 mCond.broadcast(); 182 } 183 ALOGD("quit"); 184 } 185 186 void NativeWindowRenderer::initializeEGL() { 187 mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); 188 CHECK_EGL_ERROR; 189 190 EGLint majorVersion; 191 EGLint minorVersion; 192 eglInitialize(mEglDisplay, &majorVersion, &minorVersion); 193 CHECK_EGL_ERROR; 194 195 EGLConfig config; 196 EGLint numConfigs = -1; 197 EGLint configAttribs[] = { 198 EGL_SURFACE_TYPE, EGL_WINDOW_BIT, 199 EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, 200 EGL_RED_SIZE, 8, 201 EGL_GREEN_SIZE, 8, 202 EGL_BLUE_SIZE, 8, 203 EGL_NONE 204 }; 205 eglChooseConfig(mEglDisplay, configAttribs, &config, 1, &numConfigs); 206 CHECK_EGL_ERROR; 207 208 mEglSurface = eglCreateWindowSurface(mEglDisplay, config, 209 mNativeWindow.get(), NULL); 210 CHECK_EGL_ERROR; 211 212 EGLint contextAttribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE }; 213 mEglContext = eglCreateContext(mEglDisplay, config, EGL_NO_CONTEXT, 214 contextAttribs); 215 CHECK_EGL_ERROR; 216 217 eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, mEglContext); 218 CHECK_EGL_ERROR; 219 } 220 221 void NativeWindowRenderer::terminateEGL() { 222 eglDestroyContext(mEglDisplay, mEglContext); 223 eglDestroySurface(mEglDisplay, mEglSurface); 224 eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); 225 eglTerminate(mEglDisplay); 226 } 227 228 void NativeWindowRenderer::createPrograms() { 229 GLuint vShader; 230 loadShader(GL_VERTEX_SHADER, vSrcNormal, &vShader); 231 232 const char* fSrc[NUMBER_OF_EFFECTS] = { 233 fSrcNormal, fSrcSepia, fSrcNegative, fSrcGradient 234 }; 235 236 for (int i = 0; i < NUMBER_OF_EFFECTS; i++) { 237 GLuint fShader; 238 loadShader(GL_FRAGMENT_SHADER, fSrc[i], &fShader); 239 createProgram(vShader, fShader, &mProgram[i]); 240 glDeleteShader(fShader); 241 CHECK_GL_ERROR; 242 } 243 244 glDeleteShader(vShader); 245 CHECK_GL_ERROR; 246 } 247 248 void NativeWindowRenderer::createProgram( 249 GLuint vertexShader, GLuint fragmentShader, GLuint* outPgm) { 250 251 GLuint program = glCreateProgram(); 252 CHECK_GL_ERROR; 253 254 glAttachShader(program, vertexShader); 255 CHECK_GL_ERROR; 256 257 glAttachShader(program, fragmentShader); 258 CHECK_GL_ERROR; 259 260 glLinkProgram(program); 261 CHECK_GL_ERROR; 262 263 GLint linkStatus = GL_FALSE; 264 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); 265 if (linkStatus != GL_TRUE) { 266 GLint infoLen = 0; 267 glGetProgramiv(program, GL_INFO_LOG_LENGTH, &infoLen); 268 if (infoLen) { 269 char* buf = (char*) malloc(infoLen); 270 if (buf) { 271 glGetProgramInfoLog(program, infoLen, NULL, buf); 272 ALOGE("Program link log:\n%s\n", buf); 273 free(buf); 274 } 275 } 276 glDeleteProgram(program); 277 program = 0; 278 } 279 280 *outPgm = program; 281 } 282 283 void NativeWindowRenderer::loadShader(GLenum shaderType, const char* pSource, 284 GLuint* outShader) { 285 GLuint shader = glCreateShader(shaderType); 286 CHECK_GL_ERROR; 287 288 glShaderSource(shader, 1, &pSource, NULL); 289 CHECK_GL_ERROR; 290 291 glCompileShader(shader); 292 CHECK_GL_ERROR; 293 294 GLint compiled = 0; 295 glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); 296 if (!compiled) { 297 GLint infoLen = 0; 298 glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); 299 char* buf = (char*) malloc(infoLen); 300 if (buf) { 301 glGetShaderInfoLog(shader, infoLen, NULL, buf); 302 ALOGE("Shader compile log:\n%s\n", buf); 303 free(buf); 304 } 305 glDeleteShader(shader); 306 shader = 0; 307 } 308 *outShader = shader; 309 } 310 311 NativeWindowRenderer::~NativeWindowRenderer() { 312 CHECK(mActiveInputs == 0); 313 startRequest(CMD_QUIT); 314 sendRequest(); 315 } 316 317 void NativeWindowRenderer::render(RenderInput* input) { 318 sp<GLConsumer> ST = input->mST; 319 sp<Surface> STC = input->mSTC; 320 321 if (input->mIsExternalBuffer) { 322 queueExternalBuffer(STC.get(), input->mBuffer, 323 input->mWidth, input->mHeight); 324 } else { 325 queueInternalBuffer(STC.get(), input->mBuffer); 326 } 327 328 ST->updateTexImage(); 329 glClearColor(0, 0, 0, 0); 330 glClear(GL_COLOR_BUFFER_BIT); 331 332 calculatePositionCoordinates(input->mRenderingMode, 333 input->mWidth, input->mHeight); 334 335 const GLfloat textureCoordinates[] = { 336 0.0f, 1.0f, 337 0.0f, 0.0f, 338 1.0f, 0.0f, 339 1.0f, 1.0f, 340 }; 341 342 updateProgramAndHandle(input->mVideoEffect); 343 344 glVertexAttribPointer(mPositionHandle, 2, GL_FLOAT, GL_FALSE, 0, 345 mPositionCoordinates); 346 CHECK_GL_ERROR; 347 348 glEnableVertexAttribArray(mPositionHandle); 349 CHECK_GL_ERROR; 350 351 glVertexAttribPointer(mTexPosHandle, 2, GL_FLOAT, GL_FALSE, 0, 352 textureCoordinates); 353 CHECK_GL_ERROR; 354 355 glEnableVertexAttribArray(mTexPosHandle); 356 CHECK_GL_ERROR; 357 358 GLfloat texMatrix[16]; 359 ST->getTransformMatrix(texMatrix); 360 glUniformMatrix4fv(mTexMatrixHandle, 1, GL_FALSE, texMatrix); 361 CHECK_GL_ERROR; 362 363 glBindTexture(GL_TEXTURE_EXTERNAL_OES, input->mTextureId); 364 CHECK_GL_ERROR; 365 366 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 367 glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 368 glTexParameteri( 369 GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); 370 glTexParameteri( 371 GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 372 CHECK_GL_ERROR; 373 374 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); 375 CHECK_GL_ERROR; 376 377 eglSwapBuffers(mEglDisplay, mEglSurface); 378 } 379 380 void NativeWindowRenderer::queueInternalBuffer(ANativeWindow *anw, 381 MediaBuffer* buffer) { 382 int64_t timeUs; 383 CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); 384 native_window_set_buffers_timestamp(anw, timeUs * 1000); 385 status_t err = anw->queueBuffer(anw, buffer->graphicBuffer().get(), -1); 386 if (err != 0) { 387 ALOGE("queueBuffer failed with error %s (%d)", strerror(-err), -err); 388 return; 389 } 390 391 sp<MetaData> metaData = buffer->meta_data(); 392 metaData->setInt32(kKeyRendered, 1); 393 } 394 395 void NativeWindowRenderer::queueExternalBuffer(ANativeWindow* anw, 396 MediaBuffer* buffer, int width, int height) { 397 native_window_set_buffers_geometry(anw, width, height, 398 HAL_PIXEL_FORMAT_YV12); 399 native_window_set_usage(anw, GRALLOC_USAGE_SW_WRITE_OFTEN); 400 401 ANativeWindowBuffer* anb; 402 CHECK(NO_ERROR == native_window_dequeue_buffer_and_wait(anw, &anb)); 403 CHECK(anb != NULL); 404 405 // Copy the buffer 406 uint8_t* img = NULL; 407 sp<GraphicBuffer> buf(new GraphicBuffer(anb, false)); 408 buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img)); 409 copyI420Buffer(buffer, img, width, height, buf->getStride()); 410 buf->unlock(); 411 CHECK(NO_ERROR == anw->queueBuffer(anw, buf->getNativeBuffer(), -1)); 412 } 413 414 void NativeWindowRenderer::copyI420Buffer(MediaBuffer* src, uint8_t* dst, 415 int srcWidth, int srcHeight, int stride) { 416 int strideUV = (stride / 2 + 0xf) & ~0xf; 417 uint8_t* p = (uint8_t*)src->data() + src->range_offset(); 418 // Y 419 for (int i = srcHeight; i > 0; i--) { 420 memcpy(dst, p, srcWidth); 421 dst += stride; 422 p += srcWidth; 423 } 424 // The src is I420, the dst is YV12. 425 // U 426 p += srcWidth * srcHeight / 4; 427 for (int i = srcHeight / 2; i > 0; i--) { 428 memcpy(dst, p, srcWidth / 2); 429 dst += strideUV; 430 p += srcWidth / 2; 431 } 432 // V 433 p -= srcWidth * srcHeight / 2; 434 for (int i = srcHeight / 2; i > 0; i--) { 435 memcpy(dst, p, srcWidth / 2); 436 dst += strideUV; 437 p += srcWidth / 2; 438 } 439 } 440 441 void NativeWindowRenderer::updateProgramAndHandle(uint32_t videoEffect) { 442 if (mLastVideoEffect == videoEffect) { 443 return; 444 } 445 446 mLastVideoEffect = videoEffect; 447 int i; 448 switch (mLastVideoEffect) { 449 case VIDEO_EFFECT_NONE: 450 i = 0; 451 break; 452 case VIDEO_EFFECT_SEPIA: 453 i = 1; 454 break; 455 case VIDEO_EFFECT_NEGATIVE: 456 i = 2; 457 break; 458 case VIDEO_EFFECT_GRADIENT: 459 i = 3; 460 break; 461 default: 462 i = 0; 463 break; 464 } 465 glUseProgram(mProgram[i]); 466 CHECK_GL_ERROR; 467 468 mPositionHandle = glGetAttribLocation(mProgram[i], "vPosition"); 469 mTexPosHandle = glGetAttribLocation(mProgram[i], "vTexPos"); 470 mTexMatrixHandle = glGetUniformLocation(mProgram[i], "texMatrix"); 471 CHECK_GL_ERROR; 472 } 473 474 void NativeWindowRenderer::calculatePositionCoordinates( 475 M4xVSS_MediaRendering renderingMode, int srcWidth, int srcHeight) { 476 float x, y; 477 switch (renderingMode) { 478 case M4xVSS_kResizing: 479 default: 480 x = 1; 481 y = 1; 482 break; 483 case M4xVSS_kCropping: 484 x = float(srcWidth) / mDstWidth; 485 y = float(srcHeight) / mDstHeight; 486 // Make the smaller side 1 487 if (x > y) { 488 x /= y; 489 y = 1; 490 } else { 491 y /= x; 492 x = 1; 493 } 494 break; 495 case M4xVSS_kBlackBorders: 496 x = float(srcWidth) / mDstWidth; 497 y = float(srcHeight) / mDstHeight; 498 // Make the larger side 1 499 if (x > y) { 500 y /= x; 501 x = 1; 502 } else { 503 x /= y; 504 y = 1; 505 } 506 break; 507 } 508 509 mPositionCoordinates[0] = -x; 510 mPositionCoordinates[1] = y; 511 mPositionCoordinates[2] = -x; 512 mPositionCoordinates[3] = -y; 513 mPositionCoordinates[4] = x; 514 mPositionCoordinates[5] = -y; 515 mPositionCoordinates[6] = x; 516 mPositionCoordinates[7] = y; 517 } 518 519 // 520 // The functions below run in other threads. 521 // 522 523 void NativeWindowRenderer::startRequest(int cmd) { 524 mLock.lock(); 525 while (mThreadCmd != CMD_IDLE) { 526 mCond.wait(mLock); 527 } 528 mThreadCmd = cmd; 529 } 530 531 void NativeWindowRenderer::sendRequest() { 532 mCond.broadcast(); 533 while (mThreadCmd != CMD_IDLE) { 534 mCond.wait(mLock); 535 } 536 mLock.unlock(); 537 } 538 539 RenderInput* NativeWindowRenderer::createRenderInput() { 540 ALOGD("new render input %d", mNextTextureId); 541 RenderInput* input = new RenderInput(this, mNextTextureId); 542 543 startRequest(CMD_RESERVE_TEXTURE); 544 mThreadTextureId = mNextTextureId; 545 sendRequest(); 546 547 mNextTextureId++; 548 mActiveInputs++; 549 return input; 550 } 551 552 void NativeWindowRenderer::destroyRenderInput(RenderInput* input) { 553 ALOGD("destroy render input %d", input->mTextureId); 554 GLuint textureId = input->mTextureId; 555 delete input; 556 557 startRequest(CMD_DELETE_TEXTURE); 558 mThreadTextureId = textureId; 559 sendRequest(); 560 561 mActiveInputs--; 562 } 563 564 // 565 // RenderInput 566 // 567 568 RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId) 569 : mRenderer(renderer) 570 , mTextureId(textureId) { 571 sp<BufferQueue> bq = new BufferQueue(); 572 mST = new GLConsumer(bq, mTextureId); 573 mSTC = new Surface(bq); 574 native_window_connect(mSTC.get(), NATIVE_WINDOW_API_MEDIA); 575 } 576 577 RenderInput::~RenderInput() { 578 } 579 580 ANativeWindow* RenderInput::getTargetWindow() { 581 return mSTC.get(); 582 } 583 584 void RenderInput::updateVideoSize(sp<MetaData> meta) { 585 CHECK(meta->findInt32(kKeyWidth, &mWidth)); 586 CHECK(meta->findInt32(kKeyHeight, &mHeight)); 587 588 int left, top, right, bottom; 589 if (meta->findRect(kKeyCropRect, &left, &top, &right, &bottom)) { 590 mWidth = right - left + 1; 591 mHeight = bottom - top + 1; 592 } 593 594 // If rotation degrees is 90 or 270, swap width and height 595 // (mWidth and mHeight are the _rotated_ source rectangle). 596 int32_t rotationDegrees; 597 if (!meta->findInt32(kKeyRotation, &rotationDegrees)) { 598 rotationDegrees = 0; 599 } 600 601 if (rotationDegrees == 90 || rotationDegrees == 270) { 602 int tmp = mWidth; 603 mWidth = mHeight; 604 mHeight = tmp; 605 } 606 } 607 608 void RenderInput::render(MediaBuffer* buffer, uint32_t videoEffect, 609 M4xVSS_MediaRendering renderingMode, bool isExternalBuffer) { 610 mVideoEffect = videoEffect; 611 mRenderingMode = renderingMode; 612 mIsExternalBuffer = isExternalBuffer; 613 mBuffer = buffer; 614 615 mRenderer->startRequest(NativeWindowRenderer::CMD_RENDER_INPUT); 616 mRenderer->mThreadRenderInput = this; 617 mRenderer->sendRequest(); 618 } 619 620 } // namespace android 621