Home | History | Annotate | Download | only in jni
      1 /*
      2  * Copyright (C) 2011 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include <GLES2/gl2.h>
     18 #include <GLES2/gl2ext.h>
     19 #include <jni.h>
     20 #include <math.h>
     21 #include <stdio.h>
     22 #include <stdlib.h>
     23 #include "db_utilities_camera.h"
     24 #include "mosaic/ImageUtils.h"
     25 #include "mosaic_renderer/FrameBuffer.h"
     26 #include "mosaic_renderer/WarpRenderer.h"
     27 #include "mosaic_renderer/SurfaceTextureRenderer.h"
     28 #include "mosaic_renderer/YVURenderer.h"
     29 
     30 #include "mosaic/Log.h"
     31 #define LOG_TAG "MosaicRenderer"
     32 
     33 #include "mosaic_renderer_jni.h"
     34 
     35 // Texture handle
     36 GLuint gSurfaceTextureID[1];
     37 
     38 bool gWarpImage = true;
     39 
     40 // Low-Res input image frame in YUVA format for preview rendering and processing
     41 // and high-res YUVA input image for processing.
     42 unsigned char* gPreviewImage[NR];
     43 // Low-Res & high-res preview image width
     44 int gPreviewImageWidth[NR];
     45 // Low-Res & high-res preview image height
     46 int gPreviewImageHeight[NR];
     47 
     48 // Semaphore to protect simultaneous read/writes from gPreviewImage
     49 sem_t gPreviewImage_semaphore;
     50 
     51 // Off-screen preview FBO width (large enough to store the entire
     52 // preview mosaic). FBO is frame buffer object.
     53 int gPreviewFBOWidth;
     54 // Off-screen preview FBO height (large enough to store the entire
     55 // preview mosaic).
     56 int gPreviewFBOHeight;
     57 
     58 // gK is the transformation to map the canonical {-1,1} vertex coordinate system
     59 // to the {0,gPreviewImageWidth[LR]} input image frame coordinate system before
     60 // applying the given affine transformation trs. gKm is the corresponding
     61 // transformation for going to the {0,gPreviewFBOWidth}.
     62 double gK[9];
     63 double gKinv[9];
     64 double gKm[9];
     65 double gKminv[9];
     66 
     67 // Shader to copy input SurfaceTexture into and RGBA FBO. The two shaders
     68 // render to the textures with dimensions corresponding to the low-res and
     69 // high-res image frames.
     70 SurfaceTextureRenderer gSurfTexRenderer[NR];
     71 // Off-screen FBOs to store the low-res and high-res RGBA copied out from
     72 // the SurfaceTexture by the gSurfTexRenderers.
     73 FrameBuffer gBufferInput[NR];
     74 
     75 // Shader to convert RGBA textures into YVU textures for processing
     76 YVURenderer gYVURenderer[NR];
     77 // Off-screen FBOs to store the low-res and high-res YVU textures for processing
     78 FrameBuffer gBufferInputYVU[NR];
     79 
     80 // Shader to translate the flip-flop FBO - gBuffer[1-current] -> gBuffer[current]
     81 WarpRenderer gWarper1;
     82 // Shader to add warped current frame to the flip-flop FBO - gBuffer[current]
     83 WarpRenderer gWarper2;
     84 // Off-screen FBOs (flip-flop) to store the result of gWarper1 & gWarper2
     85 FrameBuffer gBuffer[2];
     86 
     87 // Shader to warp and render the preview FBO to the screen
     88 WarpRenderer gPreview;
     89 
     90 // Index of the gBuffer FBO gWarper1 is going to write into
     91 int gCurrentFBOIndex = 0;
     92 
     93 // 3x3 Matrices holding the transformation of this frame (gThisH1t) and of
     94 // the last frame (gLastH1t) w.r.t the first frame.
     95 double gThisH1t[9];
     96 double gLastH1t[9];
     97 
     98 // Variables to represent the fixed position of the top-left corner of the
     99 // current frame in the previewFBO
    100 double gCenterOffsetX = 0.0f;
    101 double gCenterOffsetY = 0.0f;
    102 
    103 // X-Offset of the viewfinder (current frame) w.r.t
    104 // (gCenterOffsetX, gCenterOffsetY). This offset varies with time and is
    105 // used to pan the viewfinder across the UI layout.
    106 double gPanOffset = 0.0f;
    107 
    108 // Variables tracking the translation value for the current frame and the
    109 // last frame (both w.r.t the first frame). The difference between these
    110 // values is used to control the panning speed of the viewfinder display
    111 // on the UI screen.
    112 double gThisTx = 0.0f;
    113 double gLastTx = 0.0f;
    114 
    115 // These are the scale factors used by the gPreview shader to ensure that
    116 // the image frame is correctly scaled to the full UI layout height while
    117 // maintaining its aspect ratio
    118 double gUILayoutScalingX = 1.0f;
    119 double gUILayoutScalingY = 1.0f;
    120 
    121 // Whether the view that we will render preview FBO onto is in landscape or portrait
    122 // orientation.
    123 bool gIsLandscapeOrientation = true;
    124 
    125 // State of the viewfinder. Set to false when the viewfinder hits the UI edge.
    126 bool gPanViewfinder = true;
    127 
    128 // Affine transformation in GL 4x4 format (column-major) to warp the
    129 // last frame mosaic into the current frame coordinate system.
    130 GLfloat g_dAffinetransGL[16];
    131 double g_dAffinetrans[16];
    132 
    133 // Affine transformation in GL 4x4 format (column-major) to translate the
    134 // preview FBO across the screen (viewfinder panning).
    135 GLfloat g_dAffinetransPanGL[16];
    136 double g_dAffinetransPan[16];
    137 
    138 // XY translation in GL 4x4 format (column-major) to center the current
    139 // preview mosaic in the preview FBO
    140 GLfloat g_dTranslationToFBOCenterGL[16];
    141 double g_dTranslationToFBOCenter[16];
    142 
    143 // GL 4x4 Identity transformation
    144 GLfloat g_dAffinetransIdentGL[] = {
    145     1., 0., 0., 0.,
    146     0., 1., 0., 0.,
    147     0., 0., 1., 0.,
    148     0., 0., 0., 1.};
    149 
    150 // GL 4x4 Rotation transformation (column-majored): 90 degree
    151 GLfloat g_dAffinetransRotation90GL[] = {
    152     0., 1., 0., 0.,
    153     -1., 0., 0., 0.,
    154     0., 0., 1., 0.,
    155     0., 0., 0., 1.};
    156 
    157 // 3x3 Rotation transformation (row-majored): 90 degree
    158 double gRotation90[] = {
    159     0., -1., 0.,
    160     1., 0., 0.,
    161     0., 0., 1.,};
    162 
    163 
    164 float g_dIdent3x3[] = {
    165     1.0, 0.0, 0.0,
    166     0.0, 1.0, 0.0,
    167     0.0, 0.0, 1.0};
    168 
    169 const int GL_TEXTURE_EXTERNAL_OES_ENUM = 0x8D65;
    170 
    171 static void printGLString(const char *name, GLenum s) {
    172     const char *v = (const char *) glGetString(s);
    173     LOGI("GL %s = %s", name, v);
    174 }
    175 
    176 void checkFramebufferStatus(const char* name) {
    177     GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
    178     if (status == 0) {
    179       LOGE("Checking completeness of Framebuffer:%s", name);
    180       checkGlError("checkFramebufferStatus (is the target \"GL_FRAMEBUFFER\"?)");
    181     } else if (status != GL_FRAMEBUFFER_COMPLETE) {
    182         const char* msg = "not listed";
    183         switch (status) {
    184           case GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT: msg = "attachment"; break;
    185           case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS: msg = "dimensions"; break;
    186           case GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: msg = "missing attachment"; break;
    187           case GL_FRAMEBUFFER_UNSUPPORTED: msg = "unsupported"; break;
    188         }
    189         LOGE("Framebuffer: %s is INCOMPLETE: %s, %x", name, msg, status);
    190     }
    191 }
    192 
    193 // @return false if there was an error
    194 bool checkGLErrorDetail(const char* file, int line, const char* op) {
    195     GLint error = glGetError();
    196     const char* err_msg = "NOT_LISTED";
    197     if (error != 0) {
    198         switch (error) {
    199             case GL_INVALID_VALUE: err_msg = "NOT_LISTED_YET"; break;
    200             case GL_INVALID_OPERATION: err_msg = "INVALID_OPERATION"; break;
    201             case GL_INVALID_ENUM: err_msg = "INVALID_ENUM"; break;
    202         }
    203         LOGE("Error after %s(). glError: %s (0x%x) in line %d of %s", op, err_msg, error, line, file);
    204         return false;
    205     }
    206     return true;
    207 }
    208 
    209 void bindSurfaceTexture(GLuint texId)
    210 {
    211     glBindTexture(GL_TEXTURE_EXTERNAL_OES_ENUM, texId);
    212 
    213     // Can't do mipmapping with camera source
    214     glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MIN_FILTER,
    215             GL_LINEAR);
    216     glTexParameterf(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_MAG_FILTER,
    217             GL_LINEAR);
    218     // Clamp to edge is the only option
    219     glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_S,
    220             GL_CLAMP_TO_EDGE);
    221     glTexParameteri(GL_TEXTURE_EXTERNAL_OES_ENUM, GL_TEXTURE_WRAP_T,
    222             GL_CLAMP_TO_EDGE);
    223 }
    224 
    225 void ClearPreviewImage(int mID)
    226 {
    227     unsigned char* ptr = gPreviewImage[mID];
    228     for(int j = 0, i = 0;
    229             j < gPreviewImageWidth[mID] * gPreviewImageHeight[mID] * 4;
    230             j += 4)
    231     {
    232             ptr[i++] = 0;
    233             ptr[i++] = 0;
    234             ptr[i++] = 0;
    235             ptr[i++] = 255;
    236     }
    237 
    238 }
    239 
    240 void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33)
    241 {
    242     matGL44[0] = mat33[0];
    243     matGL44[1] = mat33[3];
    244     matGL44[2] = 0.0;
    245     matGL44[3] = mat33[6];
    246 
    247     matGL44[4] = mat33[1];
    248     matGL44[5] = mat33[4];
    249     matGL44[6] = 0.0;
    250     matGL44[7] = mat33[7];
    251 
    252     matGL44[8] = 0;
    253     matGL44[9] = 0;
    254     matGL44[10] = 1.0;
    255     matGL44[11] = 0.0;
    256 
    257     matGL44[12] = mat33[2];
    258     matGL44[13] = mat33[5];
    259     matGL44[14] = 0.0;
    260     matGL44[15] = mat33[8];
    261 }
    262 
    263 bool continuePanningFBO(double panOffset) {
    264     double normalizedScreenLimitLeft = -1.0 + VIEWPORT_BORDER_FACTOR_HORZ * 2.0;
    265     double normalizedScreenLimitRight = 1.0 - VIEWPORT_BORDER_FACTOR_HORZ * 2.0;
    266     double normalizedXPositionOnScreenLeft;
    267     double normalizedXPositionOnScreenRight;
    268 
    269     // Compute the position of the current frame in the screen coordinate system
    270     if (gIsLandscapeOrientation) {
    271         normalizedXPositionOnScreenLeft = (2.0 *
    272             (gCenterOffsetX + panOffset) / gPreviewFBOWidth - 1.0) *
    273             gUILayoutScalingX;
    274         normalizedXPositionOnScreenRight = (2.0 *
    275             ((gCenterOffsetX + panOffset) + gPreviewImageWidth[HR]) /
    276             gPreviewFBOWidth - 1.0) * gUILayoutScalingX;
    277     } else {
    278         normalizedXPositionOnScreenLeft = (2.0 *
    279             (gCenterOffsetX + panOffset) / gPreviewFBOWidth - 1.0) *
    280             gUILayoutScalingY;
    281         normalizedXPositionOnScreenRight = (2.0 *
    282             ((gCenterOffsetX + panOffset) + gPreviewImageWidth[HR]) /
    283             gPreviewFBOWidth - 1.0) * gUILayoutScalingY;
    284     }
    285 
    286     // Stop the viewfinder panning if we hit the maximum border allowed for
    287     // this UI layout
    288     if (normalizedXPositionOnScreenRight > normalizedScreenLimitRight ||
    289             normalizedXPositionOnScreenLeft < normalizedScreenLimitLeft) {
    290         return false;
    291     } else {
    292         return true;
    293     }
    294 }
    295 
    296 // This function computes fills the 4x4 matrices g_dAffinetrans,
    297 // and g_dAffinetransPan using the specified 3x3 affine
    298 // transformation between the first captured frame and the current frame.
    299 // The computed g_dAffinetrans is such that it warps the preview mosaic in
    300 // the last frame's coordinate system into the coordinate system of the
    301 // current frame. Thus, applying this transformation will create the current
    302 // frame mosaic but with the current frame missing. This frame will then be
    303 // pasted in by gWarper2 after translating it by g_dTranslationToFBOCenter.
    304 // The computed g_dAffinetransPan is such that it offsets the computed preview
    305 // mosaic horizontally to make the viewfinder pan within the UI layout.
    306 void UpdateWarpTransformation(float *trs)
    307 {
    308     double H[9], Hp[9], Htemp1[9], Htemp2[9], T[9];
    309 
    310     for(int i = 0; i < 9; i++)
    311     {
    312         gThisH1t[i] = trs[i];
    313     }
    314 
    315     // Alignment is done based on low-res data.
    316     // To render the preview mosaic, the translation of the high-res mosaic is estimated to
    317     // H2L_FACTOR x low-res-based tranlation.
    318     gThisH1t[2] *= H2L_FACTOR;
    319     gThisH1t[5] *= H2L_FACTOR;
    320 
    321     db_Identity3x3(T);
    322     T[2] = -gCenterOffsetX;
    323     T[5] = -gCenterOffsetY;
    324 
    325     // H = ( inv(gThisH1t) * gLastH1t ) * T
    326     db_Identity3x3(Htemp1);
    327     db_Identity3x3(Htemp2);
    328     db_Identity3x3(H);
    329     db_InvertAffineTransform(Htemp1, gThisH1t);
    330     db_Multiply3x3_3x3(Htemp2, Htemp1, gLastH1t);
    331     db_Multiply3x3_3x3(H, Htemp2, T);
    332 
    333     for(int i = 0; i < 9; i++)
    334     {
    335         gLastH1t[i] = gThisH1t[i];
    336     }
    337 
    338     // Move the origin such that the frame is centered in the previewFBO
    339     // i.e. H = inv(T) * H
    340     H[2] += gCenterOffsetX;
    341     H[5] += gCenterOffsetY;
    342 
    343     // Hp = inv(Km) * H * Km
    344     // Km moves the coordinate system from openGL to image pixels so
    345     // that the alignment transform H can be applied to them.
    346     // inv(Km) moves the coordinate system back to openGL normalized
    347     // coordinates so that the shader can correctly render it.
    348     db_Identity3x3(Htemp1);
    349     db_Multiply3x3_3x3(Htemp1, H, gKm);
    350     db_Multiply3x3_3x3(Hp, gKminv, Htemp1);
    351 
    352     ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp);
    353 
    354     ////////////////////////////////////////////////
    355     ////// Compute g_dAffinetransPan now...   //////
    356     ////////////////////////////////////////////////
    357 
    358     gThisTx = trs[2];
    359 
    360     if(gPanViewfinder)
    361     {
    362         gPanOffset += (gThisTx - gLastTx) * VIEWFINDER_PAN_FACTOR_HORZ;
    363     }
    364 
    365     gLastTx = gThisTx;
    366     gPanViewfinder = continuePanningFBO(gPanOffset);
    367 
    368     db_Identity3x3(H);
    369     H[2] = gPanOffset;
    370 
    371     // Hp = inv(Km) * H * Km
    372     db_Identity3x3(Htemp1);
    373     db_Multiply3x3_3x3(Htemp1, H, gKm);
    374     db_Multiply3x3_3x3(Hp, gKminv, Htemp1);
    375 
    376     if (gIsLandscapeOrientation) {
    377         ConvertAffine3x3toGL4x4(g_dAffinetransPan, Hp);
    378     } else {
    379         // rotate Hp by 90 degress.
    380         db_Multiply3x3_3x3(Htemp1, gRotation90, Hp);
    381         ConvertAffine3x3toGL4x4(g_dAffinetransPan, Htemp1);
    382     }
    383 }
    384 
    385 void AllocateTextureMemory(int widthHR, int heightHR, int widthLR, int heightLR)
    386 {
    387     gPreviewImageWidth[HR] = widthHR;
    388     gPreviewImageHeight[HR] = heightHR;
    389 
    390     gPreviewImageWidth[LR] = widthLR;
    391     gPreviewImageHeight[LR] = heightLR;
    392 
    393     sem_wait(&gPreviewImage_semaphore);
    394     gPreviewImage[LR] = ImageUtils::allocateImage(gPreviewImageWidth[LR],
    395             gPreviewImageHeight[LR], 4);
    396     gPreviewImage[HR] = ImageUtils::allocateImage(gPreviewImageWidth[HR],
    397             gPreviewImageHeight[HR], 4);
    398     sem_post(&gPreviewImage_semaphore);
    399 
    400     gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageWidth[HR];
    401     gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageHeight[HR];
    402 
    403     // The origin is such that the current frame will sit with its center
    404     // at the center of the previewFBO
    405     gCenterOffsetX = (gPreviewFBOWidth / 2 - gPreviewImageWidth[HR] / 2);
    406     gCenterOffsetY = (gPreviewFBOHeight / 2 - gPreviewImageHeight[HR] / 2);
    407 
    408     gPanOffset = 0.0f;
    409 
    410     db_Identity3x3(gThisH1t);
    411     db_Identity3x3(gLastH1t);
    412 
    413     gPanViewfinder = true;
    414 
    415     int w = gPreviewImageWidth[HR];
    416     int h = gPreviewImageHeight[HR];
    417 
    418     int wm = gPreviewFBOWidth;
    419     int hm = gPreviewFBOHeight;
    420 
    421     // K is the transformation to map the canonical [-1,1] vertex coordinate
    422     // system to the [0,w] image coordinate system before applying the given
    423     // affine transformation trs.
    424     gKm[0] = wm / 2.0 - 0.5;
    425     gKm[1] = 0.0;
    426     gKm[2] = wm / 2.0 - 0.5;
    427     gKm[3] = 0.0;
    428     gKm[4] = hm / 2.0 - 0.5;
    429     gKm[5] = hm / 2.0 - 0.5;
    430     gKm[6] = 0.0;
    431     gKm[7] = 0.0;
    432     gKm[8] = 1.0;
    433 
    434     gK[0] = w / 2.0 - 0.5;
    435     gK[1] = 0.0;
    436     gK[2] = w / 2.0 - 0.5;
    437     gK[3] = 0.0;
    438     gK[4] = h / 2.0 - 0.5;
    439     gK[5] = h / 2.0 - 0.5;
    440     gK[6] = 0.0;
    441     gK[7] = 0.0;
    442     gK[8] = 1.0;
    443 
    444     db_Identity3x3(gKinv);
    445     db_InvertCalibrationMatrix(gKinv, gK);
    446 
    447     db_Identity3x3(gKminv);
    448     db_InvertCalibrationMatrix(gKminv, gKm);
    449 
    450     //////////////////////////////////////////
    451     ////// Compute g_Translation now... //////
    452     //////////////////////////////////////////
    453     double T[9], Tp[9], Ttemp[9];
    454 
    455     db_Identity3x3(T);
    456     T[2] = gCenterOffsetX;
    457     T[5] = gCenterOffsetY;
    458 
    459     // Tp = inv(K) * T * K
    460     db_Identity3x3(Ttemp);
    461     db_Multiply3x3_3x3(Ttemp, T, gK);
    462     db_Multiply3x3_3x3(Tp, gKinv, Ttemp);
    463 
    464     ConvertAffine3x3toGL4x4(g_dTranslationToFBOCenter, Tp);
    465 
    466     UpdateWarpTransformation(g_dIdent3x3);
    467 }
    468 
    469 void FreeTextureMemory()
    470 {
    471     sem_wait(&gPreviewImage_semaphore);
    472     ImageUtils::freeImage(gPreviewImage[LR]);
    473     ImageUtils::freeImage(gPreviewImage[HR]);
    474     sem_post(&gPreviewImage_semaphore);
    475 }
    476 
    477 extern "C"
    478 {
    479     JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved);
    480     JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved);
    481     JNIEXPORT jint JNICALL Java_com_android_camera_MosaicRenderer_init(
    482             JNIEnv * env, jobject obj);
    483     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_reset(
    484             JNIEnv * env, jobject obj,  jint width, jint height,
    485             jboolean isLandscapeOrientation);
    486     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_preprocess(
    487             JNIEnv * env, jobject obj, jfloatArray stMatrix);
    488     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_transferGPUtoCPU(
    489             JNIEnv * env, jobject obj);
    490     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_step(
    491             JNIEnv * env, jobject obj);
    492     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_updateMatrix(
    493             JNIEnv * env, jobject obj);
    494     JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_setWarping(
    495             JNIEnv * env, jobject obj, jboolean flag);
    496 };
    497 
    498 
    499 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
    500 {
    501     sem_init(&gPreviewImage_semaphore, 0, 1);
    502 
    503     return JNI_VERSION_1_4;
    504 }
    505 
    506 
    507 JNIEXPORT void JNICALL JNI_OnUnload(JavaVM* vm, void* reserved)
    508 {
    509     sem_destroy(&gPreviewImage_semaphore);
    510 }
    511 JNIEXPORT jint JNICALL Java_com_android_camera_MosaicRenderer_init(
    512         JNIEnv * env, jobject obj)
    513 {
    514     gSurfTexRenderer[LR].InitializeGLProgram();
    515     gSurfTexRenderer[HR].InitializeGLProgram();
    516     gYVURenderer[LR].InitializeGLProgram();
    517     gYVURenderer[HR].InitializeGLProgram();
    518     gWarper1.InitializeGLProgram();
    519     gWarper2.InitializeGLProgram();
    520     gPreview.InitializeGLProgram();
    521     gBuffer[0].InitializeGLContext();
    522     gBuffer[1].InitializeGLContext();
    523     gBufferInput[LR].InitializeGLContext();
    524     gBufferInput[HR].InitializeGLContext();
    525     gBufferInputYVU[LR].InitializeGLContext();
    526     gBufferInputYVU[HR].InitializeGLContext();
    527 
    528     glBindFramebuffer(GL_FRAMEBUFFER, 0);
    529 
    530     glGenTextures(1, gSurfaceTextureID);
    531     // bind the surface texture
    532     bindSurfaceTexture(gSurfaceTextureID[0]);
    533 
    534     return (jint) gSurfaceTextureID[0];
    535 }
    536 
    537 // width: the width of the view
    538 // height: the height of the view
    539 // isLandscape: whether the device is in landscape or portrait. Android
    540 //     Compatibility Definition Document specifies that the long side of the
    541 //     camera aligns with the long side of the screen.
    542 void calculateUILayoutScaling(int width, int height, bool isLandscape) {
    543     if (isLandscape) {
    544         //  __________        ________
    545         // |          |  =>  |________|
    546         // |__________|  =>    (View)
    547         // (Preview FBO)
    548         //
    549         // Scale the preview FBO's height to the height of view and
    550         // maintain the aspect ratio of the current frame on the screen.
    551         gUILayoutScalingY = PREVIEW_FBO_HEIGHT_SCALE;
    552 
    553         // Note that OpenGL scales a texture to view's width and height automatically.
    554         // The "width / height" inverts the scaling, so as to maintain the aspect ratio
    555         // of the current frame.
    556         gUILayoutScalingX = ((float) gPreviewFBOWidth / gPreviewFBOHeight)
    557                 / ((float) width / height) * PREVIEW_FBO_HEIGHT_SCALE;
    558     } else {
    559         //                   ___
    560         //  __________      |   |     ______
    561         // |          |  => |   | => |______|
    562         // |__________|  => |   | =>  (View)
    563         // (Preview FBO)    |   |
    564         //                  |___|
    565         //
    566         // Scale the preview FBO's height to the width of view and
    567         // maintain the aspect ratio of the current frame on the screen.
    568         // In preview, Java_com_android_camera_MosaicRenderer_step rotates the
    569         // preview FBO by 90 degrees. In capture, UpdateWarpTransformation
    570         // rotates the preview FBO.
    571         gUILayoutScalingY = PREVIEW_FBO_WIDTH_SCALE;
    572 
    573         // Note that OpenGL scales a texture to view's width and height automatically.
    574         // The "height / width" inverts the scaling, so as to maintain the aspect ratio
    575         // of the current frame.
    576         gUILayoutScalingX = ((float) gPreviewFBOHeight / gPreviewFBOWidth)
    577                 / ((float) width / height) * PREVIEW_FBO_WIDTH_SCALE;
    578     }
    579 }
    580 
    581 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_reset(
    582         JNIEnv * env, jobject obj,  jint width, jint height, jboolean isLandscapeOrientation)
    583 {
    584     gIsLandscapeOrientation = isLandscapeOrientation;
    585     calculateUILayoutScaling(width, height, gIsLandscapeOrientation);
    586 
    587     gBuffer[0].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA);
    588     gBuffer[1].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA);
    589 
    590     gBufferInput[LR].Init(gPreviewImageWidth[LR],
    591             gPreviewImageHeight[LR], GL_RGBA);
    592 
    593     gBufferInput[HR].Init(gPreviewImageWidth[HR],
    594             gPreviewImageHeight[HR], GL_RGBA);
    595 
    596     gBufferInputYVU[LR].Init(gPreviewImageWidth[LR],
    597             gPreviewImageHeight[LR], GL_RGBA);
    598 
    599     gBufferInputYVU[HR].Init(gPreviewImageWidth[HR],
    600             gPreviewImageHeight[HR], GL_RGBA);
    601 
    602     // bind the surface texture
    603     bindSurfaceTexture(gSurfaceTextureID[0]);
    604 
    605     // To speed up, there is no need to clear the destination buffers
    606     // (offscreen/screen buffers) of gSurfTexRenderer, gYVURenderer
    607     // and gPreview because we always fill the whole destination buffers
    608     // when we draw something to those offscreen/screen buffers.
    609     gSurfTexRenderer[LR].SetupGraphics(&gBufferInput[LR]);
    610     gSurfTexRenderer[LR].SetViewportMatrix(1, 1, 1, 1);
    611     gSurfTexRenderer[LR].SetScalingMatrix(1.0f, -1.0f);
    612     gSurfTexRenderer[LR].SetInputTextureName(gSurfaceTextureID[0]);
    613     gSurfTexRenderer[LR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM);
    614 
    615     gSurfTexRenderer[HR].SetupGraphics(&gBufferInput[HR]);
    616     gSurfTexRenderer[HR].SetViewportMatrix(1, 1, 1, 1);
    617     gSurfTexRenderer[HR].SetScalingMatrix(1.0f, -1.0f);
    618     gSurfTexRenderer[HR].SetInputTextureName(gSurfaceTextureID[0]);
    619     gSurfTexRenderer[HR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM);
    620 
    621     gYVURenderer[LR].SetupGraphics(&gBufferInputYVU[LR]);
    622     gYVURenderer[LR].SetInputTextureName(gBufferInput[LR].GetTextureName());
    623     gYVURenderer[LR].SetInputTextureType(GL_TEXTURE_2D);
    624 
    625     gYVURenderer[HR].SetupGraphics(&gBufferInputYVU[HR]);
    626     gYVURenderer[HR].SetInputTextureName(gBufferInput[HR].GetTextureName());
    627     gYVURenderer[HR].SetInputTextureType(GL_TEXTURE_2D);
    628 
    629     // gBuffer[1-gCurrentFBOIndex] --> gWarper1 --> gBuffer[gCurrentFBOIndex]
    630     gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]);
    631 
    632     // Clear the destination buffer of gWarper1.
    633     gWarper1.Clear(0.0, 0.0, 0.0, 1.0);
    634     gWarper1.SetViewportMatrix(1, 1, 1, 1);
    635     gWarper1.SetScalingMatrix(1.0f, 1.0f);
    636     gWarper1.SetInputTextureName(gBuffer[1 - gCurrentFBOIndex].GetTextureName());
    637     gWarper1.SetInputTextureType(GL_TEXTURE_2D);
    638 
    639     // gBufferInput[HR] --> gWarper2 --> gBuffer[gCurrentFBOIndex]
    640     gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]);
    641 
    642     // gWarp2's destination buffer is the same to gWarp1's. No need to clear it
    643     // again.
    644     gWarper2.SetViewportMatrix(gPreviewImageWidth[HR],
    645             gPreviewImageHeight[HR], gBuffer[gCurrentFBOIndex].GetWidth(),
    646             gBuffer[gCurrentFBOIndex].GetHeight());
    647     gWarper2.SetScalingMatrix(1.0f, 1.0f);
    648     gWarper2.SetInputTextureName(gBufferInput[HR].GetTextureName());
    649     gWarper2.SetInputTextureType(GL_TEXTURE_2D);
    650 
    651     // gBuffer[gCurrentFBOIndex] --> gPreview --> Screen
    652     gPreview.SetupGraphics(width, height);
    653     gPreview.SetViewportMatrix(1, 1, 1, 1);
    654 
    655     // Scale the previewFBO so that the viewfinder window fills the layout height
    656     // while maintaining the image aspect ratio
    657     gPreview.SetScalingMatrix(gUILayoutScalingX, -1.0f * gUILayoutScalingY);
    658     gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName());
    659     gPreview.SetInputTextureType(GL_TEXTURE_2D);
    660 }
    661 
    662 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_preprocess(
    663         JNIEnv * env, jobject obj, jfloatArray stMatrix)
    664 {
    665     jfloat *stmat = env->GetFloatArrayElements(stMatrix, 0);
    666 
    667     gSurfTexRenderer[LR].SetSTMatrix((float*) stmat);
    668     gSurfTexRenderer[HR].SetSTMatrix((float*) stmat);
    669 
    670     env->ReleaseFloatArrayElements(stMatrix, stmat, 0);
    671 
    672     gSurfTexRenderer[LR].DrawTexture(g_dAffinetransIdentGL);
    673     gSurfTexRenderer[HR].DrawTexture(g_dAffinetransIdentGL);
    674 }
    675 
    676 #ifndef now_ms
    677 #include <time.h>
    678 static double
    679 now_ms(void)
    680 {
    681     //struct timespec res;
    682     struct timeval res;
    683     //clock_gettime(CLOCK_REALTIME, &res);
    684     gettimeofday(&res, NULL);
    685     return 1000.0*res.tv_sec + (double)res.tv_usec/1e3;
    686 }
    687 #endif
    688 
    689 
    690 
    691 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_transferGPUtoCPU(
    692         JNIEnv * env, jobject obj)
    693 {
    694     double t0, t1, time_c;
    695 
    696     gYVURenderer[LR].DrawTexture();
    697     gYVURenderer[HR].DrawTexture();
    698 
    699     sem_wait(&gPreviewImage_semaphore);
    700     // Bind to the input LR FBO and read the Low-Res data from there...
    701     glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[LR].GetFrameBufferName());
    702     t0 = now_ms();
    703     glReadPixels(0,
    704                  0,
    705                  gBufferInput[LR].GetWidth(),
    706                  gBufferInput[LR].GetHeight(),
    707                  GL_RGBA,
    708                  GL_UNSIGNED_BYTE,
    709                  gPreviewImage[LR]);
    710 
    711     checkGlError("glReadPixels LR (MosaicRenderer.transferGPUtoCPU())");
    712 
    713     // Bind to the input HR FBO and read the high-res data from there...
    714     glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[HR].GetFrameBufferName());
    715     t0 = now_ms();
    716     glReadPixels(0,
    717                  0,
    718                  gBufferInput[HR].GetWidth(),
    719                  gBufferInput[HR].GetHeight(),
    720                  GL_RGBA,
    721                  GL_UNSIGNED_BYTE,
    722                  gPreviewImage[HR]);
    723 
    724     checkGlError("glReadPixels HR (MosaicRenderer.transferGPUtoCPU())");
    725 
    726     sem_post(&gPreviewImage_semaphore);
    727 }
    728 
    729 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_step(
    730         JNIEnv * env, jobject obj)
    731 {
    732     if(!gWarpImage) // ViewFinder
    733     {
    734         gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]);
    735         gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName());
    736 
    737         gWarper2.DrawTexture(g_dTranslationToFBOCenterGL);
    738 
    739         if (gIsLandscapeOrientation) {
    740             gPreview.DrawTexture(g_dAffinetransIdentGL);
    741         } else {
    742             gPreview.DrawTexture(g_dAffinetransRotation90GL);
    743         }
    744     }
    745     else
    746     {
    747         gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]);
    748         // Clear the destination so that we can paint on it afresh
    749         gWarper1.Clear(0.0, 0.0, 0.0, 1.0);
    750         gWarper1.SetInputTextureName(
    751                 gBuffer[1 - gCurrentFBOIndex].GetTextureName());
    752         gWarper2.SetupGraphics(&gBuffer[gCurrentFBOIndex]);
    753         gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName());
    754 
    755         gWarper1.DrawTexture(g_dAffinetransGL);
    756         gWarper2.DrawTexture(g_dTranslationToFBOCenterGL);
    757         gPreview.DrawTexture(g_dAffinetransPanGL);
    758 
    759         gCurrentFBOIndex = 1 - gCurrentFBOIndex;
    760     }
    761 }
    762 
    763 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_setWarping(
    764         JNIEnv * env, jobject obj, jboolean flag)
    765 {
    766     // TODO: Review this logic
    767     if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa
    768     {
    769         // Clear gBuffer[0]
    770         gWarper1.SetupGraphics(&gBuffer[0]);
    771         gWarper1.Clear(0.0, 0.0, 0.0, 1.0);
    772         // Clear gBuffer[1]
    773         gWarper1.SetupGraphics(&gBuffer[1]);
    774         gWarper1.Clear(0.0, 0.0, 0.0, 1.0);
    775         // Clear the screen to black.
    776         gPreview.Clear(0.0, 0.0, 0.0, 1.0);
    777 
    778         gLastTx = 0.0f;
    779         gPanOffset = 0.0f;
    780         gPanViewfinder = true;
    781 
    782         db_Identity3x3(gThisH1t);
    783         db_Identity3x3(gLastH1t);
    784         // Make sure g_dAffinetransGL and g_dAffinetransPanGL are updated.
    785         // Otherwise, the first frame after setting the flag to true will be
    786         // incorrectly drawn.
    787         if ((bool) flag) {
    788             UpdateWarpTransformation(g_dIdent3x3);
    789         }
    790     }
    791 
    792     gWarpImage = (bool)flag;
    793 }
    794 
    795 JNIEXPORT void JNICALL Java_com_android_camera_MosaicRenderer_updateMatrix(
    796         JNIEnv * env, jobject obj)
    797 {
    798     for(int i=0; i<16; i++)
    799     {
    800         g_dAffinetransGL[i] = g_dAffinetrans[i];
    801         g_dAffinetransPanGL[i] = g_dAffinetransPan[i];
    802         g_dTranslationToFBOCenterGL[i] = g_dTranslationToFBOCenter[i];
    803     }
    804 }
    805