1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 /* 18 * 19 */ 20 #include <string.h> 21 #include <jni.h> 22 #include <stdio.h> 23 #include <stdlib.h> 24 #include <time.h> 25 #include <db_utilities_camera.h> 26 27 #include "mosaic/AlignFeatures.h" 28 #include "mosaic/Blend.h" 29 #include "mosaic/Mosaic.h" 30 #include "mosaic/Log.h" 31 #define LOG_TAG "FEATURE_MOS_JNI" 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 37 #include "mosaic_renderer_jni.h" 38 39 char buffer[1024]; 40 41 const int MAX_FRAMES = 100; 42 43 static double mTx; 44 45 int tWidth[NR]; 46 int tHeight[NR]; 47 48 ImageType tImage[NR][MAX_FRAMES];// = {{ImageUtils::IMAGE_TYPE_NOIMAGE}}; // YVU24 format image 49 Mosaic *mosaic[NR] = {NULL,NULL}; 50 ImageType resultYVU = ImageUtils::IMAGE_TYPE_NOIMAGE; 51 ImageType resultBGR = ImageUtils::IMAGE_TYPE_NOIMAGE; 52 float gTRS[11]; // 9 elements of the transformation, 1 for frame-number, 1 for alignment error code. 53 // Variables to keep track of the mosaic computation progress for both LR & HR. 54 float gProgress[NR]; 55 // Variables to be able to cancel the mosaic computation when the GUI says so. 56 bool gCancelComputation[NR]; 57 58 int c; 59 int width=0, height=0; 60 int mosaicWidth=0, mosaicHeight=0; 61 62 //int blendingType = Blend::BLEND_TYPE_FULL; 63 //int blendingType = Blend::BLEND_TYPE_CYLPAN; 64 int blendingType = Blend::BLEND_TYPE_HORZ; 65 int stripType = Blend::STRIP_TYPE_THIN; 66 bool high_res = false; 67 bool quarter_res[NR] = {false,false}; 68 float thresh_still[NR] = {5.0f,0.0f}; 69 70 /* return current time in milliseconds*/ 71 72 #ifndef now_ms 73 static double 74 now_ms(void) 75 { 76 //struct timespec res; 77 struct timeval res; 78 //clock_gettime(CLOCK_REALTIME, &res); 79 gettimeofday(&res, NULL); 80 return 1000.0*res.tv_sec + (double)res.tv_usec/1e3; 81 } 82 #endif 83 84 85 static int frame_number_HR = 0; 86 static int frame_number_LR = 0; 87 88 int Init(int mID, int nmax) 89 { 90 double t0, t1, time_c; 91 92 if(mosaic[mID]!=NULL) 93 { 94 delete mosaic[mID]; 95 mosaic[mID] = NULL; 96 } 97 98 mosaic[mID] = new Mosaic(); 99 100 t0 = now_ms(); 101 102 // When processing higher than 720x480 video, process low-res at 103 // quarter resolution 104 if(tWidth[LR]>180) 105 quarter_res[LR] = true; 106 107 108 // Check for initialization and if not, initialize 109 if (!mosaic[mID]->isInitialized()) 110 { 111 mosaic[mID]->initialize(blendingType, stripType, tWidth[mID], tHeight[mID], 112 nmax, quarter_res[mID], thresh_still[mID]); 113 } 114 115 t1 = now_ms(); 116 time_c = t1 - t0; 117 LOGV("Init[%d]: %g ms [%d frames]",mID,time_c,nmax); 118 return 1; 119 } 120 121 void GenerateQuarterResImagePlanar(ImageType im, int input_w, int input_h, 122 ImageType &out) 123 { 124 ImageType imp; 125 ImageType outp; 126 127 int count = 0; 128 129 for (int j = 0; j < input_h; j += H2L_FACTOR) 130 { 131 imp = im + j * input_w; 132 outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR); 133 134 for (int i = 0; i < input_w; i += H2L_FACTOR) 135 { 136 *outp++ = *(imp + i); 137 count++; 138 } 139 } 140 141 for (int j = input_h; j < 2 * input_h; j += H2L_FACTOR) 142 { 143 imp = im + j * input_w; 144 outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR); 145 146 for (int i = 0; i < input_w; i += H2L_FACTOR) 147 { 148 *outp++ = *(imp + i); 149 count++; 150 } 151 } 152 153 for (int j = 2 * input_h; j < 3 * input_h; j += H2L_FACTOR) 154 { 155 imp = im + j * input_w; 156 outp = out + (j / H2L_FACTOR) * (input_w / H2L_FACTOR); 157 158 for (int i = 0; i < input_w; i += H2L_FACTOR) 159 { 160 *outp++ = *(imp + i); 161 count++; 162 } 163 } 164 } 165 166 int AddFrame(int mID, int k, float* trs1d) 167 { 168 double t0, t1, time_c; 169 double trs[3][3]; 170 171 int ret_code = mosaic[mID]->addFrame(tImage[mID][k]); 172 173 mosaic[mID]->getAligner()->getLastTRS(trs); 174 175 if(trs1d!=NULL) 176 { 177 178 trs1d[0] = trs[0][0]; 179 trs1d[1] = trs[0][1]; 180 trs1d[2] = trs[0][2]; 181 trs1d[3] = trs[1][0]; 182 trs1d[4] = trs[1][1]; 183 trs1d[5] = trs[1][2]; 184 trs1d[6] = trs[2][0]; 185 trs1d[7] = trs[2][1]; 186 trs1d[8] = trs[2][2]; 187 } 188 189 return ret_code; 190 } 191 192 int Finalize(int mID) 193 { 194 double t0, t1, time_c; 195 196 t0 = now_ms(); 197 // Create the mosaic 198 int ret = mosaic[mID]->createMosaic(gProgress[mID], gCancelComputation[mID]); 199 t1 = now_ms(); 200 time_c = t1 - t0; 201 LOGV("CreateMosaic: %g ms",time_c); 202 203 // Get back the result 204 resultYVU = mosaic[mID]->getMosaic(mosaicWidth, mosaicHeight); 205 206 return ret; 207 } 208 209 void YUV420toYVU24(ImageType yvu24, ImageType yuv420sp, int width, int height) 210 { 211 int frameSize = width * height; 212 213 ImageType oyp = yvu24; 214 ImageType ovp = yvu24+frameSize; 215 ImageType oup = yvu24+frameSize+frameSize; 216 217 for (int j = 0, yp = 0; j < height; j++) 218 { 219 unsigned char u = 0, v = 0; 220 int uvp = frameSize + (j >> 1) * width; 221 for (int i = 0; i < width; i++, yp++) 222 { 223 *oyp++ = yuv420sp[yp]; 224 //int y = (0xff & (int)yuv420sp[yp]) -16; 225 //yvu24p[yp] = (y<0)?0:y; 226 227 if ((i & 1) == 0) 228 { 229 v = yuv420sp[uvp++]; 230 u = yuv420sp[uvp++]; 231 } 232 233 *ovp++ = v; 234 *oup++ = u; 235 } 236 } 237 } 238 239 void YUV420toYVU24_NEW(ImageType yvu24, ImageType yuv420sp, int width, 240 int height) 241 { 242 int frameSize = width * height; 243 244 ImageType oyp = yvu24; 245 ImageType ovp = yvu24 + frameSize; 246 ImageType oup = yvu24 + frameSize + frameSize; 247 248 memcpy(yvu24, yuv420sp, frameSize * sizeof(unsigned char)); 249 250 for (int j = 0; j < height; j += 2) 251 { 252 unsigned char u = 0, v = 0; 253 int uvp = frameSize + (j >> 1) * width; 254 ovp = yvu24 + frameSize + j * width; 255 oup = ovp + frameSize; 256 257 ImageType iuvp = yuv420sp + uvp; 258 259 for (int i = 0; i < width; i += 2) 260 { 261 v = *iuvp++; 262 u = *iuvp++; 263 264 *ovp++ = v; 265 *oup++ = u; 266 267 *ovp++ = v; 268 *oup++ = u; 269 270 } 271 memcpy(ovp, ovp - width, width * sizeof(unsigned char)); 272 memcpy(oup, oup - width, width * sizeof(unsigned char)); 273 } 274 } 275 276 277 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_allocateMosaicMemory( 278 JNIEnv* env, jobject thiz, jint width, jint height) 279 { 280 tWidth[HR] = width; 281 tHeight[HR] = height; 282 tWidth[LR] = int(width / H2L_FACTOR); 283 tHeight[LR] = int(height / H2L_FACTOR); 284 285 for(int i=0; i<MAX_FRAMES; i++) 286 { 287 tImage[LR][i] = ImageUtils::allocateImage(tWidth[LR], tHeight[LR], 288 ImageUtils::IMAGE_TYPE_NUM_CHANNELS); 289 tImage[HR][i] = ImageUtils::allocateImage(tWidth[HR], tHeight[HR], 290 ImageUtils::IMAGE_TYPE_NUM_CHANNELS); 291 } 292 293 AllocateTextureMemory(tWidth[HR], tHeight[HR], tWidth[LR], tHeight[LR]); 294 } 295 296 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_freeMosaicMemory( 297 JNIEnv* env, jobject thiz) 298 { 299 for(int i = 0; i < MAX_FRAMES; i++) 300 { 301 ImageUtils::freeImage(tImage[LR][i]); 302 ImageUtils::freeImage(tImage[HR][i]); 303 } 304 305 FreeTextureMemory(); 306 } 307 308 309 void decodeYUV444SP(unsigned char* rgb, unsigned char* yuv420sp, int width, 310 int height) 311 { 312 int frameSize = width * height; 313 314 for (int j = 0, yp = 0; j < height; j++) 315 { 316 int vp = frameSize + j * width, u = 0, v = 0; 317 int up = vp + frameSize; 318 319 for (int i = 0; i < width; i++, yp++, vp++, up++) 320 { 321 int y = (0xff & ((int) yuv420sp[yp])) - 16; 322 if (y < 0) y = 0; 323 324 v = (0xff & yuv420sp[vp]) - 128; 325 u = (0xff & yuv420sp[up]) - 128; 326 327 int y1192 = 1192 * y; 328 int r = (y1192 + 1634 * v); 329 int g = (y1192 - 833 * v - 400 * u); 330 int b = (y1192 + 2066 * u); 331 332 if (r < 0) r = 0; else if (r > 262143) r = 262143; 333 if (g < 0) g = 0; else if (g > 262143) g = 262143; 334 if (b < 0) b = 0; else if (b > 262143) b = 262143; 335 336 //rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff); 337 int p = j*width*3+i*3; 338 rgb[p+0] = (r<<6 & 0xFF0000)>>16; 339 rgb[p+1] = (g>>2 & 0xFF00)>>8; 340 rgb[p+2] = b>>10 & 0xFF; 341 } 342 } 343 } 344 345 static int count = 0; 346 347 void ConvertYVUAiToPlanarYVU(unsigned char *planar, unsigned char *in, int width, 348 int height) 349 { 350 int planeSize = width * height; 351 unsigned char* Yptr = planar; 352 unsigned char* Vptr = planar + planeSize; 353 unsigned char* Uptr = Vptr + planeSize; 354 355 for (int i = 0; i < planeSize; i++) 356 { 357 *Yptr++ = *in++; 358 *Vptr++ = *in++; 359 *Uptr++ = *in++; 360 in++; // Alpha 361 } 362 } 363 364 JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceImageFromGPU( 365 JNIEnv* env, jobject thiz) 366 { 367 double t0, t1, time_c; 368 t0 = now_ms(); 369 int ret_code = 0; 370 371 if(frame_number_HR<MAX_FRAMES && frame_number_LR<MAX_FRAMES) 372 { 373 double last_tx = mTx; 374 375 sem_wait(&gPreviewImage_semaphore); 376 ConvertYVUAiToPlanarYVU(tImage[LR][frame_number_LR], gPreviewImage[LR], 377 tWidth[LR], tHeight[LR]); 378 379 sem_post(&gPreviewImage_semaphore); 380 381 ret_code = AddFrame(LR, frame_number_LR, gTRS); 382 383 if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS) 384 { 385 // Copy into HR buffer only if this is a valid frame 386 sem_wait(&gPreviewImage_semaphore); 387 ConvertYVUAiToPlanarYVU(tImage[HR][frame_number_HR], gPreviewImage[HR], 388 tWidth[HR], tHeight[HR]); 389 sem_post(&gPreviewImage_semaphore); 390 391 frame_number_LR++; 392 frame_number_HR++; 393 } 394 } 395 else 396 { 397 gTRS[1] = gTRS[2] = gTRS[3] = gTRS[5] = gTRS[6] = gTRS[7] = 0.0f; 398 gTRS[0] = gTRS[4] = gTRS[8] = 1.0f; 399 } 400 401 UpdateWarpTransformation(gTRS); 402 403 gTRS[9] = frame_number_HR; 404 gTRS[10] = ret_code; 405 406 jfloatArray bytes = env->NewFloatArray(11); 407 if(bytes != 0) 408 { 409 env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS); 410 } 411 return bytes; 412 } 413 414 415 416 JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceImage( 417 JNIEnv* env, jobject thiz, jbyteArray photo_data) 418 { 419 double t0, t1, time_c; 420 t0 = now_ms(); 421 422 int ret_code; 423 424 if(frame_number_HR<MAX_FRAMES && frame_number_LR<MAX_FRAMES) 425 { 426 jbyte *pixels = env->GetByteArrayElements(photo_data, 0); 427 428 YUV420toYVU24_NEW(tImage[HR][frame_number_HR], (ImageType)pixels, 429 tWidth[HR], tHeight[HR]); 430 431 env->ReleaseByteArrayElements(photo_data, pixels, 0); 432 433 double last_tx = mTx; 434 435 t0 = now_ms(); 436 GenerateQuarterResImagePlanar(tImage[HR][frame_number_HR], tWidth[HR], 437 tHeight[HR], tImage[LR][frame_number_LR]); 438 439 440 sem_wait(&gPreviewImage_semaphore); 441 decodeYUV444SP(gPreviewImage[LR], tImage[LR][frame_number_LR], 442 gPreviewImageWidth[LR], gPreviewImageHeight[LR]); 443 sem_post(&gPreviewImage_semaphore); 444 445 ret_code = AddFrame(LR, frame_number_LR, gTRS); 446 447 if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS) 448 { 449 frame_number_LR++; 450 frame_number_HR++; 451 } 452 453 } 454 else 455 { 456 gTRS[1] = gTRS[2] = gTRS[3] = gTRS[5] = gTRS[6] = gTRS[7] = 0.0f; 457 gTRS[0] = gTRS[4] = gTRS[8] = 1.0f; 458 } 459 460 UpdateWarpTransformation(gTRS); 461 462 gTRS[9] = frame_number_HR; 463 gTRS[10] = ret_code; 464 465 jfloatArray bytes = env->NewFloatArray(11); 466 if(bytes != 0) 467 { 468 env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS); 469 } 470 return bytes; 471 } 472 473 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setBlendingType( 474 JNIEnv* env, jobject thiz, jint type) 475 { 476 blendingType = int(type); 477 } 478 479 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_setStripType( 480 JNIEnv* env, jobject thiz, jint type) 481 { 482 stripType = int(type); 483 } 484 485 JNIEXPORT void JNICALL Java_com_android_camera_panorama_Mosaic_reset( 486 JNIEnv* env, jobject thiz) 487 { 488 frame_number_HR = 0; 489 frame_number_LR = 0; 490 491 gProgress[LR] = 0.0; 492 gProgress[HR] = 0.0; 493 494 gCancelComputation[LR] = false; 495 gCancelComputation[HR] = false; 496 497 Init(LR,MAX_FRAMES); 498 } 499 500 JNIEXPORT jint JNICALL Java_com_android_camera_panorama_Mosaic_reportProgress( 501 JNIEnv* env, jobject thiz, jboolean hires, jboolean cancel_computation) 502 { 503 if(bool(hires)) 504 gCancelComputation[HR] = cancel_computation; 505 else 506 gCancelComputation[LR] = cancel_computation; 507 508 if(bool(hires)) 509 return (jint) gProgress[HR]; 510 else 511 return (jint) gProgress[LR]; 512 } 513 514 JNIEXPORT jint JNICALL Java_com_android_camera_panorama_Mosaic_createMosaic( 515 JNIEnv* env, jobject thiz, jboolean value) 516 { 517 high_res = bool(value); 518 519 int ret; 520 521 if(high_res) 522 { 523 LOGV("createMosaic() - High-Res Mode"); 524 double t0, t1, time_c; 525 526 gProgress[HR] = 0.0; 527 t0 = now_ms(); 528 529 Init(HR, frame_number_HR); 530 531 for(int k = 0; k < frame_number_HR; k++) 532 { 533 if (gCancelComputation[HR]) 534 break; 535 AddFrame(HR, k, NULL); 536 gProgress[HR] += TIME_PERCENT_ALIGN/frame_number_HR; 537 } 538 539 if (gCancelComputation[HR]) 540 { 541 ret = Mosaic::MOSAIC_RET_CANCELLED; 542 } 543 else 544 { 545 gProgress[HR] = TIME_PERCENT_ALIGN; 546 547 t1 = now_ms(); 548 time_c = t1 - t0; 549 LOGV("AlignAll - %d frames [HR]: %g ms", frame_number_HR, time_c); 550 551 ret = Finalize(HR); 552 553 gProgress[HR] = 100.0; 554 } 555 556 high_res = false; 557 } 558 else 559 { 560 LOGV("createMosaic() - Low-Res Mode"); 561 gProgress[LR] = TIME_PERCENT_ALIGN; 562 563 ret = Finalize(LR); 564 565 gProgress[LR] = 100.0; 566 } 567 568 return (jint) ret; 569 } 570 571 JNIEXPORT jintArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaic( 572 JNIEnv* env, jobject thiz) 573 { 574 int y,x; 575 int width = mosaicWidth; 576 int height = mosaicHeight; 577 int imageSize = width * height; 578 579 // Convert back to RGB24 580 resultBGR = ImageUtils::allocateImage(mosaicWidth, mosaicHeight, 581 ImageUtils::IMAGE_TYPE_NUM_CHANNELS); 582 ImageUtils::yvu2bgr(resultBGR, resultYVU, mosaicWidth, mosaicHeight); 583 584 LOGV("MosBytes: %d, W = %d, H = %d", imageSize, width, height); 585 586 int* image = new int[imageSize]; 587 int* dims = new int[2]; 588 589 for(y=0; y<height; y++) 590 { 591 for(x=0; x<width; x++) 592 { 593 image[y*width+x] = (0xFF<<24) | (resultBGR[y*width*3+x*3+2]<<16)| 594 (resultBGR[y*width*3+x*3+1]<<8)| (resultBGR[y*width*3+x*3]); 595 } 596 } 597 598 dims[0] = width; 599 dims[1] = height; 600 601 ImageUtils::freeImage(resultBGR); 602 603 jintArray bytes = env->NewIntArray(imageSize+2); 604 if (bytes == 0) { 605 LOGE("Error in creating the image."); 606 delete[] image; 607 return 0; 608 } 609 env->SetIntArrayRegion(bytes, 0, imageSize, (jint*) image); 610 env->SetIntArrayRegion(bytes, imageSize, 2, (jint*) dims); 611 delete[] image; 612 delete[] dims; 613 return bytes; 614 } 615 616 JNIEXPORT jbyteArray JNICALL Java_com_android_camera_panorama_Mosaic_getFinalMosaicNV21( 617 JNIEnv* env, jobject thiz) 618 { 619 int y,x; 620 int width; 621 int height; 622 623 width = mosaicWidth; 624 height = mosaicHeight; 625 626 int imageSize = 1.5*width * height; 627 628 // Convert YVU to NV21 format in-place 629 ImageType V = resultYVU+mosaicWidth*mosaicHeight; 630 ImageType U = V+mosaicWidth*mosaicHeight; 631 for(int j=0; j<mosaicHeight/2; j++) 632 { 633 for(int i=0; i<mosaicWidth; i+=2) 634 { 635 V[j*mosaicWidth+i] = V[(2*j)*mosaicWidth+i]; // V 636 V[j*mosaicWidth+i+1] = U[(2*j)*mosaicWidth+i]; // U 637 } 638 } 639 640 LOGV("MosBytes: %d, W = %d, H = %d", imageSize, width, height); 641 642 unsigned char* dims = new unsigned char[8]; 643 644 dims[0] = (unsigned char)(width >> 24); 645 dims[1] = (unsigned char)(width >> 16); 646 dims[2] = (unsigned char)(width >> 8); 647 dims[3] = (unsigned char)width; 648 649 dims[4] = (unsigned char)(height >> 24); 650 dims[5] = (unsigned char)(height >> 16); 651 dims[6] = (unsigned char)(height >> 8); 652 dims[7] = (unsigned char)height; 653 654 jbyteArray bytes = env->NewByteArray(imageSize+8); 655 if (bytes == 0) { 656 LOGE("Error in creating the image."); 657 ImageUtils::freeImage(resultYVU); 658 return 0; 659 } 660 env->SetByteArrayRegion(bytes, 0, imageSize, (jbyte*) resultYVU); 661 env->SetByteArrayRegion(bytes, imageSize, 8, (jbyte*) dims); 662 delete[] dims; 663 ImageUtils::freeImage(resultYVU); 664 return bytes; 665 } 666 667 #ifdef __cplusplus 668 } 669 #endif 670
