1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 //#define LOG_NNDEBUG 0 19 #define LOG_TAG "EmulatedCamera2_Sensor" 20 21 #ifdef LOG_NNDEBUG 22 #define ALOGVV(...) ALOGV(__VA_ARGS__) 23 #else 24 #define ALOGVV(...) ((void)0) 25 #endif 26 27 #include <utils/Log.h> 28 29 #include "../EmulatedFakeCamera2.h" 30 #include "Sensor.h" 31 #include <cmath> 32 #include <cstdlib> 33 #include "system/camera_metadata.h" 34 35 namespace android { 36 37 const unsigned int Sensor::kResolution[2] = {640, 480}; 38 39 const nsecs_t Sensor::kExposureTimeRange[2] = 40 {1000L, 30000000000L} ; // 1 us - 30 sec 41 const nsecs_t Sensor::kFrameDurationRange[2] = 42 {33331760L, 30000000000L}; // ~1/30 s - 30 sec 43 const nsecs_t Sensor::kMinVerticalBlank = 10000L; 44 45 const uint8_t Sensor::kColorFilterArrangement = 46 ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_RGGB; 47 48 // Output image data characteristics 49 const uint32_t Sensor::kMaxRawValue = 4000; 50 const uint32_t Sensor::kBlackLevel = 1000; 51 52 // Sensor sensitivity 53 const float Sensor::kSaturationVoltage = 0.520f; 54 const uint32_t Sensor::kSaturationElectrons = 2000; 55 const float Sensor::kVoltsPerLuxSecond = 0.100f; 56 57 const float Sensor::kElectronsPerLuxSecond = 58 Sensor::kSaturationElectrons / Sensor::kSaturationVoltage 59 * Sensor::kVoltsPerLuxSecond; 60 61 const float Sensor::kBaseGainFactor = (float)Sensor::kMaxRawValue / 62 Sensor::kSaturationElectrons; 63 64 const float Sensor::kReadNoiseStddevBeforeGain = 1.177; // in electrons 65 const float Sensor::kReadNoiseStddevAfterGain = 2.100; // in digital counts 66 const float Sensor::kReadNoiseVarBeforeGain = 67 Sensor::kReadNoiseStddevBeforeGain * 68 Sensor::kReadNoiseStddevBeforeGain; 69 const float Sensor::kReadNoiseVarAfterGain = 70 Sensor::kReadNoiseStddevAfterGain * 71 Sensor::kReadNoiseStddevAfterGain; 72 73 // While each row has to read out, reset, and then expose, the (reset + 74 // expose) sequence can be overlapped by other row readouts, so the final 75 // minimum frame duration is purely a function of row readout time, at least 76 // if there's a reasonable number of rows. 77 const nsecs_t Sensor::kRowReadoutTime = 78 Sensor::kFrameDurationRange[0] / Sensor::kResolution[1]; 79 80 const uint32_t Sensor::kAvailableSensitivities[5] = 81 {100, 200, 400, 800, 1600}; 82 const uint32_t Sensor::kDefaultSensitivity = 100; 83 84 /** A few utility functions for math, normal distributions */ 85 86 // Take advantage of IEEE floating-point format to calculate an approximate 87 // square root. Accurate to within +-3.6% 88 float sqrtf_approx(float r) { 89 // Modifier is based on IEEE floating-point representation; the 90 // manipulations boil down to finding approximate log2, dividing by two, and 91 // then inverting the log2. A bias is added to make the relative error 92 // symmetric about the real answer. 93 const int32_t modifier = 0x1FBB4000; 94 95 int32_t r_i = *(int32_t*)(&r); 96 r_i = (r_i >> 1) + modifier; 97 98 return *(float*)(&r_i); 99 } 100 101 102 103 Sensor::Sensor(): 104 Thread(false), 105 mGotVSync(false), 106 mExposureTime(kFrameDurationRange[0]-kMinVerticalBlank), 107 mFrameDuration(kFrameDurationRange[0]), 108 mGainFactor(kDefaultSensitivity), 109 mNextBuffers(NULL), 110 mFrameNumber(0), 111 mCapturedBuffers(NULL), 112 mListener(NULL), 113 mScene(kResolution[0], kResolution[1], kElectronsPerLuxSecond) 114 { 115 116 } 117 118 Sensor::~Sensor() { 119 shutDown(); 120 } 121 122 status_t Sensor::startUp() { 123 ALOGV("%s: E", __FUNCTION__); 124 125 int res; 126 mCapturedBuffers = NULL; 127 res = run("EmulatedFakeCamera2::Sensor", 128 ANDROID_PRIORITY_URGENT_DISPLAY); 129 130 if (res != OK) { 131 ALOGE("Unable to start up sensor capture thread: %d", res); 132 } 133 return res; 134 } 135 136 status_t Sensor::shutDown() { 137 ALOGV("%s: E", __FUNCTION__); 138 139 int res; 140 res = requestExitAndWait(); 141 if (res != OK) { 142 ALOGE("Unable to shut down sensor capture thread: %d", res); 143 } 144 return res; 145 } 146 147 Scene &Sensor::getScene() { 148 return mScene; 149 } 150 151 void Sensor::setExposureTime(uint64_t ns) { 152 Mutex::Autolock lock(mControlMutex); 153 ALOGVV("Exposure set to %f", ns/1000000.f); 154 mExposureTime = ns; 155 } 156 157 void Sensor::setFrameDuration(uint64_t ns) { 158 Mutex::Autolock lock(mControlMutex); 159 ALOGVV("Frame duration set to %f", ns/1000000.f); 160 mFrameDuration = ns; 161 } 162 163 void Sensor::setSensitivity(uint32_t gain) { 164 Mutex::Autolock lock(mControlMutex); 165 ALOGVV("Gain set to %d", gain); 166 mGainFactor = gain; 167 } 168 169 void Sensor::setDestinationBuffers(Buffers *buffers) { 170 Mutex::Autolock lock(mControlMutex); 171 mNextBuffers = buffers; 172 } 173 174 void Sensor::setFrameNumber(uint32_t frameNumber) { 175 Mutex::Autolock lock(mControlMutex); 176 mFrameNumber = frameNumber; 177 } 178 179 bool Sensor::waitForVSync(nsecs_t reltime) { 180 int res; 181 Mutex::Autolock lock(mControlMutex); 182 183 mGotVSync = false; 184 res = mVSync.waitRelative(mControlMutex, reltime); 185 if (res != OK && res != TIMED_OUT) { 186 ALOGE("%s: Error waiting for VSync signal: %d", __FUNCTION__, res); 187 return false; 188 } 189 return mGotVSync; 190 } 191 192 bool Sensor::waitForNewFrame(nsecs_t reltime, 193 nsecs_t *captureTime) { 194 Mutex::Autolock lock(mReadoutMutex); 195 uint8_t *ret; 196 if (mCapturedBuffers == NULL) { 197 int res; 198 res = mReadoutAvailable.waitRelative(mReadoutMutex, reltime); 199 if (res == TIMED_OUT) { 200 return false; 201 } else if (res != OK || mCapturedBuffers == NULL) { 202 ALOGE("Error waiting for sensor readout signal: %d", res); 203 return false; 204 } 205 } else { 206 mReadoutComplete.signal(); 207 } 208 209 *captureTime = mCaptureTime; 210 mCapturedBuffers = NULL; 211 return true; 212 } 213 214 Sensor::SensorListener::~SensorListener() { 215 } 216 217 void Sensor::setSensorListener(SensorListener *listener) { 218 Mutex::Autolock lock(mControlMutex); 219 mListener = listener; 220 } 221 222 status_t Sensor::readyToRun() { 223 ALOGV("Starting up sensor thread"); 224 mStartupTime = systemTime(); 225 mNextCaptureTime = 0; 226 mNextCapturedBuffers = NULL; 227 return OK; 228 } 229 230 bool Sensor::threadLoop() { 231 /** 232 * Sensor capture operation main loop. 233 * 234 * Stages are out-of-order relative to a single frame's processing, but 235 * in-order in time. 236 */ 237 238 /** 239 * Stage 1: Read in latest control parameters 240 */ 241 uint64_t exposureDuration; 242 uint64_t frameDuration; 243 uint32_t gain; 244 Buffers *nextBuffers; 245 uint32_t frameNumber; 246 SensorListener *listener = NULL; 247 { 248 Mutex::Autolock lock(mControlMutex); 249 exposureDuration = mExposureTime; 250 frameDuration = mFrameDuration; 251 gain = mGainFactor; 252 nextBuffers = mNextBuffers; 253 frameNumber = mFrameNumber; 254 listener = mListener; 255 // Don't reuse a buffer set 256 mNextBuffers = NULL; 257 258 // Signal VSync for start of readout 259 ALOGVV("Sensor VSync"); 260 mGotVSync = true; 261 mVSync.signal(); 262 } 263 264 /** 265 * Stage 3: Read out latest captured image 266 */ 267 268 Buffers *capturedBuffers = NULL; 269 nsecs_t captureTime = 0; 270 271 nsecs_t startRealTime = systemTime(); 272 // Stagefright cares about system time for timestamps, so base simulated 273 // time on that. 274 nsecs_t simulatedTime = startRealTime; 275 nsecs_t frameEndRealTime = startRealTime + frameDuration; 276 nsecs_t frameReadoutEndRealTime = startRealTime + 277 kRowReadoutTime * kResolution[1]; 278 279 if (mNextCapturedBuffers != NULL) { 280 ALOGVV("Sensor starting readout"); 281 // Pretend we're doing readout now; will signal once enough time has elapsed 282 capturedBuffers = mNextCapturedBuffers; 283 captureTime = mNextCaptureTime; 284 } 285 simulatedTime += kRowReadoutTime + kMinVerticalBlank; 286 287 // TODO: Move this signal to another thread to simulate readout 288 // time properly 289 if (capturedBuffers != NULL) { 290 ALOGVV("Sensor readout complete"); 291 Mutex::Autolock lock(mReadoutMutex); 292 if (mCapturedBuffers != NULL) { 293 ALOGV("Waiting for readout thread to catch up!"); 294 mReadoutComplete.wait(mReadoutMutex); 295 } 296 297 mCapturedBuffers = capturedBuffers; 298 mCaptureTime = captureTime; 299 mReadoutAvailable.signal(); 300 capturedBuffers = NULL; 301 } 302 303 /** 304 * Stage 2: Capture new image 305 */ 306 mNextCaptureTime = simulatedTime; 307 mNextCapturedBuffers = nextBuffers; 308 309 if (mNextCapturedBuffers != NULL) { 310 if (listener != NULL) { 311 listener->onSensorEvent(frameNumber, SensorListener::EXPOSURE_START, 312 mNextCaptureTime); 313 } 314 ALOGVV("Starting next capture: Exposure: %f ms, gain: %d", 315 (float)exposureDuration/1e6, gain); 316 mScene.setExposureDuration((float)exposureDuration/1e9); 317 mScene.calculateScene(mNextCaptureTime); 318 319 // Might be adding more buffers, so size isn't constant 320 for (size_t i = 0; i < mNextCapturedBuffers->size(); i++) { 321 const StreamBuffer &b = (*mNextCapturedBuffers)[i]; 322 ALOGVV("Sensor capturing buffer %d: stream %d," 323 " %d x %d, format %x, stride %d, buf %p, img %p", 324 i, b.streamId, b.width, b.height, b.format, b.stride, 325 b.buffer, b.img); 326 switch(b.format) { 327 case HAL_PIXEL_FORMAT_RAW_SENSOR: 328 captureRaw(b.img, gain, b.stride); 329 break; 330 case HAL_PIXEL_FORMAT_RGB_888: 331 captureRGB(b.img, gain, b.stride); 332 break; 333 case HAL_PIXEL_FORMAT_RGBA_8888: 334 captureRGBA(b.img, gain, b.stride); 335 break; 336 case HAL_PIXEL_FORMAT_BLOB: 337 // Add auxillary buffer of the right size 338 // Assumes only one BLOB (JPEG) buffer in 339 // mNextCapturedBuffers 340 StreamBuffer bAux; 341 bAux.streamId = 0; 342 bAux.width = b.width; 343 bAux.height = b.height; 344 bAux.format = HAL_PIXEL_FORMAT_RGB_888; 345 bAux.stride = b.width; 346 bAux.buffer = NULL; 347 // TODO: Reuse these 348 bAux.img = new uint8_t[b.width * b.height * 3]; 349 mNextCapturedBuffers->push_back(bAux); 350 break; 351 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 352 captureNV21(b.img, gain, b.stride); 353 break; 354 case HAL_PIXEL_FORMAT_YV12: 355 // TODO: 356 ALOGE("%s: Format %x is TODO", __FUNCTION__, b.format); 357 break; 358 default: 359 ALOGE("%s: Unknown format %x, no output", __FUNCTION__, 360 b.format); 361 break; 362 } 363 } 364 } 365 366 ALOGVV("Sensor vertical blanking interval"); 367 nsecs_t workDoneRealTime = systemTime(); 368 const nsecs_t timeAccuracy = 2e6; // 2 ms of imprecision is ok 369 if (workDoneRealTime < frameEndRealTime - timeAccuracy) { 370 timespec t; 371 t.tv_sec = (frameEndRealTime - workDoneRealTime) / 1000000000L; 372 t.tv_nsec = (frameEndRealTime - workDoneRealTime) % 1000000000L; 373 374 int ret; 375 do { 376 ret = nanosleep(&t, &t); 377 } while (ret != 0); 378 } 379 nsecs_t endRealTime = systemTime(); 380 ALOGVV("Frame cycle took %d ms, target %d ms", 381 (int)((endRealTime - startRealTime)/1000000), 382 (int)(frameDuration / 1000000)); 383 return true; 384 }; 385 386 void Sensor::captureRaw(uint8_t *img, uint32_t gain, uint32_t stride) { 387 float totalGain = gain/100.0 * kBaseGainFactor; 388 float noiseVarGain = totalGain * totalGain; 389 float readNoiseVar = kReadNoiseVarBeforeGain * noiseVarGain 390 + kReadNoiseVarAfterGain; 391 392 int bayerSelect[4] = {Scene::R, Scene::Gr, Scene::Gb, Scene::B}; // RGGB 393 mScene.setReadoutPixel(0,0); 394 for (unsigned int y = 0; y < kResolution[1]; y++ ) { 395 int *bayerRow = bayerSelect + (y & 0x1) * 2; 396 uint16_t *px = (uint16_t*)img + y * stride; 397 for (unsigned int x = 0; x < kResolution[0]; x++) { 398 uint32_t electronCount; 399 electronCount = mScene.getPixelElectrons()[bayerRow[x & 0x1]]; 400 401 // TODO: Better pixel saturation curve? 402 electronCount = (electronCount < kSaturationElectrons) ? 403 electronCount : kSaturationElectrons; 404 405 // TODO: Better A/D saturation curve? 406 uint16_t rawCount = electronCount * totalGain; 407 rawCount = (rawCount < kMaxRawValue) ? rawCount : kMaxRawValue; 408 409 // Calculate noise value 410 // TODO: Use more-correct Gaussian instead of uniform noise 411 float photonNoiseVar = electronCount * noiseVarGain; 412 float noiseStddev = sqrtf_approx(readNoiseVar + photonNoiseVar); 413 // Scaled to roughly match gaussian/uniform noise stddev 414 float noiseSample = std::rand() * (2.5 / (1.0 + RAND_MAX)) - 1.25; 415 416 rawCount += kBlackLevel; 417 rawCount += noiseStddev * noiseSample; 418 419 *px++ = rawCount; 420 } 421 // TODO: Handle this better 422 //simulatedTime += kRowReadoutTime; 423 } 424 ALOGVV("Raw sensor image captured"); 425 } 426 427 void Sensor::captureRGBA(uint8_t *img, uint32_t gain, uint32_t stride) { 428 float totalGain = gain/100.0 * kBaseGainFactor; 429 // In fixed-point math, calculate total scaling from electrons to 8bpp 430 int scale64x = 64 * totalGain * 255 / kMaxRawValue; 431 uint32_t inc = kResolution[0] / stride; 432 433 for (unsigned int y = 0, outY = 0; y < kResolution[1]; y+=inc, outY++ ) { 434 uint8_t *px = img + outY * stride * 4; 435 mScene.setReadoutPixel(0, y); 436 for (unsigned int x = 0; x < kResolution[0]; x+=inc) { 437 uint32_t rCount, gCount, bCount; 438 // TODO: Perfect demosaicing is a cheat 439 const uint32_t *pixel = mScene.getPixelElectrons(); 440 rCount = pixel[Scene::R] * scale64x; 441 gCount = pixel[Scene::Gr] * scale64x; 442 bCount = pixel[Scene::B] * scale64x; 443 444 *px++ = rCount < 255*64 ? rCount / 64 : 255; 445 *px++ = gCount < 255*64 ? gCount / 64 : 255; 446 *px++ = bCount < 255*64 ? bCount / 64 : 255; 447 *px++ = 255; 448 for (unsigned int j = 1; j < inc; j++) 449 mScene.getPixelElectrons(); 450 } 451 // TODO: Handle this better 452 //simulatedTime += kRowReadoutTime; 453 } 454 ALOGVV("RGBA sensor image captured"); 455 } 456 457 void Sensor::captureRGB(uint8_t *img, uint32_t gain, uint32_t stride) { 458 float totalGain = gain/100.0 * kBaseGainFactor; 459 // In fixed-point math, calculate total scaling from electrons to 8bpp 460 int scale64x = 64 * totalGain * 255 / kMaxRawValue; 461 uint32_t inc = kResolution[0] / stride; 462 463 for (unsigned int y = 0, outY = 0; y < kResolution[1]; y += inc, outY++ ) { 464 mScene.setReadoutPixel(0, y); 465 uint8_t *px = img + outY * stride * 3; 466 for (unsigned int x = 0; x < kResolution[0]; x += inc) { 467 uint32_t rCount, gCount, bCount; 468 // TODO: Perfect demosaicing is a cheat 469 const uint32_t *pixel = mScene.getPixelElectrons(); 470 rCount = pixel[Scene::R] * scale64x; 471 gCount = pixel[Scene::Gr] * scale64x; 472 bCount = pixel[Scene::B] * scale64x; 473 474 *px++ = rCount < 255*64 ? rCount / 64 : 255; 475 *px++ = gCount < 255*64 ? gCount / 64 : 255; 476 *px++ = bCount < 255*64 ? bCount / 64 : 255; 477 for (unsigned int j = 1; j < inc; j++) 478 mScene.getPixelElectrons(); 479 } 480 // TODO: Handle this better 481 //simulatedTime += kRowReadoutTime; 482 } 483 ALOGVV("RGB sensor image captured"); 484 } 485 486 void Sensor::captureNV21(uint8_t *img, uint32_t gain, uint32_t stride) { 487 float totalGain = gain/100.0 * kBaseGainFactor; 488 // In fixed-point math, calculate total scaling from electrons to 8bpp 489 int scale64x = 64 * totalGain * 255 / kMaxRawValue; 490 491 // TODO: Make full-color 492 uint32_t inc = kResolution[0] / stride; 493 uint32_t outH = kResolution[1] / inc; 494 for (unsigned int y = 0, outY = 0, outUV = outH; 495 y < kResolution[1]; y+=inc, outY++, outUV ) { 496 uint8_t *pxY = img + outY * stride; 497 mScene.setReadoutPixel(0,y); 498 for (unsigned int x = 0; x < kResolution[0]; x+=inc) { 499 uint32_t rCount, gCount, bCount; 500 // TODO: Perfect demosaicing is a cheat 501 const uint32_t *pixel = mScene.getPixelElectrons(); 502 rCount = pixel[Scene::R] * scale64x; 503 gCount = pixel[Scene::Gr] * scale64x; 504 bCount = pixel[Scene::B] * scale64x; 505 uint32_t avg = (rCount + gCount + bCount) / 3; 506 *pxY++ = avg < 255*64 ? avg / 64 : 255; 507 for (unsigned int j = 1; j < inc; j++) 508 mScene.getPixelElectrons(); 509 } 510 } 511 for (unsigned int y = 0, outY = outH; y < kResolution[1]/2; y+=inc, outY++) { 512 uint8_t *px = img + outY * stride; 513 for (unsigned int x = 0; x < kResolution[0]; x+=inc) { 514 // UV to neutral 515 *px++ = 128; 516 *px++ = 128; 517 } 518 } 519 ALOGVV("NV21 sensor image captured"); 520 } 521 522 } // namespace android 523