Home | History | Annotate | Download | only in fake-pipeline2 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 //#define LOG_NNDEBUG 0
     19 #define LOG_TAG "EmulatedCamera2_Sensor"
     20 
     21 #ifdef LOG_NNDEBUG
     22 #define ALOGVV(...) ALOGV(__VA_ARGS__)
     23 #else
     24 #define ALOGVV(...) ((void)0)
     25 #endif
     26 
     27 #include <utils/Log.h>
     28 
     29 #include "../EmulatedFakeCamera2.h"
     30 #include "Sensor.h"
     31 #include <cmath>
     32 #include <cstdlib>
     33 #include "system/camera_metadata.h"
     34 
     35 namespace android {
     36 
     37 const unsigned int Sensor::kResolution[2]  = {640, 480};
     38 
     39 const nsecs_t Sensor::kExposureTimeRange[2] =
     40     {1000L, 30000000000L} ; // 1 us - 30 sec
     41 const nsecs_t Sensor::kFrameDurationRange[2] =
     42     {33331760L, 30000000000L}; // ~1/30 s - 30 sec
     43 const nsecs_t Sensor::kMinVerticalBlank = 10000L;
     44 
     45 const uint8_t Sensor::kColorFilterArrangement =
     46     ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_RGGB;
     47 
     48 // Output image data characteristics
     49 const uint32_t Sensor::kMaxRawValue = 4000;
     50 const uint32_t Sensor::kBlackLevel  = 1000;
     51 
     52 // Sensor sensitivity
     53 const float Sensor::kSaturationVoltage      = 0.520f;
     54 const uint32_t Sensor::kSaturationElectrons = 2000;
     55 const float Sensor::kVoltsPerLuxSecond      = 0.100f;
     56 
     57 const float Sensor::kElectronsPerLuxSecond =
     58         Sensor::kSaturationElectrons / Sensor::kSaturationVoltage
     59         * Sensor::kVoltsPerLuxSecond;
     60 
     61 const float Sensor::kBaseGainFactor = (float)Sensor::kMaxRawValue /
     62             Sensor::kSaturationElectrons;
     63 
     64 const float Sensor::kReadNoiseStddevBeforeGain = 1.177; // in electrons
     65 const float Sensor::kReadNoiseStddevAfterGain =  2.100; // in digital counts
     66 const float Sensor::kReadNoiseVarBeforeGain =
     67             Sensor::kReadNoiseStddevBeforeGain *
     68             Sensor::kReadNoiseStddevBeforeGain;
     69 const float Sensor::kReadNoiseVarAfterGain =
     70             Sensor::kReadNoiseStddevAfterGain *
     71             Sensor::kReadNoiseStddevAfterGain;
     72 
     73 // While each row has to read out, reset, and then expose, the (reset +
     74 // expose) sequence can be overlapped by other row readouts, so the final
     75 // minimum frame duration is purely a function of row readout time, at least
     76 // if there's a reasonable number of rows.
     77 const nsecs_t Sensor::kRowReadoutTime =
     78             Sensor::kFrameDurationRange[0] / Sensor::kResolution[1];
     79 
     80 const uint32_t Sensor::kAvailableSensitivities[5] =
     81     {100, 200, 400, 800, 1600};
     82 const uint32_t Sensor::kDefaultSensitivity = 100;
     83 
     84 /** A few utility functions for math, normal distributions */
     85 
     86 // Take advantage of IEEE floating-point format to calculate an approximate
     87 // square root. Accurate to within +-3.6%
     88 float sqrtf_approx(float r) {
     89     // Modifier is based on IEEE floating-point representation; the
     90     // manipulations boil down to finding approximate log2, dividing by two, and
     91     // then inverting the log2. A bias is added to make the relative error
     92     // symmetric about the real answer.
     93     const int32_t modifier = 0x1FBB4000;
     94 
     95     int32_t r_i = *(int32_t*)(&r);
     96     r_i = (r_i >> 1) + modifier;
     97 
     98     return *(float*)(&r_i);
     99 }
    100 
    101 
    102 
    103 Sensor::Sensor():
    104         Thread(false),
    105         mGotVSync(false),
    106         mExposureTime(kFrameDurationRange[0]-kMinVerticalBlank),
    107         mFrameDuration(kFrameDurationRange[0]),
    108         mGainFactor(kDefaultSensitivity),
    109         mNextBuffers(NULL),
    110         mFrameNumber(0),
    111         mCapturedBuffers(NULL),
    112         mListener(NULL),
    113         mScene(kResolution[0], kResolution[1], kElectronsPerLuxSecond)
    114 {
    115 
    116 }
    117 
    118 Sensor::~Sensor() {
    119     shutDown();
    120 }
    121 
    122 status_t Sensor::startUp() {
    123     ALOGV("%s: E", __FUNCTION__);
    124 
    125     int res;
    126     mCapturedBuffers = NULL;
    127     res = run("EmulatedFakeCamera2::Sensor",
    128             ANDROID_PRIORITY_URGENT_DISPLAY);
    129 
    130     if (res != OK) {
    131         ALOGE("Unable to start up sensor capture thread: %d", res);
    132     }
    133     return res;
    134 }
    135 
    136 status_t Sensor::shutDown() {
    137     ALOGV("%s: E", __FUNCTION__);
    138 
    139     int res;
    140     res = requestExitAndWait();
    141     if (res != OK) {
    142         ALOGE("Unable to shut down sensor capture thread: %d", res);
    143     }
    144     return res;
    145 }
    146 
    147 Scene &Sensor::getScene() {
    148     return mScene;
    149 }
    150 
    151 void Sensor::setExposureTime(uint64_t ns) {
    152     Mutex::Autolock lock(mControlMutex);
    153     ALOGVV("Exposure set to %f", ns/1000000.f);
    154     mExposureTime = ns;
    155 }
    156 
    157 void Sensor::setFrameDuration(uint64_t ns) {
    158     Mutex::Autolock lock(mControlMutex);
    159     ALOGVV("Frame duration set to %f", ns/1000000.f);
    160     mFrameDuration = ns;
    161 }
    162 
    163 void Sensor::setSensitivity(uint32_t gain) {
    164     Mutex::Autolock lock(mControlMutex);
    165     ALOGVV("Gain set to %d", gain);
    166     mGainFactor = gain;
    167 }
    168 
    169 void Sensor::setDestinationBuffers(Buffers *buffers) {
    170     Mutex::Autolock lock(mControlMutex);
    171     mNextBuffers = buffers;
    172 }
    173 
    174 void Sensor::setFrameNumber(uint32_t frameNumber) {
    175     Mutex::Autolock lock(mControlMutex);
    176     mFrameNumber = frameNumber;
    177 }
    178 
    179 bool Sensor::waitForVSync(nsecs_t reltime) {
    180     int res;
    181     Mutex::Autolock lock(mControlMutex);
    182 
    183     mGotVSync = false;
    184     res = mVSync.waitRelative(mControlMutex, reltime);
    185     if (res != OK && res != TIMED_OUT) {
    186         ALOGE("%s: Error waiting for VSync signal: %d", __FUNCTION__, res);
    187         return false;
    188     }
    189     return mGotVSync;
    190 }
    191 
    192 bool Sensor::waitForNewFrame(nsecs_t reltime,
    193         nsecs_t *captureTime) {
    194     Mutex::Autolock lock(mReadoutMutex);
    195     uint8_t *ret;
    196     if (mCapturedBuffers == NULL) {
    197         int res;
    198         res = mReadoutAvailable.waitRelative(mReadoutMutex, reltime);
    199         if (res == TIMED_OUT) {
    200             return false;
    201         } else if (res != OK || mCapturedBuffers == NULL) {
    202             ALOGE("Error waiting for sensor readout signal: %d", res);
    203             return false;
    204         }
    205     } else {
    206         mReadoutComplete.signal();
    207     }
    208 
    209     *captureTime = mCaptureTime;
    210     mCapturedBuffers = NULL;
    211     return true;
    212 }
    213 
    214 Sensor::SensorListener::~SensorListener() {
    215 }
    216 
    217 void Sensor::setSensorListener(SensorListener *listener) {
    218     Mutex::Autolock lock(mControlMutex);
    219     mListener = listener;
    220 }
    221 
    222 status_t Sensor::readyToRun() {
    223     ALOGV("Starting up sensor thread");
    224     mStartupTime = systemTime();
    225     mNextCaptureTime = 0;
    226     mNextCapturedBuffers = NULL;
    227     return OK;
    228 }
    229 
    230 bool Sensor::threadLoop() {
    231     /**
    232      * Sensor capture operation main loop.
    233      *
    234      * Stages are out-of-order relative to a single frame's processing, but
    235      * in-order in time.
    236      */
    237 
    238     /**
    239      * Stage 1: Read in latest control parameters
    240      */
    241     uint64_t exposureDuration;
    242     uint64_t frameDuration;
    243     uint32_t gain;
    244     Buffers *nextBuffers;
    245     uint32_t frameNumber;
    246     SensorListener *listener = NULL;
    247     {
    248         Mutex::Autolock lock(mControlMutex);
    249         exposureDuration = mExposureTime;
    250         frameDuration    = mFrameDuration;
    251         gain             = mGainFactor;
    252         nextBuffers      = mNextBuffers;
    253         frameNumber      = mFrameNumber;
    254         listener         = mListener;
    255         // Don't reuse a buffer set
    256         mNextBuffers = NULL;
    257 
    258         // Signal VSync for start of readout
    259         ALOGVV("Sensor VSync");
    260         mGotVSync = true;
    261         mVSync.signal();
    262     }
    263 
    264     /**
    265      * Stage 3: Read out latest captured image
    266      */
    267 
    268     Buffers *capturedBuffers = NULL;
    269     nsecs_t captureTime = 0;
    270 
    271     nsecs_t startRealTime  = systemTime();
    272     // Stagefright cares about system time for timestamps, so base simulated
    273     // time on that.
    274     nsecs_t simulatedTime    = startRealTime;
    275     nsecs_t frameEndRealTime = startRealTime + frameDuration;
    276     nsecs_t frameReadoutEndRealTime = startRealTime +
    277             kRowReadoutTime * kResolution[1];
    278 
    279     if (mNextCapturedBuffers != NULL) {
    280         ALOGVV("Sensor starting readout");
    281         // Pretend we're doing readout now; will signal once enough time has elapsed
    282         capturedBuffers = mNextCapturedBuffers;
    283         captureTime    = mNextCaptureTime;
    284     }
    285     simulatedTime += kRowReadoutTime + kMinVerticalBlank;
    286 
    287     // TODO: Move this signal to another thread to simulate readout
    288     // time properly
    289     if (capturedBuffers != NULL) {
    290         ALOGVV("Sensor readout complete");
    291         Mutex::Autolock lock(mReadoutMutex);
    292         if (mCapturedBuffers != NULL) {
    293             ALOGV("Waiting for readout thread to catch up!");
    294             mReadoutComplete.wait(mReadoutMutex);
    295         }
    296 
    297         mCapturedBuffers = capturedBuffers;
    298         mCaptureTime = captureTime;
    299         mReadoutAvailable.signal();
    300         capturedBuffers = NULL;
    301     }
    302 
    303     /**
    304      * Stage 2: Capture new image
    305      */
    306     mNextCaptureTime = simulatedTime;
    307     mNextCapturedBuffers = nextBuffers;
    308 
    309     if (mNextCapturedBuffers != NULL) {
    310         if (listener != NULL) {
    311             listener->onSensorEvent(frameNumber, SensorListener::EXPOSURE_START,
    312                     mNextCaptureTime);
    313         }
    314         ALOGVV("Starting next capture: Exposure: %f ms, gain: %d",
    315                 (float)exposureDuration/1e6, gain);
    316         mScene.setExposureDuration((float)exposureDuration/1e9);
    317         mScene.calculateScene(mNextCaptureTime);
    318 
    319         // Might be adding more buffers, so size isn't constant
    320         for (size_t i = 0; i < mNextCapturedBuffers->size(); i++) {
    321             const StreamBuffer &b = (*mNextCapturedBuffers)[i];
    322             ALOGVV("Sensor capturing buffer %d: stream %d,"
    323                     " %d x %d, format %x, stride %d, buf %p, img %p",
    324                     i, b.streamId, b.width, b.height, b.format, b.stride,
    325                     b.buffer, b.img);
    326             switch(b.format) {
    327                 case HAL_PIXEL_FORMAT_RAW_SENSOR:
    328                     captureRaw(b.img, gain, b.stride);
    329                     break;
    330                 case HAL_PIXEL_FORMAT_RGB_888:
    331                     captureRGB(b.img, gain, b.stride);
    332                     break;
    333                 case HAL_PIXEL_FORMAT_RGBA_8888:
    334                     captureRGBA(b.img, gain, b.stride);
    335                     break;
    336                 case HAL_PIXEL_FORMAT_BLOB:
    337                     // Add auxillary buffer of the right size
    338                     // Assumes only one BLOB (JPEG) buffer in
    339                     // mNextCapturedBuffers
    340                     StreamBuffer bAux;
    341                     bAux.streamId = 0;
    342                     bAux.width = b.width;
    343                     bAux.height = b.height;
    344                     bAux.format = HAL_PIXEL_FORMAT_RGB_888;
    345                     bAux.stride = b.width;
    346                     bAux.buffer = NULL;
    347                     // TODO: Reuse these
    348                     bAux.img = new uint8_t[b.width * b.height * 3];
    349                     mNextCapturedBuffers->push_back(bAux);
    350                     break;
    351                 case HAL_PIXEL_FORMAT_YCrCb_420_SP:
    352                     captureNV21(b.img, gain, b.stride);
    353                     break;
    354                 case HAL_PIXEL_FORMAT_YV12:
    355                     // TODO:
    356                     ALOGE("%s: Format %x is TODO", __FUNCTION__, b.format);
    357                     break;
    358                 default:
    359                     ALOGE("%s: Unknown format %x, no output", __FUNCTION__,
    360                             b.format);
    361                     break;
    362             }
    363         }
    364     }
    365 
    366     ALOGVV("Sensor vertical blanking interval");
    367     nsecs_t workDoneRealTime = systemTime();
    368     const nsecs_t timeAccuracy = 2e6; // 2 ms of imprecision is ok
    369     if (workDoneRealTime < frameEndRealTime - timeAccuracy) {
    370         timespec t;
    371         t.tv_sec = (frameEndRealTime - workDoneRealTime)  / 1000000000L;
    372         t.tv_nsec = (frameEndRealTime - workDoneRealTime) % 1000000000L;
    373 
    374         int ret;
    375         do {
    376             ret = nanosleep(&t, &t);
    377         } while (ret != 0);
    378     }
    379     nsecs_t endRealTime = systemTime();
    380     ALOGVV("Frame cycle took %d ms, target %d ms",
    381             (int)((endRealTime - startRealTime)/1000000),
    382             (int)(frameDuration / 1000000));
    383     return true;
    384 };
    385 
    386 void Sensor::captureRaw(uint8_t *img, uint32_t gain, uint32_t stride) {
    387     float totalGain = gain/100.0 * kBaseGainFactor;
    388     float noiseVarGain =  totalGain * totalGain;
    389     float readNoiseVar = kReadNoiseVarBeforeGain * noiseVarGain
    390             + kReadNoiseVarAfterGain;
    391 
    392     int bayerSelect[4] = {Scene::R, Scene::Gr, Scene::Gb, Scene::B}; // RGGB
    393     mScene.setReadoutPixel(0,0);
    394     for (unsigned int y = 0; y < kResolution[1]; y++ ) {
    395         int *bayerRow = bayerSelect + (y & 0x1) * 2;
    396         uint16_t *px = (uint16_t*)img + y * stride;
    397         for (unsigned int x = 0; x < kResolution[0]; x++) {
    398             uint32_t electronCount;
    399             electronCount = mScene.getPixelElectrons()[bayerRow[x & 0x1]];
    400 
    401             // TODO: Better pixel saturation curve?
    402             electronCount = (electronCount < kSaturationElectrons) ?
    403                     electronCount : kSaturationElectrons;
    404 
    405             // TODO: Better A/D saturation curve?
    406             uint16_t rawCount = electronCount * totalGain;
    407             rawCount = (rawCount < kMaxRawValue) ? rawCount : kMaxRawValue;
    408 
    409             // Calculate noise value
    410             // TODO: Use more-correct Gaussian instead of uniform noise
    411             float photonNoiseVar = electronCount * noiseVarGain;
    412             float noiseStddev = sqrtf_approx(readNoiseVar + photonNoiseVar);
    413             // Scaled to roughly match gaussian/uniform noise stddev
    414             float noiseSample = std::rand() * (2.5 / (1.0 + RAND_MAX)) - 1.25;
    415 
    416             rawCount += kBlackLevel;
    417             rawCount += noiseStddev * noiseSample;
    418 
    419             *px++ = rawCount;
    420         }
    421         // TODO: Handle this better
    422         //simulatedTime += kRowReadoutTime;
    423     }
    424     ALOGVV("Raw sensor image captured");
    425 }
    426 
    427 void Sensor::captureRGBA(uint8_t *img, uint32_t gain, uint32_t stride) {
    428     float totalGain = gain/100.0 * kBaseGainFactor;
    429     // In fixed-point math, calculate total scaling from electrons to 8bpp
    430     int scale64x = 64 * totalGain * 255 / kMaxRawValue;
    431     uint32_t inc = kResolution[0] / stride;
    432 
    433     for (unsigned int y = 0, outY = 0; y < kResolution[1]; y+=inc, outY++ ) {
    434         uint8_t *px = img + outY * stride * 4;
    435         mScene.setReadoutPixel(0, y);
    436         for (unsigned int x = 0; x < kResolution[0]; x+=inc) {
    437             uint32_t rCount, gCount, bCount;
    438             // TODO: Perfect demosaicing is a cheat
    439             const uint32_t *pixel = mScene.getPixelElectrons();
    440             rCount = pixel[Scene::R]  * scale64x;
    441             gCount = pixel[Scene::Gr] * scale64x;
    442             bCount = pixel[Scene::B]  * scale64x;
    443 
    444             *px++ = rCount < 255*64 ? rCount / 64 : 255;
    445             *px++ = gCount < 255*64 ? gCount / 64 : 255;
    446             *px++ = bCount < 255*64 ? bCount / 64 : 255;
    447             *px++ = 255;
    448             for (unsigned int j = 1; j < inc; j++)
    449                 mScene.getPixelElectrons();
    450         }
    451         // TODO: Handle this better
    452         //simulatedTime += kRowReadoutTime;
    453     }
    454     ALOGVV("RGBA sensor image captured");
    455 }
    456 
    457 void Sensor::captureRGB(uint8_t *img, uint32_t gain, uint32_t stride) {
    458     float totalGain = gain/100.0 * kBaseGainFactor;
    459     // In fixed-point math, calculate total scaling from electrons to 8bpp
    460     int scale64x = 64 * totalGain * 255 / kMaxRawValue;
    461     uint32_t inc = kResolution[0] / stride;
    462 
    463     for (unsigned int y = 0, outY = 0; y < kResolution[1]; y += inc, outY++ ) {
    464         mScene.setReadoutPixel(0, y);
    465         uint8_t *px = img + outY * stride * 3;
    466         for (unsigned int x = 0; x < kResolution[0]; x += inc) {
    467             uint32_t rCount, gCount, bCount;
    468             // TODO: Perfect demosaicing is a cheat
    469             const uint32_t *pixel = mScene.getPixelElectrons();
    470             rCount = pixel[Scene::R]  * scale64x;
    471             gCount = pixel[Scene::Gr] * scale64x;
    472             bCount = pixel[Scene::B]  * scale64x;
    473 
    474             *px++ = rCount < 255*64 ? rCount / 64 : 255;
    475             *px++ = gCount < 255*64 ? gCount / 64 : 255;
    476             *px++ = bCount < 255*64 ? bCount / 64 : 255;
    477             for (unsigned int j = 1; j < inc; j++)
    478                 mScene.getPixelElectrons();
    479         }
    480         // TODO: Handle this better
    481         //simulatedTime += kRowReadoutTime;
    482     }
    483     ALOGVV("RGB sensor image captured");
    484 }
    485 
    486 void Sensor::captureNV21(uint8_t *img, uint32_t gain, uint32_t stride) {
    487     float totalGain = gain/100.0 * kBaseGainFactor;
    488     // In fixed-point math, calculate total scaling from electrons to 8bpp
    489     int scale64x = 64 * totalGain * 255 / kMaxRawValue;
    490 
    491     // TODO: Make full-color
    492     uint32_t inc = kResolution[0] / stride;
    493     uint32_t outH = kResolution[1] / inc;
    494     for (unsigned int y = 0, outY = 0, outUV = outH;
    495          y < kResolution[1]; y+=inc, outY++, outUV ) {
    496         uint8_t *pxY = img + outY * stride;
    497         mScene.setReadoutPixel(0,y);
    498         for (unsigned int x = 0; x < kResolution[0]; x+=inc) {
    499             uint32_t rCount, gCount, bCount;
    500             // TODO: Perfect demosaicing is a cheat
    501             const uint32_t *pixel = mScene.getPixelElectrons();
    502             rCount = pixel[Scene::R]  * scale64x;
    503             gCount = pixel[Scene::Gr] * scale64x;
    504             bCount = pixel[Scene::B]  * scale64x;
    505             uint32_t avg = (rCount + gCount + bCount) / 3;
    506             *pxY++ = avg < 255*64 ? avg / 64 : 255;
    507             for (unsigned int j = 1; j < inc; j++)
    508                 mScene.getPixelElectrons();
    509         }
    510     }
    511     for (unsigned int y = 0, outY = outH; y < kResolution[1]/2; y+=inc, outY++) {
    512         uint8_t *px = img + outY * stride;
    513         for (unsigned int x = 0; x < kResolution[0]; x+=inc) {
    514             // UV to neutral
    515             *px++ = 128;
    516             *px++ = 128;
    517         }
    518     }
    519     ALOGVV("NV21 sensor image captured");
    520 }
    521 
    522 } // namespace android
    523