Home | History | Annotate | Download | only in libstagefright 
      1 /*
      2  * Copyright (C) 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "VideoFrameScheduler"
     19 #include <utils/Log.h>
     20 #define ATRACE_TAG ATRACE_TAG_VIDEO
     21 #include <utils/Trace.h>
     22 
     23 #include <sys/time.h>
     24 
     25 #include <binder/IServiceManager.h>
     26 #include <gui/ISurfaceComposer.h>
     27 #include <ui/DisplayStatInfo.h>
     28 
     29 #include <media/stagefright/foundation/ADebug.h>
     30 #include <media/stagefright/foundation/AUtils.h>
     31 #include <media/stagefright/VideoFrameScheduler.h>
     32 
     33 namespace android {
     34 
     35 static const nsecs_t kNanosIn1s = 1000000000;
     36 
     37 template<class T>
     38 static int compare(const T *lhs, const T *rhs) {
     39     if (*lhs < *rhs) {
     40         return -1;
     41     } else if (*lhs > *rhs) {
     42         return 1;
     43     } else {
     44         return 0;
     45     }
     46 }
     47 
     48 /* ======================================================================= */
     49 /*                                   PLL                                   */
     50 /* ======================================================================= */
     51 
     52 static const size_t kMinSamplesToStartPrime = 3;
     53 static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize;
     54 static const size_t kMinSamplesToEstimatePeriod = 3;
     55 static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize;
     56 
     57 static const size_t kPrecision = 12;
     58 static const int64_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
     59 static const int64_t kMultiplesThresholdDiv = 4;            // 25%
     60 static const int64_t kReFitThresholdDiv = 100;              // 1%
     61 static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s;     // 1 sec
     62 static const nsecs_t kMinPeriod = kNanosIn1s / 120;         // 120Hz
     63 static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec
     64 
     65 VideoFrameScheduler::PLL::PLL()
     66     : mPeriod(-1),
     67       mPhase(0),
     68       mPrimed(false),
     69       mSamplesUsedForPriming(0),
     70       mLastTime(-1),
     71       mNumSamples(0) {
     72 }
     73 
     74 void VideoFrameScheduler::PLL::reset(float fps) {
     75     //test();
     76 
     77     mSamplesUsedForPriming = 0;
     78     mLastTime = -1;
     79 
     80     // set up or reset video PLL
     81     if (fps <= 0.f) {
     82         mPeriod = -1;
     83         mPrimed = false;
     84     } else {
     85         ALOGV("reset at %.1f fps", fps);
     86         mPeriod = (nsecs_t)(1e9 / fps + 0.5);
     87         mPrimed = true;
     88     }
     89 
     90     restart();
     91 }
     92 
     93 // reset PLL but keep previous period estimate
     94 void VideoFrameScheduler::PLL::restart() {
     95     mNumSamples = 0;
     96     mPhase = -1;
     97 }
     98 
     99 #if 0
    100 
    101 void VideoFrameScheduler::PLL::test() {
    102     nsecs_t period = kNanosIn1s / 60;
    103     mTimes[0] = 0;
    104     mTimes[1] = period;
    105     mTimes[2] = period * 3;
    106     mTimes[3] = period * 4;
    107     mTimes[4] = period * 7;
    108     mTimes[5] = period * 8;
    109     mTimes[6] = period * 10;
    110     mTimes[7] = period * 12;
    111     mNumSamples = 8;
    112     int64_t a, b, err;
    113     fit(0, period * 12 / 7, 8, &a, &b, &err);
    114     // a = 0.8(5)+
    115     // b = -0.14097(2)+
    116     // err = 0.2750578(703)+
    117     ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
    118             (long long)a, (a / (float)(1 << kPrecision)),
    119             (long long)b, (b / (float)(1 << kPrecision)),
    120             (long long)err, (err / (float)(1 << (kPrecision * 2))));
    121 }
    122 
    123 #endif
    124 
    125 bool VideoFrameScheduler::PLL::fit(
    126         nsecs_t phase, nsecs_t period, size_t numSamplesToUse,
    127         int64_t *a, int64_t *b, int64_t *err) {
    128     if (numSamplesToUse > mNumSamples) {
    129         numSamplesToUse = mNumSamples;
    130     }
    131 
    132     if ((period >> kPrecision) == 0 ) {
    133         ALOGW("Period is 0, or after including precision is 0 - would cause div0, returning");
    134         return false;
    135     }
    136 
    137     int64_t sumX = 0;
    138     int64_t sumXX = 0;
    139     int64_t sumXY = 0;
    140     int64_t sumYY = 0;
    141     int64_t sumY = 0;
    142 
    143     int64_t x = 0; // x usually is in [0..numSamplesToUse)
    144     nsecs_t lastTime;
    145     for (size_t i = 0; i < numSamplesToUse; i++) {
    146         size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize;
    147         nsecs_t time = mTimes[ix];
    148         if (i > 0) {
    149             x += divRound(time - lastTime, period);
    150         }
    151         // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision
    152         //   ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during
    153         //   priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod
    154         //   while we are not refitting.
    155         int64_t y = divRound(time - phase, period >> kPrecision);
    156         sumX += x;
    157         sumY += y;
    158         sumXX += x * x;
    159         sumXY += x * y;
    160         sumYY += y * y;
    161         lastTime = time;
    162     }
    163 
    164     int64_t div   = (int64_t)numSamplesToUse * sumXX - sumX * sumX;
    165     if (div == 0) {
    166         return false;
    167     }
    168 
    169     int64_t a_nom = (int64_t)numSamplesToUse * sumXY - sumX * sumY;
    170     int64_t b_nom = sumXX * sumY            - sumX * sumXY;
    171     *a = divRound(a_nom, div);
    172     *b = divRound(b_nom, div);
    173     // don't use a and b directly as the rounding error is significant
    174     *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div);
    175     ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
    176             numSamplesToUse,
    177             (long long)*a,   (*a / (float)(1 << kPrecision)),
    178             (long long)*b,   (*b / (float)(1 << kPrecision)),
    179             (long long)*err, (*err / (float)(1 << (kPrecision * 2))));
    180     return true;
    181 }
    182 
    183 void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) {
    184     if (numSamplesToUse > mNumSamples) {
    185         numSamplesToUse = mNumSamples;
    186     }
    187     CHECK(numSamplesToUse >= 3);  // must have at least 3 samples
    188 
    189     // estimate video framerate from deltas between timestamps, and
    190     // 2nd order deltas
    191     Vector<nsecs_t> deltas;
    192     nsecs_t lastTime, firstTime;
    193     for (size_t i = 0; i < numSamplesToUse; ++i) {
    194         size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize;
    195         nsecs_t time = mTimes[index];
    196         if (i > 0) {
    197             if (time - lastTime > kMinPeriod) {
    198                 //ALOGV("delta: %lld", (long long)(time - lastTime));
    199                 deltas.push(time - lastTime);
    200             }
    201         } else {
    202             firstTime = time;
    203         }
    204         lastTime = time;
    205     }
    206     deltas.sort(compare<nsecs_t>);
    207     size_t numDeltas = deltas.size();
    208     if (numDeltas > 1) {
    209         nsecs_t deltaMinLimit = max(deltas[0] / kMultiplesThresholdDiv, kMinPeriod);
    210         nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv;
    211         for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) {
    212             if (deltas[i] > deltaMaxLimit) {
    213                 deltas.resize(i);
    214                 numDeltas = i;
    215                 break;
    216             }
    217         }
    218         for (size_t i = 1; i < numDeltas; ++i) {
    219             nsecs_t delta2nd = deltas[i] - deltas[i - 1];
    220             if (delta2nd >= deltaMinLimit) {
    221                 //ALOGV("delta2: %lld", (long long)(delta2nd));
    222                 deltas.push(delta2nd);
    223             }
    224         }
    225     }
    226 
    227     // use the one that yields the best match
    228     int64_t bestScore;
    229     for (size_t i = 0; i < deltas.size(); ++i) {
    230         nsecs_t delta = deltas[i];
    231         int64_t score = 0;
    232 #if 1
    233         // simplest score: number of deltas that are near multiples
    234         size_t matches = 0;
    235         for (size_t j = 0; j < deltas.size(); ++j) {
    236             nsecs_t err = periodicError(deltas[j], delta);
    237             if (err < delta / kMultiplesThresholdDiv) {
    238                 ++matches;
    239             }
    240         }
    241         score = matches;
    242 #if 0
    243         // could be weighed by the (1 - normalized error)
    244         if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
    245             int64_t a, b, err;
    246             fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
    247             err = (1 << (2 * kPrecision)) - err;
    248             score *= max(0, err);
    249         }
    250 #endif
    251 #else
    252         // or use the error as a negative score
    253         if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
    254             int64_t a, b, err;
    255             fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
    256             score = -delta * err;
    257         }
    258 #endif
    259         if (i == 0 || score > bestScore) {
    260             bestScore = score;
    261             mPeriod = delta;
    262             mPhase = firstTime;
    263         }
    264     }
    265     ALOGV("priming[%zu] phase:%lld period:%lld",
    266             numSamplesToUse, (long long)mPhase, (long long)mPeriod);
    267 }
    268 
    269 nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) {
    270     if (mLastTime >= 0
    271             // if time goes backward, or we skipped rendering
    272             && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) {
    273         restart();
    274     }
    275 
    276     mLastTime = time;
    277     mTimes[mNumSamples % kHistorySize] = time;
    278     ++mNumSamples;
    279 
    280     bool doFit = time > mRefitAt;
    281     if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) {
    282         prime(kMinSamplesToStopPrime);
    283         ++mSamplesUsedForPriming;
    284         doFit = true;
    285     }
    286     if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) {
    287         if (mPhase < 0) {
    288             // initialize phase to the current render time
    289             mPhase = time;
    290             doFit = true;
    291         } else if (!doFit) {
    292             int64_t err = periodicError(time - mPhase, mPeriod);
    293             doFit = err > mPeriod / kReFitThresholdDiv;
    294         }
    295 
    296         if (doFit) {
    297             int64_t a, b, err;
    298             if (!fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err)) {
    299                 // samples are not suitable for fitting.  this means they are
    300                 // also not suitable for priming.
    301                 ALOGV("could not fit - keeping old period:%lld", (long long)mPeriod);
    302                 return mPeriod;
    303             }
    304 
    305             mRefitAt = time + kRefitRefreshPeriod;
    306 
    307             mPhase += (mPeriod * b) >> kPrecision;
    308             mPeriod = (mPeriod * a) >> kPrecision;
    309             ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod);
    310 
    311             if (err < kErrorThreshold) {
    312                 if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) {
    313                     mPrimed = true;
    314                 }
    315             } else {
    316                 mPrimed = false;
    317                 mSamplesUsedForPriming = 0;
    318             }
    319         }
    320     }
    321     return mPeriod;
    322 }
    323 
    324 nsecs_t VideoFrameScheduler::PLL::getPeriod() const {
    325     return mPrimed ? mPeriod : 0;
    326 }
    327 
    328 /* ======================================================================= */
    329 /*                             Frame Scheduler                             */
    330 /* ======================================================================= */
    331 
    332 static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60;  // 60Hz
    333 static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s;       // 1 sec
    334 
    335 VideoFrameScheduler::VideoFrameScheduler()
    336     : mVsyncTime(0),
    337       mVsyncPeriod(0),
    338       mVsyncRefreshAt(0),
    339       mLastVsyncTime(-1),
    340       mTimeCorrection(0) {
    341 }
    342 
    343 void VideoFrameScheduler::updateVsync() {
    344     mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod;
    345     mVsyncPeriod = 0;
    346     mVsyncTime = 0;
    347 
    348     // TODO: schedule frames for the destination surface
    349     // For now, surface flinger only schedules frames on the primary display
    350     if (mComposer == NULL) {
    351         String16 name("SurfaceFlinger");
    352         sp<IServiceManager> sm = defaultServiceManager();
    353         mComposer = interface_cast<ISurfaceComposer>(sm->checkService(name));
    354     }
    355     if (mComposer != NULL) {
    356         DisplayStatInfo stats;
    357         status_t res = mComposer->getDisplayStats(NULL /* display */, &stats);
    358         if (res == OK) {
    359             ALOGV("vsync time:%lld period:%lld",
    360                     (long long)stats.vsyncTime, (long long)stats.vsyncPeriod);
    361             mVsyncTime = stats.vsyncTime;
    362             mVsyncPeriod = stats.vsyncPeriod;
    363         } else {
    364             ALOGW("getDisplayStats returned %d", res);
    365         }
    366     } else {
    367         ALOGW("could not get surface mComposer service");
    368     }
    369 }
    370 
    371 void VideoFrameScheduler::init(float videoFps) {
    372     updateVsync();
    373 
    374     mLastVsyncTime = -1;
    375     mTimeCorrection = 0;
    376 
    377     mPll.reset(videoFps);
    378 }
    379 
    380 void VideoFrameScheduler::restart() {
    381     mLastVsyncTime = -1;
    382     mTimeCorrection = 0;
    383 
    384     mPll.restart();
    385 }
    386 
    387 nsecs_t VideoFrameScheduler::getVsyncPeriod() {
    388     if (mVsyncPeriod > 0) {
    389         return mVsyncPeriod;
    390     }
    391     return kDefaultVsyncPeriod;
    392 }
    393 
    394 float VideoFrameScheduler::getFrameRate() {
    395     nsecs_t videoPeriod = mPll.getPeriod();
    396     if (videoPeriod > 0) {
    397         return 1e9 / videoPeriod;
    398     }
    399     return 0.f;
    400 }
    401 
    402 nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) {
    403     nsecs_t origRenderTime = renderTime;
    404 
    405     nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
    406     if (now >= mVsyncRefreshAt) {
    407         updateVsync();
    408     }
    409 
    410     // without VSYNC info, there is nothing to do
    411     if (mVsyncPeriod == 0) {
    412         ALOGV("no vsync: render=%lld", (long long)renderTime);
    413         return renderTime;
    414     }
    415 
    416     // ensure vsync time is well before (corrected) render time
    417     if (mVsyncTime > renderTime - 4 * mVsyncPeriod) {
    418         mVsyncTime -=
    419             ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod;
    420     }
    421 
    422     // Video presentation takes place at the VSYNC _after_ renderTime.  Adjust renderTime
    423     // so this effectively becomes a rounding operation (to the _closest_ VSYNC.)
    424     renderTime -= mVsyncPeriod / 2;
    425 
    426     const nsecs_t videoPeriod = mPll.addSample(origRenderTime);
    427     if (videoPeriod > 0) {
    428         // Smooth out rendering
    429         size_t N = 12;
    430         nsecs_t fiveSixthDev =
    431             abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod)
    432                     / (mVsyncPeriod / 100);
    433         // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz)
    434         if (fiveSixthDev < 12) {  /* 12% / 6 = 2% */
    435             N = 20;
    436         }
    437 
    438         nsecs_t offset = 0;
    439         nsecs_t edgeRemainder = 0;
    440         for (size_t i = 1; i <= N; i++) {
    441             offset +=
    442                 (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod;
    443             edgeRemainder += (videoPeriod * i) % mVsyncPeriod;
    444         }
    445         mTimeCorrection += mVsyncPeriod / 2 - offset / (nsecs_t)N;
    446         renderTime += mTimeCorrection;
    447         nsecs_t correctionLimit = mVsyncPeriod * 3 / 5;
    448         edgeRemainder = abs(edgeRemainder / (nsecs_t)N - mVsyncPeriod / 2);
    449         if (edgeRemainder <= mVsyncPeriod / 3) {
    450             correctionLimit /= 2;
    451         }
    452 
    453         // estimate how many VSYNCs a frame will spend on the display
    454         nsecs_t nextVsyncTime =
    455             renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod);
    456         if (mLastVsyncTime >= 0) {
    457             size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod;
    458             size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod);
    459             bool vsyncsPerFrameAreNearlyConstant =
    460                 periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0;
    461 
    462             if (mTimeCorrection > correctionLimit &&
    463                     (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) {
    464                 // remove a VSYNC
    465                 mTimeCorrection -= mVsyncPeriod / 2;
    466                 renderTime -= mVsyncPeriod / 2;
    467                 nextVsyncTime -= mVsyncPeriod;
    468                 if (vsyncsForLastFrame > 0)
    469                     --vsyncsForLastFrame;
    470             } else if (mTimeCorrection < -correctionLimit &&
    471                     (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) {
    472                 // add a VSYNC
    473                 mTimeCorrection += mVsyncPeriod / 2;
    474                 renderTime += mVsyncPeriod / 2;
    475                 nextVsyncTime += mVsyncPeriod;
    476                 if (vsyncsForLastFrame < ULONG_MAX)
    477                     ++vsyncsForLastFrame;
    478             }
    479             ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame);
    480         }
    481         mLastVsyncTime = nextVsyncTime;
    482     }
    483 
    484     // align rendertime to the center between VSYNC edges
    485     renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod;
    486     renderTime += mVsyncPeriod / 2;
    487     ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime);
    488     ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000);
    489     return renderTime;
    490 }
    491 
    492 void VideoFrameScheduler::release() {
    493     mComposer.clear();
    494 }
    495 
    496 VideoFrameScheduler::~VideoFrameScheduler() {
    497     release();
    498 }
    499 
    500 } // namespace android
    501 
    502