Home | History | Annotate | Download | only in surfaceflinger 
      1 /*
      2  * Copyright 2019 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
     19 #undef LOG_TAG
     20 #define LOG_TAG "RegionSamplingThread"
     21 
     22 #include "RegionSamplingThread.h"
     23 
     24 #include <cutils/properties.h>
     25 #include <gui/IRegionSamplingListener.h>
     26 #include <utils/Trace.h>
     27 #include <string>
     28 
     29 #include <compositionengine/Display.h>
     30 #include <compositionengine/impl/OutputCompositionState.h>
     31 #include "DisplayDevice.h"
     32 #include "Layer.h"
     33 #include "SurfaceFlinger.h"
     34 
     35 namespace android {
     36 using namespace std::chrono_literals;
     37 
     38 template <typename T>
     39 struct SpHash {
     40     size_t operator()(const sp<T>& p) const { return std::hash<T*>()(p.get()); }
     41 };
     42 
     43 constexpr auto lumaSamplingStepTag = "LumaSamplingStep";
     44 enum class samplingStep {
     45     noWorkNeeded,
     46     idleTimerWaiting,
     47     waitForZeroPhase,
     48     waitForSamplePhase,
     49     sample
     50 };
     51 
     52 constexpr auto defaultRegionSamplingOffset = -3ms;
     53 constexpr auto defaultRegionSamplingPeriod = 100ms;
     54 constexpr auto defaultRegionSamplingTimerTimeout = 100ms;
     55 // TODO: (b/127403193) duration to string conversion could probably be constexpr
     56 template <typename Rep, typename Per>
     57 inline std::string toNsString(std::chrono::duration<Rep, Per> t) {
     58     return std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(t).count());
     59 }
     60 
     61 RegionSamplingThread::EnvironmentTimingTunables::EnvironmentTimingTunables() {
     62     char value[PROPERTY_VALUE_MAX] = {};
     63 
     64     property_get("debug.sf.region_sampling_offset_ns", value,
     65                  toNsString(defaultRegionSamplingOffset).c_str());
     66     int const samplingOffsetNsRaw = atoi(value);
     67 
     68     property_get("debug.sf.region_sampling_period_ns", value,
     69                  toNsString(defaultRegionSamplingPeriod).c_str());
     70     int const samplingPeriodNsRaw = atoi(value);
     71 
     72     property_get("debug.sf.region_sampling_timer_timeout_ns", value,
     73                  toNsString(defaultRegionSamplingTimerTimeout).c_str());
     74     int const samplingTimerTimeoutNsRaw = atoi(value);
     75 
     76     if ((samplingPeriodNsRaw < 0) || (samplingTimerTimeoutNsRaw < 0)) {
     77         ALOGW("User-specified sampling tuning options nonsensical. Using defaults");
     78         mSamplingOffset = defaultRegionSamplingOffset;
     79         mSamplingPeriod = defaultRegionSamplingPeriod;
     80         mSamplingTimerTimeout = defaultRegionSamplingTimerTimeout;
     81     } else {
     82         mSamplingOffset = std::chrono::nanoseconds(samplingOffsetNsRaw);
     83         mSamplingPeriod = std::chrono::nanoseconds(samplingPeriodNsRaw);
     84         mSamplingTimerTimeout = std::chrono::nanoseconds(samplingTimerTimeoutNsRaw);
     85     }
     86 }
     87 
     88 struct SamplingOffsetCallback : DispSync::Callback {
     89     SamplingOffsetCallback(RegionSamplingThread& samplingThread, Scheduler& scheduler,
     90                            std::chrono::nanoseconds targetSamplingOffset)
     91           : mRegionSamplingThread(samplingThread),
     92             mScheduler(scheduler),
     93             mTargetSamplingOffset(targetSamplingOffset) {}
     94 
     95     ~SamplingOffsetCallback() { stopVsyncListener(); }
     96 
     97     SamplingOffsetCallback(const SamplingOffsetCallback&) = delete;
     98     SamplingOffsetCallback& operator=(const SamplingOffsetCallback&) = delete;
     99 
    100     void startVsyncListener() {
    101         std::lock_guard lock(mMutex);
    102         if (mVsyncListening) return;
    103 
    104         mPhaseIntervalSetting = Phase::ZERO;
    105         mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
    106             sync.addEventListener("SamplingThreadDispSyncListener", 0, this, mLastCallbackTime);
    107         });
    108         mVsyncListening = true;
    109     }
    110 
    111     void stopVsyncListener() {
    112         std::lock_guard lock(mMutex);
    113         stopVsyncListenerLocked();
    114     }
    115 
    116 private:
    117     void stopVsyncListenerLocked() /*REQUIRES(mMutex)*/ {
    118         if (!mVsyncListening) return;
    119 
    120         mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
    121             sync.removeEventListener(this, &mLastCallbackTime);
    122         });
    123         mVsyncListening = false;
    124     }
    125 
    126     void onDispSyncEvent(nsecs_t /* when */) final {
    127         std::unique_lock<decltype(mMutex)> lock(mMutex);
    128 
    129         if (mPhaseIntervalSetting == Phase::ZERO) {
    130             ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForSamplePhase));
    131             mPhaseIntervalSetting = Phase::SAMPLING;
    132             mScheduler.withPrimaryDispSync([this](android::DispSync& sync) {
    133                 sync.changePhaseOffset(this, mTargetSamplingOffset.count());
    134             });
    135             return;
    136         }
    137 
    138         if (mPhaseIntervalSetting == Phase::SAMPLING) {
    139             mPhaseIntervalSetting = Phase::ZERO;
    140             mScheduler.withPrimaryDispSync(
    141                     [this](android::DispSync& sync) { sync.changePhaseOffset(this, 0); });
    142             stopVsyncListenerLocked();
    143             lock.unlock();
    144             mRegionSamplingThread.notifySamplingOffset();
    145             return;
    146         }
    147     }
    148 
    149     RegionSamplingThread& mRegionSamplingThread;
    150     Scheduler& mScheduler;
    151     const std::chrono::nanoseconds mTargetSamplingOffset;
    152     mutable std::mutex mMutex;
    153     nsecs_t mLastCallbackTime = 0;
    154     enum class Phase {
    155         ZERO,
    156         SAMPLING
    157     } mPhaseIntervalSetting /*GUARDED_BY(mMutex) macro doesnt work with unique_lock?*/
    158             = Phase::ZERO;
    159     bool mVsyncListening /*GUARDED_BY(mMutex)*/ = false;
    160 };
    161 
    162 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler,
    163                                            const TimingTunables& tunables)
    164       : mFlinger(flinger),
    165         mScheduler(scheduler),
    166         mTunables(tunables),
    167         mIdleTimer(std::chrono::duration_cast<std::chrono::milliseconds>(
    168                            mTunables.mSamplingTimerTimeout),
    169                    [] {}, [this] { checkForStaleLuma(); }),
    170         mPhaseCallback(std::make_unique<SamplingOffsetCallback>(*this, mScheduler,
    171                                                                 tunables.mSamplingOffset)),
    172         lastSampleTime(0ns) {
    173     mThread = std::thread([this]() { threadMain(); });
    174     pthread_setname_np(mThread.native_handle(), "RegionSamplingThread");
    175     mIdleTimer.start();
    176 }
    177 
    178 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, Scheduler& scheduler)
    179       : RegionSamplingThread(flinger, scheduler,
    180                              TimingTunables{defaultRegionSamplingOffset,
    181                                             defaultRegionSamplingPeriod,
    182                                             defaultRegionSamplingTimerTimeout}) {}
    183 
    184 RegionSamplingThread::~RegionSamplingThread() {
    185     mIdleTimer.stop();
    186 
    187     {
    188         std::lock_guard lock(mThreadControlMutex);
    189         mRunning = false;
    190         mCondition.notify_one();
    191     }
    192 
    193     if (mThread.joinable()) {
    194         mThread.join();
    195     }
    196 }
    197 
    198 void RegionSamplingThread::addListener(const Rect& samplingArea, const sp<IBinder>& stopLayerHandle,
    199                                        const sp<IRegionSamplingListener>& listener) {
    200     wp<Layer> stopLayer = stopLayerHandle != nullptr
    201             ? static_cast<Layer::Handle*>(stopLayerHandle.get())->owner
    202             : nullptr;
    203 
    204     sp<IBinder> asBinder = IInterface::asBinder(listener);
    205     asBinder->linkToDeath(this);
    206     std::lock_guard lock(mSamplingMutex);
    207     mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayer, listener});
    208 }
    209 
    210 void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) {
    211     std::lock_guard lock(mSamplingMutex);
    212     mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener)));
    213 }
    214 
    215 void RegionSamplingThread::checkForStaleLuma() {
    216     std::lock_guard lock(mThreadControlMutex);
    217 
    218     if (mDiscardedFrames) {
    219         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForZeroPhase));
    220         mDiscardedFrames = false;
    221         mPhaseCallback->startVsyncListener();
    222     }
    223 }
    224 
    225 void RegionSamplingThread::notifyNewContent() {
    226     doSample();
    227 }
    228 
    229 void RegionSamplingThread::notifySamplingOffset() {
    230     doSample();
    231 }
    232 
    233 void RegionSamplingThread::doSample() {
    234     std::lock_guard lock(mThreadControlMutex);
    235     auto now = std::chrono::nanoseconds(systemTime(SYSTEM_TIME_MONOTONIC));
    236     if (lastSampleTime + mTunables.mSamplingPeriod > now) {
    237         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::idleTimerWaiting));
    238         mDiscardedFrames = true;
    239         return;
    240     }
    241 
    242     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::sample));
    243 
    244     mDiscardedFrames = false;
    245     lastSampleTime = now;
    246 
    247     mIdleTimer.reset();
    248     mPhaseCallback->stopVsyncListener();
    249 
    250     mSampleRequested = true;
    251     mCondition.notify_one();
    252 }
    253 
    254 void RegionSamplingThread::binderDied(const wp<IBinder>& who) {
    255     std::lock_guard lock(mSamplingMutex);
    256     mDescriptors.erase(who);
    257 }
    258 
    259 namespace {
    260 // Using Rec. 709 primaries
    261 float getLuma(float r, float g, float b) {
    262     constexpr auto rec709_red_primary = 0.2126f;
    263     constexpr auto rec709_green_primary = 0.7152f;
    264     constexpr auto rec709_blue_primary = 0.0722f;
    265     return rec709_red_primary * r + rec709_green_primary * g + rec709_blue_primary * b;
    266 }
    267 } // anonymous namespace
    268 
    269 float sampleArea(const uint32_t* data, int32_t width, int32_t height, int32_t stride,
    270                  uint32_t orientation, const Rect& sample_area) {
    271     if (!sample_area.isValid() || (sample_area.getWidth() > width) ||
    272         (sample_area.getHeight() > height)) {
    273         ALOGE("invalid sampling region requested");
    274         return 0.0f;
    275     }
    276 
    277     // (b/133849373) ROT_90 screencap images produced upside down
    278     auto area = sample_area;
    279     if (orientation & ui::Transform::ROT_90) {
    280         area.top = height - area.top;
    281         area.bottom = height - area.bottom;
    282         std::swap(area.top, area.bottom);
    283 
    284         area.left = width - area.left;
    285         area.right = width - area.right;
    286         std::swap(area.left, area.right);
    287     }
    288 
    289     std::array<int32_t, 256> brightnessBuckets = {};
    290     const int32_t majoritySampleNum = area.getWidth() * area.getHeight() / 2;
    291 
    292     for (int32_t row = area.top; row < area.bottom; ++row) {
    293         const uint32_t* rowBase = data + row * stride;
    294         for (int32_t column = area.left; column < area.right; ++column) {
    295             uint32_t pixel = rowBase[column];
    296             const float r = (pixel & 0xFF) / 255.0f;
    297             const float g = ((pixel >> 8) & 0xFF) / 255.0f;
    298             const float b = ((pixel >> 16) & 0xFF) / 255.0f;
    299             const uint8_t luma = std::round(getLuma(r, g, b) * 255.0f);
    300             ++brightnessBuckets[luma];
    301             if (brightnessBuckets[luma] > majoritySampleNum) return luma / 255.0f;
    302         }
    303     }
    304 
    305     int32_t accumulated = 0;
    306     size_t bucket = 0;
    307     for (; bucket < brightnessBuckets.size(); bucket++) {
    308         accumulated += brightnessBuckets[bucket];
    309         if (accumulated > majoritySampleNum) break;
    310     }
    311 
    312     return bucket / 255.0f;
    313 }
    314 
    315 std::vector<float> RegionSamplingThread::sampleBuffer(
    316         const sp<GraphicBuffer>& buffer, const Point& leftTop,
    317         const std::vector<RegionSamplingThread::Descriptor>& descriptors, uint32_t orientation) {
    318     void* data_raw = nullptr;
    319     buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw);
    320     std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw),
    321                                    [&buffer](auto) { buffer->unlock(); });
    322     if (!data) return {};
    323 
    324     const int32_t width = buffer->getWidth();
    325     const int32_t height = buffer->getHeight();
    326     const int32_t stride = buffer->getStride();
    327     std::vector<float> lumas(descriptors.size());
    328     std::transform(descriptors.begin(), descriptors.end(), lumas.begin(),
    329                    [&](auto const& descriptor) {
    330                        return sampleArea(data.get(), width, height, stride, orientation,
    331                                          descriptor.area - leftTop);
    332                    });
    333     return lumas;
    334 }
    335 
    336 void RegionSamplingThread::captureSample() {
    337     ATRACE_CALL();
    338     std::lock_guard lock(mSamplingMutex);
    339 
    340     if (mDescriptors.empty()) {
    341         return;
    342     }
    343 
    344     const auto device = mFlinger.getDefaultDisplayDevice();
    345     const auto display = device->getCompositionDisplay();
    346     const auto state = display->getState();
    347     const auto orientation = static_cast<ui::Transform::orientation_flags>(state.orientation);
    348 
    349     std::vector<RegionSamplingThread::Descriptor> descriptors;
    350     Region sampleRegion;
    351     for (const auto& [listener, descriptor] : mDescriptors) {
    352         sampleRegion.orSelf(descriptor.area);
    353         descriptors.emplace_back(descriptor);
    354     }
    355 
    356     const Rect sampledArea = sampleRegion.bounds();
    357 
    358     auto dx = 0;
    359     auto dy = 0;
    360     switch (orientation) {
    361         case ui::Transform::ROT_90:
    362             dx = device->getWidth();
    363             break;
    364         case ui::Transform::ROT_180:
    365             dx = device->getWidth();
    366             dy = device->getHeight();
    367             break;
    368         case ui::Transform::ROT_270:
    369             dy = device->getHeight();
    370             break;
    371         default:
    372             break;
    373     }
    374 
    375     ui::Transform t(orientation);
    376     auto screencapRegion = t.transform(sampleRegion);
    377     screencapRegion = screencapRegion.translate(dx, dy);
    378     DisplayRenderArea renderArea(device, screencapRegion.bounds(), sampledArea.getWidth(),
    379                                  sampledArea.getHeight(), ui::Dataspace::V0_SRGB, orientation);
    380 
    381     std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners;
    382 
    383     auto traverseLayers = [&](const LayerVector::Visitor& visitor) {
    384         bool stopLayerFound = false;
    385         auto filterVisitor = [&](Layer* layer) {
    386             // We don't want to capture any layers beyond the stop layer
    387             if (stopLayerFound) return;
    388 
    389             // Likewise if we just found a stop layer, set the flag and abort
    390             for (const auto& [area, stopLayer, listener] : descriptors) {
    391                 if (layer == stopLayer.promote().get()) {
    392                     stopLayerFound = true;
    393                     return;
    394                 }
    395             }
    396 
    397             // Compute the layer's position on the screen
    398             const Rect bounds = Rect(layer->getBounds());
    399             const ui::Transform transform = layer->getTransform();
    400             constexpr bool roundOutwards = true;
    401             Rect transformed = transform.transform(bounds, roundOutwards);
    402 
    403             // If this layer doesn't intersect with the larger sampledArea, skip capturing it
    404             Rect ignore;
    405             if (!transformed.intersect(sampledArea, &ignore)) return;
    406 
    407             // If the layer doesn't intersect a sampling area, skip capturing it
    408             bool intersectsAnyArea = false;
    409             for (const auto& [area, stopLayer, listener] : descriptors) {
    410                 if (transformed.intersect(area, &ignore)) {
    411                     intersectsAnyArea = true;
    412                     listeners.insert(listener);
    413                 }
    414             }
    415             if (!intersectsAnyArea) return;
    416 
    417             ALOGV("Traversing [%s] [%d, %d, %d, %d]", layer->getName().string(), bounds.left,
    418                   bounds.top, bounds.right, bounds.bottom);
    419             visitor(layer);
    420         };
    421         mFlinger.traverseLayersInDisplay(device, filterVisitor);
    422     };
    423 
    424     sp<GraphicBuffer> buffer = nullptr;
    425     if (mCachedBuffer && mCachedBuffer->getWidth() == sampledArea.getWidth() &&
    426         mCachedBuffer->getHeight() == sampledArea.getHeight()) {
    427         buffer = mCachedBuffer;
    428     } else {
    429         const uint32_t usage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER;
    430         buffer = new GraphicBuffer(sampledArea.getWidth(), sampledArea.getHeight(),
    431                                    PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread");
    432     }
    433 
    434     bool ignored;
    435     mFlinger.captureScreenCommon(renderArea, traverseLayers, buffer, false, ignored);
    436 
    437     std::vector<Descriptor> activeDescriptors;
    438     for (const auto& descriptor : descriptors) {
    439         if (listeners.count(descriptor.listener) != 0) {
    440             activeDescriptors.emplace_back(descriptor);
    441         }
    442     }
    443 
    444     ALOGV("Sampling %zu descriptors", activeDescriptors.size());
    445     std::vector<float> lumas =
    446             sampleBuffer(buffer, sampledArea.leftTop(), activeDescriptors, orientation);
    447     if (lumas.size() != activeDescriptors.size()) {
    448         ALOGW("collected %zu median luma values for %zu descriptors", lumas.size(),
    449               activeDescriptors.size());
    450         return;
    451     }
    452 
    453     for (size_t d = 0; d < activeDescriptors.size(); ++d) {
    454         activeDescriptors[d].listener->onSampleCollected(lumas[d]);
    455     }
    456 
    457     // Extend the lifetime of mCachedBuffer from the previous frame to here to ensure that:
    458     // 1) The region sampling thread is the last owner of the buffer, and the freeing of the buffer
    459     // happens in this thread, as opposed to the main thread.
    460     // 2) The listener(s) receive their notifications prior to freeing the buffer.
    461     mCachedBuffer = buffer;
    462     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::noWorkNeeded));
    463 }
    464 
    465 // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations.
    466 void RegionSamplingThread::threadMain() NO_THREAD_SAFETY_ANALYSIS {
    467     std::unique_lock<std::mutex> lock(mThreadControlMutex);
    468     while (mRunning) {
    469         if (mSampleRequested) {
    470             mSampleRequested = false;
    471             lock.unlock();
    472             captureSample();
    473             lock.lock();
    474         }
    475         mCondition.wait(lock, [this]() REQUIRES(mThreadControlMutex) {
    476             return mSampleRequested || !mRunning;
    477         });
    478     }
    479 }
    480 
    481 } // namespace android
    482