Home | History | Annotate | Download | only in metrics 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #define DEBUG false  // STOPSHIP if true
     18 #include "Log.h"
     19 
     20 #include "CountMetricProducer.h"
     21 #include "guardrail/StatsdStats.h"
     22 #include "stats_util.h"
     23 #include "stats_log_util.h"
     24 
     25 #include <limits.h>
     26 #include <stdlib.h>
     27 
     28 using android::util::FIELD_COUNT_REPEATED;
     29 using android::util::FIELD_TYPE_BOOL;
     30 using android::util::FIELD_TYPE_FLOAT;
     31 using android::util::FIELD_TYPE_INT32;
     32 using android::util::FIELD_TYPE_INT64;
     33 using android::util::FIELD_TYPE_MESSAGE;
     34 using android::util::FIELD_TYPE_STRING;
     35 using android::util::ProtoOutputStream;
     36 using std::map;
     37 using std::string;
     38 using std::unordered_map;
     39 using std::vector;
     40 
     41 namespace android {
     42 namespace os {
     43 namespace statsd {
     44 
     45 // for StatsLogReport
     46 const int FIELD_ID_ID = 1;
     47 const int FIELD_ID_COUNT_METRICS = 5;
     48 const int FIELD_ID_TIME_BASE = 9;
     49 const int FIELD_ID_BUCKET_SIZE = 10;
     50 const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
     51 const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
     52 const int FIELD_ID_IS_ACTIVE = 14;
     53 
     54 // for CountMetricDataWrapper
     55 const int FIELD_ID_DATA = 1;
     56 // for CountMetricData
     57 const int FIELD_ID_DIMENSION_IN_WHAT = 1;
     58 const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
     59 const int FIELD_ID_BUCKET_INFO = 3;
     60 const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
     61 const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
     62 // for CountBucketInfo
     63 const int FIELD_ID_COUNT = 3;
     64 const int FIELD_ID_BUCKET_NUM = 4;
     65 const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
     66 const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
     67 
     68 CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric& metric,
     69                                          const int conditionIndex,
     70                                          const sp<ConditionWizard>& wizard,
     71                                          const int64_t timeBaseNs, const int64_t startTimeNs)
     72     : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard) {
     73     if (metric.has_bucket()) {
     74         mBucketSizeNs =
     75                 TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket()) * 1000000;
     76     } else {
     77         mBucketSizeNs = LLONG_MAX;
     78     }
     79 
     80     if (metric.has_dimensions_in_what()) {
     81         translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
     82         mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
     83     }
     84 
     85     mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
     86             HasPositionALL(metric.dimensions_in_condition());
     87 
     88     if (metric.has_dimensions_in_condition()) {
     89         translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition);
     90     }
     91 
     92     if (metric.links().size() > 0) {
     93         for (const auto& link : metric.links()) {
     94             Metric2Condition mc;
     95             mc.conditionId = link.condition();
     96             translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
     97             translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
     98             mMetric2ConditionLinks.push_back(mc);
     99         }
    100         mConditionSliced = true;
    101     }
    102 
    103     mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0);
    104 
    105     flushIfNeededLocked(startTimeNs);
    106     // Adjust start for partial bucket
    107     mCurrentBucketStartTimeNs = startTimeNs;
    108 
    109     VLOG("metric %lld created. bucket size %lld start_time: %lld", (long long)metric.id(),
    110          (long long)mBucketSizeNs, (long long)mTimeBaseNs);
    111 }
    112 
    113 CountMetricProducer::~CountMetricProducer() {
    114     VLOG("~CountMetricProducer() called");
    115 }
    116 
    117 void CountMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
    118     if (mCurrentSlicedCounter == nullptr ||
    119         mCurrentSlicedCounter->size() == 0) {
    120         return;
    121     }
    122 
    123     fprintf(out, "CountMetric %lld dimension size %lu\n", (long long)mMetricId,
    124             (unsigned long)mCurrentSlicedCounter->size());
    125     if (verbose) {
    126         for (const auto& it : *mCurrentSlicedCounter) {
    127             fprintf(out, "\t(what)%s\t(condition)%s  %lld\n",
    128                 it.first.getDimensionKeyInWhat().toString().c_str(),
    129                 it.first.getDimensionKeyInCondition().toString().c_str(),
    130                 (unsigned long long)it.second);
    131         }
    132     }
    133 }
    134 
    135 void CountMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
    136                                                            const int64_t eventTime) {
    137     VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
    138 }
    139 
    140 
    141 void CountMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
    142     mPastBuckets.clear();
    143 }
    144 
    145 void CountMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
    146                                              const bool include_current_partial_bucket,
    147                                              const bool erase_data,
    148                                              const DumpLatency dumpLatency,
    149                                              std::set<string> *str_set,
    150                                              ProtoOutputStream* protoOutput) {
    151     if (include_current_partial_bucket) {
    152         flushLocked(dumpTimeNs);
    153     } else {
    154         flushIfNeededLocked(dumpTimeNs);
    155     }
    156     protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
    157     protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked());
    158 
    159 
    160     if (mPastBuckets.empty()) {
    161         return;
    162     }
    163     protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
    164     protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
    165 
    166     // Fills the dimension path if not slicing by ALL.
    167     if (!mSliceByPositionALL) {
    168         if (!mDimensionsInWhat.empty()) {
    169             uint64_t dimenPathToken = protoOutput->start(
    170                     FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
    171             writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
    172             protoOutput->end(dimenPathToken);
    173         }
    174         if (!mDimensionsInCondition.empty()) {
    175             uint64_t dimenPathToken = protoOutput->start(
    176                     FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
    177             writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
    178             protoOutput->end(dimenPathToken);
    179         }
    180 
    181     }
    182 
    183     uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_COUNT_METRICS);
    184 
    185     for (const auto& counter : mPastBuckets) {
    186         const MetricDimensionKey& dimensionKey = counter.first;
    187         VLOG("  dimension key %s", dimensionKey.toString().c_str());
    188 
    189         uint64_t wrapperToken =
    190                 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
    191 
    192         // First fill dimension.
    193         if (mSliceByPositionALL) {
    194             uint64_t dimensionToken = protoOutput->start(
    195                     FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
    196             writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
    197             protoOutput->end(dimensionToken);
    198 
    199             if (dimensionKey.hasDimensionKeyInCondition()) {
    200                 uint64_t dimensionInConditionToken = protoOutput->start(
    201                         FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
    202                 writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
    203                                       str_set, protoOutput);
    204                 protoOutput->end(dimensionInConditionToken);
    205             }
    206         } else {
    207             writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
    208                                            FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
    209             if (dimensionKey.hasDimensionKeyInCondition()) {
    210                 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
    211                                                FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
    212                                                str_set, protoOutput);
    213             }
    214         }
    215         // Then fill bucket_info (CountBucketInfo).
    216         for (const auto& bucket : counter.second) {
    217             uint64_t bucketInfoToken = protoOutput->start(
    218                     FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
    219             // Partial bucket.
    220             if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
    221                 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
    222                                    (long long)NanoToMillis(bucket.mBucketStartNs));
    223                 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
    224                                    (long long)NanoToMillis(bucket.mBucketEndNs));
    225             } else {
    226                 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
    227                                    (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
    228             }
    229             protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_COUNT, (long long)bucket.mCount);
    230             protoOutput->end(bucketInfoToken);
    231             VLOG("\t bucket [%lld - %lld] count: %lld", (long long)bucket.mBucketStartNs,
    232                  (long long)bucket.mBucketEndNs, (long long)bucket.mCount);
    233         }
    234         protoOutput->end(wrapperToken);
    235     }
    236 
    237     protoOutput->end(protoToken);
    238 
    239     if (erase_data) {
    240         mPastBuckets.clear();
    241     }
    242 }
    243 
    244 void CountMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
    245     flushIfNeededLocked(dropTimeNs);
    246     StatsdStats::getInstance().noteBucketDropped(mMetricId);
    247     mPastBuckets.clear();
    248 }
    249 
    250 void CountMetricProducer::onConditionChangedLocked(const bool conditionMet,
    251                                                    const int64_t eventTime) {
    252     VLOG("Metric %lld onConditionChanged", (long long)mMetricId);
    253     mCondition = conditionMet ? ConditionState::kTrue : ConditionState::kFalse;
    254 }
    255 
    256 bool CountMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
    257     if (mCurrentSlicedCounter->find(newKey) != mCurrentSlicedCounter->end()) {
    258         return false;
    259     }
    260     // ===========GuardRail==============
    261     // 1. Report the tuple count if the tuple count > soft limit
    262     if (mCurrentSlicedCounter->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
    263         size_t newTupleCount = mCurrentSlicedCounter->size() + 1;
    264         StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
    265         // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
    266         if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
    267             ALOGE("CountMetric %lld dropping data for dimension key %s",
    268                 (long long)mMetricId, newKey.toString().c_str());
    269             return true;
    270         }
    271     }
    272 
    273     return false;
    274 }
    275 
    276 void CountMetricProducer::onMatchedLogEventInternalLocked(
    277         const size_t matcherIndex, const MetricDimensionKey& eventKey,
    278         const ConditionKey& conditionKey, bool condition,
    279         const LogEvent& event) {
    280     int64_t eventTimeNs = event.GetElapsedTimestampNs();
    281     flushIfNeededLocked(eventTimeNs);
    282 
    283     if (condition == false) {
    284         return;
    285     }
    286 
    287     auto it = mCurrentSlicedCounter->find(eventKey);
    288     if (it == mCurrentSlicedCounter->end()) {
    289         // ===========GuardRail==============
    290         if (hitGuardRailLocked(eventKey)) {
    291             return;
    292         }
    293         // create a counter for the new key
    294         (*mCurrentSlicedCounter)[eventKey] = 1;
    295     } else {
    296         // increment the existing value
    297         auto& count = it->second;
    298         count++;
    299     }
    300     for (auto& tracker : mAnomalyTrackers) {
    301         int64_t countWholeBucket = mCurrentSlicedCounter->find(eventKey)->second;
    302         auto prev = mCurrentFullCounters->find(eventKey);
    303         if (prev != mCurrentFullCounters->end()) {
    304             countWholeBucket += prev->second;
    305         }
    306         tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, mMetricId, eventKey,
    307                                          countWholeBucket);
    308     }
    309 
    310     VLOG("metric %lld %s->%lld", (long long)mMetricId, eventKey.toString().c_str(),
    311          (long long)(*mCurrentSlicedCounter)[eventKey]);
    312 }
    313 
    314 // When a new matched event comes in, we check if event falls into the current
    315 // bucket. If not, flush the old counter to past buckets and initialize the new bucket.
    316 void CountMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) {
    317     int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
    318     if (eventTimeNs < currentBucketEndTimeNs) {
    319         return;
    320     }
    321 
    322     // Setup the bucket start time and number.
    323     int64_t numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
    324     int64_t nextBucketNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
    325     flushCurrentBucketLocked(eventTimeNs, nextBucketNs);
    326 
    327     mCurrentBucketNum += numBucketsForward;
    328     VLOG("metric %lld: new bucket start time: %lld", (long long)mMetricId,
    329          (long long)mCurrentBucketStartTimeNs);
    330 }
    331 
    332 void CountMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
    333                                                    const int64_t& nextBucketStartTimeNs) {
    334     int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs();
    335     CountBucket info;
    336     info.mBucketStartNs = mCurrentBucketStartTimeNs;
    337     if (eventTimeNs < fullBucketEndTimeNs) {
    338         info.mBucketEndNs = eventTimeNs;
    339     } else {
    340         info.mBucketEndNs = fullBucketEndTimeNs;
    341     }
    342     for (const auto& counter : *mCurrentSlicedCounter) {
    343         info.mCount = counter.second;
    344         auto& bucketList = mPastBuckets[counter.first];
    345         bucketList.push_back(info);
    346         VLOG("metric %lld, dump key value: %s -> %lld", (long long)mMetricId,
    347              counter.first.toString().c_str(),
    348              (long long)counter.second);
    349     }
    350 
    351     // If we have finished a full bucket, then send this to anomaly tracker.
    352     if (eventTimeNs > fullBucketEndTimeNs) {
    353         // Accumulate partial buckets with current value and then send to anomaly tracker.
    354         if (mCurrentFullCounters->size() > 0) {
    355             for (const auto& keyValuePair : *mCurrentSlicedCounter) {
    356                 (*mCurrentFullCounters)[keyValuePair.first] += keyValuePair.second;
    357             }
    358             for (auto& tracker : mAnomalyTrackers) {
    359                 tracker->addPastBucket(mCurrentFullCounters, mCurrentBucketNum);
    360             }
    361             mCurrentFullCounters = std::make_shared<DimToValMap>();
    362         } else {
    363             // Skip aggregating the partial buckets since there's no previous partial bucket.
    364             for (auto& tracker : mAnomalyTrackers) {
    365                 tracker->addPastBucket(mCurrentSlicedCounter, mCurrentBucketNum);
    366             }
    367         }
    368     } else {
    369         // Accumulate partial bucket.
    370         for (const auto& keyValuePair : *mCurrentSlicedCounter) {
    371             (*mCurrentFullCounters)[keyValuePair.first] += keyValuePair.second;
    372         }
    373     }
    374 
    375     StatsdStats::getInstance().noteBucketCount(mMetricId);
    376     // Only resets the counters, but doesn't setup the times nor numbers.
    377     // (Do not clear since the old one is still referenced in mAnomalyTrackers).
    378     mCurrentSlicedCounter = std::make_shared<DimToValMap>();
    379     mCurrentBucketStartTimeNs = nextBucketStartTimeNs;
    380 }
    381 
    382 // Rough estimate of CountMetricProducer buffer stored. This number will be
    383 // greater than actual data size as it contains each dimension of
    384 // CountMetricData is  duplicated.
    385 size_t CountMetricProducer::byteSizeLocked() const {
    386     size_t totalSize = 0;
    387     for (const auto& pair : mPastBuckets) {
    388         totalSize += pair.second.size() * kBucketSize;
    389     }
    390     return totalSize;
    391 }
    392 
    393 }  // namespace statsd
    394 }  // namespace os
    395 }  // namespace android
    396