1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <gtest/gtest_prod.h> 20 #include <utils/threads.h> 21 #include <list> 22 #include "anomaly/AnomalyTracker.h" 23 #include "condition/ConditionTimer.h" 24 #include "condition/ConditionTracker.h" 25 #include "external/PullDataReceiver.h" 26 #include "external/StatsPullerManager.h" 27 #include "matchers/EventMatcherWizard.h" 28 #include "stats_log_util.h" 29 #include "MetricProducer.h" 30 #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" 31 32 namespace android { 33 namespace os { 34 namespace statsd { 35 36 struct ValueBucket { 37 int64_t mBucketStartNs; 38 int64_t mBucketEndNs; 39 std::vector<int> valueIndex; 40 std::vector<Value> values; 41 // If the metric has no condition, then this field is just wasted. 42 // When we tune statsd memory usage in the future, this is a candidate to optimize. 43 int64_t mConditionTrueNs; 44 }; 45 46 47 // Aggregates values within buckets. 48 // 49 // There are different events that might complete a bucket 50 // - a condition change 51 // - an app upgrade 52 // - an alarm set to the end of the bucket 53 class ValueMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { 54 public: 55 ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, 56 const int conditionIndex, const sp<ConditionWizard>& conditionWizard, 57 const int whatMatcherIndex, 58 const sp<EventMatcherWizard>& matcherWizard, 59 const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs, 60 const sp<StatsPullerManager>& pullerManager); 61 62 virtual ~ValueMetricProducer(); 63 64 // Process data pulled on bucket boundary. 65 void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, 66 bool pullSuccess, int64_t originalPullTimeNs) override; 67 68 // ValueMetric needs special logic if it's a pulled atom. 69 void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, 70 const int64_t version) override { 71 std::lock_guard<std::mutex> lock(mMutex); 72 if (!mSplitBucketForAppUpgrade) { 73 return; 74 } 75 if (mIsPulled && mCondition) { 76 pullAndMatchEventsLocked(eventTimeNs, mCondition); 77 } 78 flushCurrentBucketLocked(eventTimeNs, eventTimeNs); 79 }; 80 81 protected: 82 void onMatchedLogEventInternalLocked( 83 const size_t matcherIndex, const MetricDimensionKey& eventKey, 84 const ConditionKey& conditionKey, bool condition, 85 const LogEvent& event) override; 86 87 private: 88 void onDumpReportLocked(const int64_t dumpTimeNs, 89 const bool include_current_partial_bucket, 90 const bool erase_data, 91 const DumpLatency dumpLatency, 92 std::set<string> *str_set, 93 android::util::ProtoOutputStream* protoOutput) override; 94 void clearPastBucketsLocked(const int64_t dumpTimeNs) override; 95 96 // Internal interface to handle active state change. 97 void onActiveStateChangedLocked(const int64_t& eventTimeNs) override; 98 99 // Internal interface to handle condition change. 100 void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; 101 102 // Internal interface to handle sliced condition change. 103 void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; 104 105 // Internal function to calculate the current used bytes. 106 size_t byteSizeLocked() const override; 107 108 void dumpStatesLocked(FILE* out, bool verbose) const override; 109 110 // For pulled metrics, this method should only be called if a pull has be done. Else we will 111 // not have complete data for the bucket. 112 void flushIfNeededLocked(const int64_t& eventTime) override; 113 114 // For pulled metrics, this method should only be called if a pulled have be done. Else we will 115 // not have complete data for the bucket. 116 void flushCurrentBucketLocked(const int64_t& eventTimeNs, 117 const int64_t& nextBucketStartTimeNs) override; 118 119 void prepareFirstBucketLocked() override; 120 121 void dropDataLocked(const int64_t dropTimeNs) override; 122 123 // Calculate previous bucket end time based on current time. 124 int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); 125 126 // Calculate how many buckets are present between the current bucket and eventTimeNs. 127 int64_t calcBucketsForwardCount(const int64_t& eventTimeNs) const; 128 129 // Mark the data as invalid. 130 void invalidateCurrentBucket(); 131 void invalidateCurrentBucketWithoutResetBase(); 132 133 const int mWhatMatcherIndex; 134 135 sp<EventMatcherWizard> mEventMatcherWizard; 136 137 sp<StatsPullerManager> mPullerManager; 138 139 // Value fields for matching. 140 std::vector<Matcher> mFieldMatchers; 141 142 // Value fields for matching. 143 std::set<MetricDimensionKey> mMatchedMetricDimensionKeys; 144 145 // tagId for pulled data. -1 if this is not pulled 146 const int mPullTagId; 147 148 // if this is pulled metric 149 const bool mIsPulled; 150 151 // internal state of an ongoing aggregation bucket. 152 typedef struct { 153 // Index in multi value aggregation. 154 int valueIndex; 155 // Holds current base value of the dimension. Take diff and update if necessary. 156 Value base; 157 // Whether there is a base to diff to. 158 bool hasBase; 159 // Current value, depending on the aggregation type. 160 Value value; 161 // Number of samples collected. 162 int sampleSize; 163 // If this dimension has any non-tainted value. If not, don't report the 164 // dimension. 165 bool hasValue = false; 166 // Whether new data is seen in the bucket. 167 bool seenNewData = false; 168 } Interval; 169 170 std::unordered_map<MetricDimensionKey, std::vector<Interval>> mCurrentSlicedBucket; 171 172 std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; 173 174 // Save the past buckets and we can clear when the StatsLogReport is dumped. 175 std::unordered_map<MetricDimensionKey, std::vector<ValueBucket>> mPastBuckets; 176 177 // Pairs of (elapsed start, elapsed end) denoting buckets that were skipped. 178 std::list<std::pair<int64_t, int64_t>> mSkippedBuckets; 179 180 const int64_t mMinBucketSizeNs; 181 182 // Util function to check whether the specified dimension hits the guardrail. 183 bool hitGuardRailLocked(const MetricDimensionKey& newKey); 184 bool hasReachedGuardRailLimit() const; 185 186 bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey); 187 188 void pullAndMatchEventsLocked(const int64_t timestampNs, ConditionState condition); 189 190 void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData, 191 int64_t originalPullTimeNs, int64_t eventElapsedTimeNs, 192 ConditionState condition); 193 194 ValueBucket buildPartialBucket(int64_t bucketEndTime, 195 const std::vector<Interval>& intervals); 196 void initCurrentSlicedBucket(int64_t nextBucketStartTimeNs); 197 void appendToFullBucket(int64_t eventTimeNs, int64_t fullBucketEndTimeNs); 198 199 // Reset diff base and mHasGlobalBase 200 void resetBase(); 201 202 static const size_t kBucketSize = sizeof(ValueBucket{}); 203 204 const size_t mDimensionSoftLimit; 205 206 const size_t mDimensionHardLimit; 207 208 const bool mUseAbsoluteValueOnReset; 209 210 const ValueMetric::AggregationType mAggregationType; 211 212 const bool mUseDiff; 213 214 const ValueMetric::ValueDirection mValueDirection; 215 216 const bool mSkipZeroDiffOutput; 217 218 // If true, use a zero value as base to compute the diff. 219 // This is used for new keys which are present in the new data but was not 220 // present in the base data. 221 // The default base will only be used if we have a global base. 222 const bool mUseZeroDefaultBase; 223 224 // For pulled metrics, this is always set to true whenever a pull succeeds. 225 // It is set to false when a pull fails, or upon condition change to false. 226 // This is used to decide if we have the right base data to compute the 227 // diff against. 228 bool mHasGlobalBase; 229 230 // Invalid bucket. There was a problem in collecting data in the current bucket so we cannot 231 // trust any of the data in this bucket. 232 // 233 // For instance, one pull failed. 234 bool mCurrentBucketIsInvalid; 235 236 const int64_t mMaxPullDelayNs; 237 238 const bool mSplitBucketForAppUpgrade; 239 240 ConditionTimer mConditionTimer; 241 242 FRIEND_TEST(ValueMetricProducerTest, TestAnomalyDetection); 243 FRIEND_TEST(ValueMetricProducerTest, TestBaseSetOnConditionChange); 244 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnAppUpgrade); 245 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnConditionChange); 246 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition); 247 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition); 248 FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2); 249 FRIEND_TEST(ValueMetricProducerTest, TestBucketIncludingUnknownConditionIsInvalid); 250 FRIEND_TEST(ValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet); 251 FRIEND_TEST(ValueMetricProducerTest, TestCalcPreviousBucketEndTime); 252 FRIEND_TEST(ValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged); 253 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary); 254 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged); 255 FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled); 256 FRIEND_TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition); 257 FRIEND_TEST(ValueMetricProducerTest, TestFirstBucket); 258 FRIEND_TEST(ValueMetricProducerTest, TestFullBucketResetWhenLastBucketInvalid); 259 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenGuardRailHit); 260 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenInitialPullFailed); 261 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenLastPullFailed); 262 FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenOneConditionFailed); 263 FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithDiff); 264 FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithoutDiff); 265 FRIEND_TEST(ValueMetricProducerTest, TestPartialBucketCreated); 266 FRIEND_TEST(ValueMetricProducerTest, TestPartialResetOnBucketBoundaries); 267 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse); 268 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue); 269 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withFailure); 270 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges); 271 FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withoutCondition); 272 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsNoCondition); 273 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); 274 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); 275 FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsWithFiltering); 276 FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgrade); 277 FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse); 278 FRIEND_TEST(ValueMetricProducerTest, TestPulledWithAppUpgradeDisabled); 279 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateAvg); 280 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMax); 281 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMin); 282 FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateSum); 283 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithCondition); 284 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade); 285 FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition); 286 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullDelayExceeded); 287 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange); 288 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange_EndOfBucket); 289 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange); 290 FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullTooLate); 291 FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput); 292 FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue); 293 FRIEND_TEST(ValueMetricProducerTest, TestTrimUnusedDimensionKey); 294 FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBase); 295 FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures); 296 friend class ValueMetricProducerTestHelper; 297 }; 298 299 } // namespace statsd 300 } // namespace os 301 } // namespace android 302
