1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.net; 18 19 import static android.net.NetworkStats.TAG_NONE; 20 import static android.net.TrafficStats.KB_IN_BYTES; 21 import static android.net.TrafficStats.MB_IN_BYTES; 22 import static com.android.internal.util.Preconditions.checkNotNull; 23 24 import android.net.NetworkStats; 25 import android.net.NetworkStats.NonMonotonicObserver; 26 import android.net.NetworkStatsHistory; 27 import android.net.NetworkTemplate; 28 import android.net.TrafficStats; 29 import android.os.DropBoxManager; 30 import android.util.Log; 31 import android.util.MathUtils; 32 import android.util.Slog; 33 34 import com.android.internal.util.FileRotator; 35 import com.android.internal.util.IndentingPrintWriter; 36 import com.google.android.collect.Sets; 37 38 import java.io.ByteArrayOutputStream; 39 import java.io.DataOutputStream; 40 import java.io.File; 41 import java.io.IOException; 42 import java.io.InputStream; 43 import java.io.OutputStream; 44 import java.io.PrintWriter; 45 import java.lang.ref.WeakReference; 46 import java.util.Arrays; 47 import java.util.HashSet; 48 import java.util.Map; 49 50 import libcore.io.IoUtils; 51 52 /** 53 * Logic to record deltas between periodic {@link NetworkStats} snapshots into 54 * {@link NetworkStatsHistory} that belong to {@link NetworkStatsCollection}. 55 * Keeps pending changes in memory until they pass a specific threshold, in 56 * bytes. Uses {@link FileRotator} for persistence logic. 57 * <p> 58 * Not inherently thread safe. 59 */ 60 public class NetworkStatsRecorder { 61 private static final String TAG = "NetworkStatsRecorder"; 62 private static final boolean LOGD = false; 63 private static final boolean LOGV = false; 64 65 private static final String TAG_NETSTATS_DUMP = "netstats_dump"; 66 67 /** Dump before deleting in {@link #recoverFromWtf()}. */ 68 private static final boolean DUMP_BEFORE_DELETE = true; 69 70 private final FileRotator mRotator; 71 private final NonMonotonicObserver<String> mObserver; 72 private final DropBoxManager mDropBox; 73 private final String mCookie; 74 75 private final long mBucketDuration; 76 private final boolean mOnlyTags; 77 78 private long mPersistThresholdBytes = 2 * MB_IN_BYTES; 79 private NetworkStats mLastSnapshot; 80 81 private final NetworkStatsCollection mPending; 82 private final NetworkStatsCollection mSinceBoot; 83 84 private final CombiningRewriter mPendingRewriter; 85 86 private WeakReference<NetworkStatsCollection> mComplete; 87 88 public NetworkStatsRecorder(FileRotator rotator, NonMonotonicObserver<String> observer, 89 DropBoxManager dropBox, String cookie, long bucketDuration, boolean onlyTags) { 90 mRotator = checkNotNull(rotator, "missing FileRotator"); 91 mObserver = checkNotNull(observer, "missing NonMonotonicObserver"); 92 mDropBox = checkNotNull(dropBox, "missing DropBoxManager"); 93 mCookie = cookie; 94 95 mBucketDuration = bucketDuration; 96 mOnlyTags = onlyTags; 97 98 mPending = new NetworkStatsCollection(bucketDuration); 99 mSinceBoot = new NetworkStatsCollection(bucketDuration); 100 101 mPendingRewriter = new CombiningRewriter(mPending); 102 } 103 104 public void setPersistThreshold(long thresholdBytes) { 105 if (LOGV) Slog.v(TAG, "setPersistThreshold() with " + thresholdBytes); 106 mPersistThresholdBytes = MathUtils.constrain( 107 thresholdBytes, 1 * KB_IN_BYTES, 100 * MB_IN_BYTES); 108 } 109 110 public void resetLocked() { 111 mLastSnapshot = null; 112 mPending.reset(); 113 mSinceBoot.reset(); 114 mComplete.clear(); 115 } 116 117 public NetworkStats.Entry getTotalSinceBootLocked(NetworkTemplate template) { 118 return mSinceBoot.getSummary(template, Long.MIN_VALUE, Long.MAX_VALUE).getTotal(null); 119 } 120 121 /** 122 * Load complete history represented by {@link FileRotator}. Caches 123 * internally as a {@link WeakReference}, and updated with future 124 * {@link #recordSnapshotLocked(NetworkStats, Map, long)} snapshots as long 125 * as reference is valid. 126 */ 127 public NetworkStatsCollection getOrLoadCompleteLocked() { 128 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 129 if (res == null) { 130 res = loadLocked(Long.MIN_VALUE, Long.MAX_VALUE); 131 mComplete = new WeakReference<NetworkStatsCollection>(res); 132 } 133 return res; 134 } 135 136 public NetworkStatsCollection getOrLoadPartialLocked(long start, long end) { 137 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 138 if (res == null) { 139 res = loadLocked(start, end); 140 } 141 return res; 142 } 143 144 private NetworkStatsCollection loadLocked(long start, long end) { 145 if (LOGD) Slog.d(TAG, "loadLocked() reading from disk for " + mCookie); 146 final NetworkStatsCollection res = new NetworkStatsCollection(mBucketDuration); 147 try { 148 mRotator.readMatching(res, start, end); 149 res.recordCollection(mPending); 150 } catch (IOException e) { 151 Log.wtf(TAG, "problem completely reading network stats", e); 152 recoverFromWtf(); 153 } catch (OutOfMemoryError e) { 154 Log.wtf(TAG, "problem completely reading network stats", e); 155 recoverFromWtf(); 156 } 157 return res; 158 } 159 160 /** 161 * Record any delta that occurred since last {@link NetworkStats} snapshot, 162 * using the given {@link Map} to identify network interfaces. First 163 * snapshot is considered bootstrap, and is not counted as delta. 164 */ 165 public void recordSnapshotLocked(NetworkStats snapshot, 166 Map<String, NetworkIdentitySet> ifaceIdent, long currentTimeMillis) { 167 final HashSet<String> unknownIfaces = Sets.newHashSet(); 168 169 // skip recording when snapshot missing 170 if (snapshot == null) return; 171 172 // assume first snapshot is bootstrap and don't record 173 if (mLastSnapshot == null) { 174 mLastSnapshot = snapshot; 175 return; 176 } 177 178 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 179 180 final NetworkStats delta = NetworkStats.subtract( 181 snapshot, mLastSnapshot, mObserver, mCookie); 182 final long end = currentTimeMillis; 183 final long start = end - delta.getElapsedRealtime(); 184 185 NetworkStats.Entry entry = null; 186 for (int i = 0; i < delta.size(); i++) { 187 entry = delta.getValues(i, entry); 188 final NetworkIdentitySet ident = ifaceIdent.get(entry.iface); 189 if (ident == null) { 190 unknownIfaces.add(entry.iface); 191 continue; 192 } 193 194 // skip when no delta occurred 195 if (entry.isEmpty()) continue; 196 197 // only record tag data when requested 198 if ((entry.tag == TAG_NONE) != mOnlyTags) { 199 mPending.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 200 201 // also record against boot stats when present 202 if (mSinceBoot != null) { 203 mSinceBoot.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 204 } 205 206 // also record against complete dataset when present 207 if (complete != null) { 208 complete.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 209 } 210 } 211 } 212 213 mLastSnapshot = snapshot; 214 215 if (LOGV && unknownIfaces.size() > 0) { 216 Slog.w(TAG, "unknown interfaces " + unknownIfaces + ", ignoring those stats"); 217 } 218 } 219 220 /** 221 * Consider persisting any pending deltas, if they are beyond 222 * {@link #mPersistThresholdBytes}. 223 */ 224 public void maybePersistLocked(long currentTimeMillis) { 225 final long pendingBytes = mPending.getTotalBytes(); 226 if (pendingBytes >= mPersistThresholdBytes) { 227 forcePersistLocked(currentTimeMillis); 228 } else { 229 mRotator.maybeRotate(currentTimeMillis); 230 } 231 } 232 233 /** 234 * Force persisting any pending deltas. 235 */ 236 public void forcePersistLocked(long currentTimeMillis) { 237 if (mPending.isDirty()) { 238 if (LOGD) Slog.d(TAG, "forcePersistLocked() writing for " + mCookie); 239 try { 240 mRotator.rewriteActive(mPendingRewriter, currentTimeMillis); 241 mRotator.maybeRotate(currentTimeMillis); 242 mPending.reset(); 243 } catch (IOException e) { 244 Log.wtf(TAG, "problem persisting pending stats", e); 245 recoverFromWtf(); 246 } catch (OutOfMemoryError e) { 247 Log.wtf(TAG, "problem persisting pending stats", e); 248 recoverFromWtf(); 249 } 250 } 251 } 252 253 /** 254 * Remove the given UID from all {@link FileRotator} history, migrating it 255 * to {@link TrafficStats#UID_REMOVED}. 256 */ 257 public void removeUidsLocked(int[] uids) { 258 try { 259 // Rewrite all persisted data to migrate UID stats 260 mRotator.rewriteAll(new RemoveUidRewriter(mBucketDuration, uids)); 261 } catch (IOException e) { 262 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 263 recoverFromWtf(); 264 } catch (OutOfMemoryError e) { 265 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 266 recoverFromWtf(); 267 } 268 269 // Remove any pending stats 270 mPending.removeUids(uids); 271 mSinceBoot.removeUids(uids); 272 273 // Clear UID from current stats snapshot 274 if (mLastSnapshot != null) { 275 mLastSnapshot = mLastSnapshot.withoutUids(uids); 276 } 277 278 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 279 if (complete != null) { 280 complete.removeUids(uids); 281 } 282 } 283 284 /** 285 * Rewriter that will combine current {@link NetworkStatsCollection} values 286 * with anything read from disk, and write combined set to disk. Clears the 287 * original {@link NetworkStatsCollection} when finished writing. 288 */ 289 private static class CombiningRewriter implements FileRotator.Rewriter { 290 private final NetworkStatsCollection mCollection; 291 292 public CombiningRewriter(NetworkStatsCollection collection) { 293 mCollection = checkNotNull(collection, "missing NetworkStatsCollection"); 294 } 295 296 @Override 297 public void reset() { 298 // ignored 299 } 300 301 @Override 302 public void read(InputStream in) throws IOException { 303 mCollection.read(in); 304 } 305 306 @Override 307 public boolean shouldWrite() { 308 return true; 309 } 310 311 @Override 312 public void write(OutputStream out) throws IOException { 313 mCollection.write(new DataOutputStream(out)); 314 mCollection.reset(); 315 } 316 } 317 318 /** 319 * Rewriter that will remove any {@link NetworkStatsHistory} attributed to 320 * the requested UID, only writing data back when modified. 321 */ 322 public static class RemoveUidRewriter implements FileRotator.Rewriter { 323 private final NetworkStatsCollection mTemp; 324 private final int[] mUids; 325 326 public RemoveUidRewriter(long bucketDuration, int[] uids) { 327 mTemp = new NetworkStatsCollection(bucketDuration); 328 mUids = uids; 329 } 330 331 @Override 332 public void reset() { 333 mTemp.reset(); 334 } 335 336 @Override 337 public void read(InputStream in) throws IOException { 338 mTemp.read(in); 339 mTemp.clearDirty(); 340 mTemp.removeUids(mUids); 341 } 342 343 @Override 344 public boolean shouldWrite() { 345 return mTemp.isDirty(); 346 } 347 348 @Override 349 public void write(OutputStream out) throws IOException { 350 mTemp.write(new DataOutputStream(out)); 351 } 352 } 353 354 public void importLegacyNetworkLocked(File file) throws IOException { 355 // legacy file still exists; start empty to avoid double importing 356 mRotator.deleteAll(); 357 358 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 359 collection.readLegacyNetwork(file); 360 361 final long startMillis = collection.getStartMillis(); 362 final long endMillis = collection.getEndMillis(); 363 364 if (!collection.isEmpty()) { 365 // process legacy data, creating active file at starting time, then 366 // using end time to possibly trigger rotation. 367 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 368 mRotator.maybeRotate(endMillis); 369 } 370 } 371 372 public void importLegacyUidLocked(File file) throws IOException { 373 // legacy file still exists; start empty to avoid double importing 374 mRotator.deleteAll(); 375 376 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 377 collection.readLegacyUid(file, mOnlyTags); 378 379 final long startMillis = collection.getStartMillis(); 380 final long endMillis = collection.getEndMillis(); 381 382 if (!collection.isEmpty()) { 383 // process legacy data, creating active file at starting time, then 384 // using end time to possibly trigger rotation. 385 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 386 mRotator.maybeRotate(endMillis); 387 } 388 } 389 390 public void dumpLocked(IndentingPrintWriter pw, boolean fullHistory) { 391 pw.print("Pending bytes: "); pw.println(mPending.getTotalBytes()); 392 if (fullHistory) { 393 pw.println("Complete history:"); 394 getOrLoadCompleteLocked().dump(pw); 395 } else { 396 pw.println("History since boot:"); 397 mSinceBoot.dump(pw); 398 } 399 } 400 401 public void dumpCheckin(PrintWriter pw, long start, long end) { 402 // Only load and dump stats from the requested window 403 getOrLoadPartialLocked(start, end).dumpCheckin(pw, start, end); 404 } 405 406 /** 407 * Recover from {@link FileRotator} failure by dumping state to 408 * {@link DropBoxManager} and deleting contents. 409 */ 410 private void recoverFromWtf() { 411 if (DUMP_BEFORE_DELETE) { 412 final ByteArrayOutputStream os = new ByteArrayOutputStream(); 413 try { 414 mRotator.dumpAll(os); 415 } catch (IOException e) { 416 // ignore partial contents 417 os.reset(); 418 } finally { 419 IoUtils.closeQuietly(os); 420 } 421 mDropBox.addData(TAG_NETSTATS_DUMP, os.toByteArray(), 0); 422 } 423 424 mRotator.deleteAll(); 425 } 426 } 427