1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.net; 18 19 import static android.net.NetworkStats.TAG_NONE; 20 import static android.net.TrafficStats.KB_IN_BYTES; 21 import static android.net.TrafficStats.MB_IN_BYTES; 22 import static android.text.format.DateUtils.YEAR_IN_MILLIS; 23 import static com.android.internal.util.Preconditions.checkNotNull; 24 25 import android.annotation.Nullable; 26 import android.net.NetworkStats; 27 import android.net.NetworkStats.NonMonotonicObserver; 28 import android.net.NetworkStatsHistory; 29 import android.net.NetworkTemplate; 30 import android.net.TrafficStats; 31 import android.os.DropBoxManager; 32 import android.service.NetworkStatsRecorderProto; 33 import android.util.Log; 34 import android.util.MathUtils; 35 import android.util.Slog; 36 import android.util.proto.ProtoOutputStream; 37 38 import com.android.internal.net.VpnInfo; 39 import com.android.internal.util.FileRotator; 40 import com.android.internal.util.IndentingPrintWriter; 41 import com.google.android.collect.Sets; 42 43 import java.io.ByteArrayOutputStream; 44 import java.io.DataOutputStream; 45 import java.io.File; 46 import java.io.IOException; 47 import java.io.InputStream; 48 import java.io.OutputStream; 49 import java.io.PrintWriter; 50 import java.lang.ref.WeakReference; 51 import java.util.Arrays; 52 import java.util.HashSet; 53 import java.util.Map; 54 55 import libcore.io.IoUtils; 56 57 /** 58 * Logic to record deltas between periodic {@link NetworkStats} snapshots into 59 * {@link NetworkStatsHistory} that belong to {@link NetworkStatsCollection}. 60 * Keeps pending changes in memory until they pass a specific threshold, in 61 * bytes. Uses {@link FileRotator} for persistence logic if present. 62 * <p> 63 * Not inherently thread safe. 64 */ 65 public class NetworkStatsRecorder { 66 private static final String TAG = "NetworkStatsRecorder"; 67 private static final boolean LOGD = false; 68 private static final boolean LOGV = false; 69 70 private static final String TAG_NETSTATS_DUMP = "netstats_dump"; 71 72 /** Dump before deleting in {@link #recoverFromWtf()}. */ 73 private static final boolean DUMP_BEFORE_DELETE = true; 74 75 private final FileRotator mRotator; 76 private final NonMonotonicObserver<String> mObserver; 77 private final DropBoxManager mDropBox; 78 private final String mCookie; 79 80 private final long mBucketDuration; 81 private final boolean mOnlyTags; 82 83 private long mPersistThresholdBytes = 2 * MB_IN_BYTES; 84 private NetworkStats mLastSnapshot; 85 86 private final NetworkStatsCollection mPending; 87 private final NetworkStatsCollection mSinceBoot; 88 89 private final CombiningRewriter mPendingRewriter; 90 91 private WeakReference<NetworkStatsCollection> mComplete; 92 93 /** 94 * Non-persisted recorder, with only one bucket. Used by {@link NetworkStatsObservers}. 95 */ 96 public NetworkStatsRecorder() { 97 mRotator = null; 98 mObserver = null; 99 mDropBox = null; 100 mCookie = null; 101 102 // set the bucket big enough to have all data in one bucket, but allow some 103 // slack to avoid overflow 104 mBucketDuration = YEAR_IN_MILLIS; 105 mOnlyTags = false; 106 107 mPending = null; 108 mSinceBoot = new NetworkStatsCollection(mBucketDuration); 109 110 mPendingRewriter = null; 111 } 112 113 /** 114 * Persisted recorder. 115 */ 116 public NetworkStatsRecorder(FileRotator rotator, NonMonotonicObserver<String> observer, 117 DropBoxManager dropBox, String cookie, long bucketDuration, boolean onlyTags) { 118 mRotator = checkNotNull(rotator, "missing FileRotator"); 119 mObserver = checkNotNull(observer, "missing NonMonotonicObserver"); 120 mDropBox = checkNotNull(dropBox, "missing DropBoxManager"); 121 mCookie = cookie; 122 123 mBucketDuration = bucketDuration; 124 mOnlyTags = onlyTags; 125 126 mPending = new NetworkStatsCollection(bucketDuration); 127 mSinceBoot = new NetworkStatsCollection(bucketDuration); 128 129 mPendingRewriter = new CombiningRewriter(mPending); 130 } 131 132 public void setPersistThreshold(long thresholdBytes) { 133 if (LOGV) Slog.v(TAG, "setPersistThreshold() with " + thresholdBytes); 134 mPersistThresholdBytes = MathUtils.constrain( 135 thresholdBytes, 1 * KB_IN_BYTES, 100 * MB_IN_BYTES); 136 } 137 138 public void resetLocked() { 139 mLastSnapshot = null; 140 if (mPending != null) { 141 mPending.reset(); 142 } 143 if (mSinceBoot != null) { 144 mSinceBoot.reset(); 145 } 146 if (mComplete != null) { 147 mComplete.clear(); 148 } 149 } 150 151 public NetworkStats.Entry getTotalSinceBootLocked(NetworkTemplate template) { 152 return mSinceBoot.getSummary(template, Long.MIN_VALUE, Long.MAX_VALUE, 153 NetworkStatsAccess.Level.DEVICE).getTotal(null); 154 } 155 156 public NetworkStatsCollection getSinceBoot() { 157 return mSinceBoot; 158 } 159 160 /** 161 * Load complete history represented by {@link FileRotator}. Caches 162 * internally as a {@link WeakReference}, and updated with future 163 * {@link #recordSnapshotLocked(NetworkStats, Map, long)} snapshots as long 164 * as reference is valid. 165 */ 166 public NetworkStatsCollection getOrLoadCompleteLocked() { 167 checkNotNull(mRotator, "missing FileRotator"); 168 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 169 if (res == null) { 170 res = loadLocked(Long.MIN_VALUE, Long.MAX_VALUE); 171 mComplete = new WeakReference<NetworkStatsCollection>(res); 172 } 173 return res; 174 } 175 176 public NetworkStatsCollection getOrLoadPartialLocked(long start, long end) { 177 checkNotNull(mRotator, "missing FileRotator"); 178 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 179 if (res == null) { 180 res = loadLocked(start, end); 181 } 182 return res; 183 } 184 185 private NetworkStatsCollection loadLocked(long start, long end) { 186 if (LOGD) Slog.d(TAG, "loadLocked() reading from disk for " + mCookie); 187 final NetworkStatsCollection res = new NetworkStatsCollection(mBucketDuration); 188 try { 189 mRotator.readMatching(res, start, end); 190 res.recordCollection(mPending); 191 } catch (IOException e) { 192 Log.wtf(TAG, "problem completely reading network stats", e); 193 recoverFromWtf(); 194 } catch (OutOfMemoryError e) { 195 Log.wtf(TAG, "problem completely reading network stats", e); 196 recoverFromWtf(); 197 } 198 return res; 199 } 200 201 /** 202 * Record any delta that occurred since last {@link NetworkStats} snapshot, 203 * using the given {@link Map} to identify network interfaces. First 204 * snapshot is considered bootstrap, and is not counted as delta. 205 * 206 * @param vpnArray Optional info about the currently active VPN, if any. This is used to 207 * redistribute traffic from the VPN app to the underlying responsible apps. 208 * This should always be set to null if the provided snapshot is aggregated 209 * across all UIDs (e.g. contains UID_ALL buckets), regardless of VPN state. 210 */ 211 public void recordSnapshotLocked(NetworkStats snapshot, 212 Map<String, NetworkIdentitySet> ifaceIdent, @Nullable VpnInfo[] vpnArray, 213 long currentTimeMillis) { 214 final HashSet<String> unknownIfaces = Sets.newHashSet(); 215 216 // skip recording when snapshot missing 217 if (snapshot == null) return; 218 219 // assume first snapshot is bootstrap and don't record 220 if (mLastSnapshot == null) { 221 mLastSnapshot = snapshot; 222 return; 223 } 224 225 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 226 227 final NetworkStats delta = NetworkStats.subtract( 228 snapshot, mLastSnapshot, mObserver, mCookie); 229 final long end = currentTimeMillis; 230 final long start = end - delta.getElapsedRealtime(); 231 232 if (vpnArray != null) { 233 for (VpnInfo info : vpnArray) { 234 delta.migrateTun(info.ownerUid, info.vpnIface, info.primaryUnderlyingIface); 235 } 236 } 237 238 NetworkStats.Entry entry = null; 239 for (int i = 0; i < delta.size(); i++) { 240 entry = delta.getValues(i, entry); 241 final NetworkIdentitySet ident = ifaceIdent.get(entry.iface); 242 if (ident == null) { 243 unknownIfaces.add(entry.iface); 244 continue; 245 } 246 247 // skip when no delta occurred 248 if (entry.isEmpty()) continue; 249 250 // only record tag data when requested 251 if ((entry.tag == TAG_NONE) != mOnlyTags) { 252 if (mPending != null) { 253 mPending.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 254 } 255 256 // also record against boot stats when present 257 if (mSinceBoot != null) { 258 mSinceBoot.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 259 } 260 261 // also record against complete dataset when present 262 if (complete != null) { 263 complete.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 264 } 265 } 266 } 267 268 mLastSnapshot = snapshot; 269 270 if (LOGV && unknownIfaces.size() > 0) { 271 Slog.w(TAG, "unknown interfaces " + unknownIfaces + ", ignoring those stats"); 272 } 273 } 274 275 /** 276 * Consider persisting any pending deltas, if they are beyond 277 * {@link #mPersistThresholdBytes}. 278 */ 279 public void maybePersistLocked(long currentTimeMillis) { 280 checkNotNull(mRotator, "missing FileRotator"); 281 final long pendingBytes = mPending.getTotalBytes(); 282 if (pendingBytes >= mPersistThresholdBytes) { 283 forcePersistLocked(currentTimeMillis); 284 } else { 285 mRotator.maybeRotate(currentTimeMillis); 286 } 287 } 288 289 /** 290 * Force persisting any pending deltas. 291 */ 292 public void forcePersistLocked(long currentTimeMillis) { 293 checkNotNull(mRotator, "missing FileRotator"); 294 if (mPending.isDirty()) { 295 if (LOGD) Slog.d(TAG, "forcePersistLocked() writing for " + mCookie); 296 try { 297 mRotator.rewriteActive(mPendingRewriter, currentTimeMillis); 298 mRotator.maybeRotate(currentTimeMillis); 299 mPending.reset(); 300 } catch (IOException e) { 301 Log.wtf(TAG, "problem persisting pending stats", e); 302 recoverFromWtf(); 303 } catch (OutOfMemoryError e) { 304 Log.wtf(TAG, "problem persisting pending stats", e); 305 recoverFromWtf(); 306 } 307 } 308 } 309 310 /** 311 * Remove the given UID from all {@link FileRotator} history, migrating it 312 * to {@link TrafficStats#UID_REMOVED}. 313 */ 314 public void removeUidsLocked(int[] uids) { 315 if (mRotator != null) { 316 try { 317 // Rewrite all persisted data to migrate UID stats 318 mRotator.rewriteAll(new RemoveUidRewriter(mBucketDuration, uids)); 319 } catch (IOException e) { 320 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 321 recoverFromWtf(); 322 } catch (OutOfMemoryError e) { 323 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 324 recoverFromWtf(); 325 } 326 } 327 328 // Remove any pending stats 329 if (mPending != null) { 330 mPending.removeUids(uids); 331 } 332 if (mSinceBoot != null) { 333 mSinceBoot.removeUids(uids); 334 } 335 336 // Clear UID from current stats snapshot 337 if (mLastSnapshot != null) { 338 mLastSnapshot = mLastSnapshot.withoutUids(uids); 339 } 340 341 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 342 if (complete != null) { 343 complete.removeUids(uids); 344 } 345 } 346 347 /** 348 * Rewriter that will combine current {@link NetworkStatsCollection} values 349 * with anything read from disk, and write combined set to disk. Clears the 350 * original {@link NetworkStatsCollection} when finished writing. 351 */ 352 private static class CombiningRewriter implements FileRotator.Rewriter { 353 private final NetworkStatsCollection mCollection; 354 355 public CombiningRewriter(NetworkStatsCollection collection) { 356 mCollection = checkNotNull(collection, "missing NetworkStatsCollection"); 357 } 358 359 @Override 360 public void reset() { 361 // ignored 362 } 363 364 @Override 365 public void read(InputStream in) throws IOException { 366 mCollection.read(in); 367 } 368 369 @Override 370 public boolean shouldWrite() { 371 return true; 372 } 373 374 @Override 375 public void write(OutputStream out) throws IOException { 376 mCollection.write(new DataOutputStream(out)); 377 mCollection.reset(); 378 } 379 } 380 381 /** 382 * Rewriter that will remove any {@link NetworkStatsHistory} attributed to 383 * the requested UID, only writing data back when modified. 384 */ 385 public static class RemoveUidRewriter implements FileRotator.Rewriter { 386 private final NetworkStatsCollection mTemp; 387 private final int[] mUids; 388 389 public RemoveUidRewriter(long bucketDuration, int[] uids) { 390 mTemp = new NetworkStatsCollection(bucketDuration); 391 mUids = uids; 392 } 393 394 @Override 395 public void reset() { 396 mTemp.reset(); 397 } 398 399 @Override 400 public void read(InputStream in) throws IOException { 401 mTemp.read(in); 402 mTemp.clearDirty(); 403 mTemp.removeUids(mUids); 404 } 405 406 @Override 407 public boolean shouldWrite() { 408 return mTemp.isDirty(); 409 } 410 411 @Override 412 public void write(OutputStream out) throws IOException { 413 mTemp.write(new DataOutputStream(out)); 414 } 415 } 416 417 public void importLegacyNetworkLocked(File file) throws IOException { 418 checkNotNull(mRotator, "missing FileRotator"); 419 420 // legacy file still exists; start empty to avoid double importing 421 mRotator.deleteAll(); 422 423 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 424 collection.readLegacyNetwork(file); 425 426 final long startMillis = collection.getStartMillis(); 427 final long endMillis = collection.getEndMillis(); 428 429 if (!collection.isEmpty()) { 430 // process legacy data, creating active file at starting time, then 431 // using end time to possibly trigger rotation. 432 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 433 mRotator.maybeRotate(endMillis); 434 } 435 } 436 437 public void importLegacyUidLocked(File file) throws IOException { 438 checkNotNull(mRotator, "missing FileRotator"); 439 440 // legacy file still exists; start empty to avoid double importing 441 mRotator.deleteAll(); 442 443 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 444 collection.readLegacyUid(file, mOnlyTags); 445 446 final long startMillis = collection.getStartMillis(); 447 final long endMillis = collection.getEndMillis(); 448 449 if (!collection.isEmpty()) { 450 // process legacy data, creating active file at starting time, then 451 // using end time to possibly trigger rotation. 452 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 453 mRotator.maybeRotate(endMillis); 454 } 455 } 456 457 public void dumpLocked(IndentingPrintWriter pw, boolean fullHistory) { 458 if (mPending != null) { 459 pw.print("Pending bytes: "); pw.println(mPending.getTotalBytes()); 460 } 461 if (fullHistory) { 462 pw.println("Complete history:"); 463 getOrLoadCompleteLocked().dump(pw); 464 } else { 465 pw.println("History since boot:"); 466 mSinceBoot.dump(pw); 467 } 468 } 469 470 public void writeToProtoLocked(ProtoOutputStream proto, long tag) { 471 final long start = proto.start(tag); 472 if (mPending != null) { 473 proto.write(NetworkStatsRecorderProto.PENDING_TOTAL_BYTES, mPending.getTotalBytes()); 474 } 475 getOrLoadCompleteLocked().writeToProto(proto, NetworkStatsRecorderProto.COMPLETE_HISTORY); 476 proto.end(start); 477 } 478 479 public void dumpCheckin(PrintWriter pw, long start, long end) { 480 // Only load and dump stats from the requested window 481 getOrLoadPartialLocked(start, end).dumpCheckin(pw, start, end); 482 } 483 484 /** 485 * Recover from {@link FileRotator} failure by dumping state to 486 * {@link DropBoxManager} and deleting contents. 487 */ 488 private void recoverFromWtf() { 489 if (DUMP_BEFORE_DELETE) { 490 final ByteArrayOutputStream os = new ByteArrayOutputStream(); 491 try { 492 mRotator.dumpAll(os); 493 } catch (IOException e) { 494 // ignore partial contents 495 os.reset(); 496 } finally { 497 IoUtils.closeQuietly(os); 498 } 499 mDropBox.addData(TAG_NETSTATS_DUMP, os.toByteArray(), 0); 500 } 501 502 mRotator.deleteAll(); 503 } 504 } 505
