1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.net; 18 19 import static android.net.NetworkStats.TAG_NONE; 20 import static android.net.TrafficStats.KB_IN_BYTES; 21 import static android.net.TrafficStats.MB_IN_BYTES; 22 import static android.text.format.DateUtils.YEAR_IN_MILLIS; 23 24 import static com.android.internal.util.Preconditions.checkNotNull; 25 26 import android.annotation.Nullable; 27 import android.net.NetworkStats; 28 import android.net.NetworkStats.NonMonotonicObserver; 29 import android.net.NetworkStatsHistory; 30 import android.net.NetworkTemplate; 31 import android.net.TrafficStats; 32 import android.os.Binder; 33 import android.os.DropBoxManager; 34 import android.service.NetworkStatsRecorderProto; 35 import android.util.Log; 36 import android.util.MathUtils; 37 import android.util.Slog; 38 import android.util.proto.ProtoOutputStream; 39 40 import com.android.internal.net.VpnInfo; 41 import com.android.internal.util.FileRotator; 42 import com.android.internal.util.IndentingPrintWriter; 43 44 import libcore.io.IoUtils; 45 46 import com.google.android.collect.Sets; 47 48 import java.io.ByteArrayOutputStream; 49 import java.io.DataOutputStream; 50 import java.io.File; 51 import java.io.IOException; 52 import java.io.InputStream; 53 import java.io.OutputStream; 54 import java.io.PrintWriter; 55 import java.lang.ref.WeakReference; 56 import java.util.Arrays; 57 import java.util.HashSet; 58 import java.util.Map; 59 60 /** 61 * Logic to record deltas between periodic {@link NetworkStats} snapshots into 62 * {@link NetworkStatsHistory} that belong to {@link NetworkStatsCollection}. 63 * Keeps pending changes in memory until they pass a specific threshold, in 64 * bytes. Uses {@link FileRotator} for persistence logic if present. 65 * <p> 66 * Not inherently thread safe. 67 */ 68 public class NetworkStatsRecorder { 69 private static final String TAG = "NetworkStatsRecorder"; 70 private static final boolean LOGD = false; 71 private static final boolean LOGV = false; 72 73 private static final String TAG_NETSTATS_DUMP = "netstats_dump"; 74 75 /** Dump before deleting in {@link #recoverFromWtf()}. */ 76 private static final boolean DUMP_BEFORE_DELETE = true; 77 78 private final FileRotator mRotator; 79 private final NonMonotonicObserver<String> mObserver; 80 private final DropBoxManager mDropBox; 81 private final String mCookie; 82 83 private final long mBucketDuration; 84 private final boolean mOnlyTags; 85 86 private long mPersistThresholdBytes = 2 * MB_IN_BYTES; 87 private NetworkStats mLastSnapshot; 88 89 private final NetworkStatsCollection mPending; 90 private final NetworkStatsCollection mSinceBoot; 91 92 private final CombiningRewriter mPendingRewriter; 93 94 private WeakReference<NetworkStatsCollection> mComplete; 95 96 /** 97 * Non-persisted recorder, with only one bucket. Used by {@link NetworkStatsObservers}. 98 */ 99 public NetworkStatsRecorder() { 100 mRotator = null; 101 mObserver = null; 102 mDropBox = null; 103 mCookie = null; 104 105 // set the bucket big enough to have all data in one bucket, but allow some 106 // slack to avoid overflow 107 mBucketDuration = YEAR_IN_MILLIS; 108 mOnlyTags = false; 109 110 mPending = null; 111 mSinceBoot = new NetworkStatsCollection(mBucketDuration); 112 113 mPendingRewriter = null; 114 } 115 116 /** 117 * Persisted recorder. 118 */ 119 public NetworkStatsRecorder(FileRotator rotator, NonMonotonicObserver<String> observer, 120 DropBoxManager dropBox, String cookie, long bucketDuration, boolean onlyTags) { 121 mRotator = checkNotNull(rotator, "missing FileRotator"); 122 mObserver = checkNotNull(observer, "missing NonMonotonicObserver"); 123 mDropBox = checkNotNull(dropBox, "missing DropBoxManager"); 124 mCookie = cookie; 125 126 mBucketDuration = bucketDuration; 127 mOnlyTags = onlyTags; 128 129 mPending = new NetworkStatsCollection(bucketDuration); 130 mSinceBoot = new NetworkStatsCollection(bucketDuration); 131 132 mPendingRewriter = new CombiningRewriter(mPending); 133 } 134 135 public void setPersistThreshold(long thresholdBytes) { 136 if (LOGV) Slog.v(TAG, "setPersistThreshold() with " + thresholdBytes); 137 mPersistThresholdBytes = MathUtils.constrain( 138 thresholdBytes, 1 * KB_IN_BYTES, 100 * MB_IN_BYTES); 139 } 140 141 public void resetLocked() { 142 mLastSnapshot = null; 143 if (mPending != null) { 144 mPending.reset(); 145 } 146 if (mSinceBoot != null) { 147 mSinceBoot.reset(); 148 } 149 if (mComplete != null) { 150 mComplete.clear(); 151 } 152 } 153 154 public NetworkStats.Entry getTotalSinceBootLocked(NetworkTemplate template) { 155 return mSinceBoot.getSummary(template, Long.MIN_VALUE, Long.MAX_VALUE, 156 NetworkStatsAccess.Level.DEVICE, Binder.getCallingUid()).getTotal(null); 157 } 158 159 public NetworkStatsCollection getSinceBoot() { 160 return mSinceBoot; 161 } 162 163 /** 164 * Load complete history represented by {@link FileRotator}. Caches 165 * internally as a {@link WeakReference}, and updated with future 166 * {@link #recordSnapshotLocked(NetworkStats, Map, long)} snapshots as long 167 * as reference is valid. 168 */ 169 public NetworkStatsCollection getOrLoadCompleteLocked() { 170 checkNotNull(mRotator, "missing FileRotator"); 171 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 172 if (res == null) { 173 res = loadLocked(Long.MIN_VALUE, Long.MAX_VALUE); 174 mComplete = new WeakReference<NetworkStatsCollection>(res); 175 } 176 return res; 177 } 178 179 public NetworkStatsCollection getOrLoadPartialLocked(long start, long end) { 180 checkNotNull(mRotator, "missing FileRotator"); 181 NetworkStatsCollection res = mComplete != null ? mComplete.get() : null; 182 if (res == null) { 183 res = loadLocked(start, end); 184 } 185 return res; 186 } 187 188 private NetworkStatsCollection loadLocked(long start, long end) { 189 if (LOGD) Slog.d(TAG, "loadLocked() reading from disk for " + mCookie); 190 final NetworkStatsCollection res = new NetworkStatsCollection(mBucketDuration); 191 try { 192 mRotator.readMatching(res, start, end); 193 res.recordCollection(mPending); 194 } catch (IOException e) { 195 Log.wtf(TAG, "problem completely reading network stats", e); 196 recoverFromWtf(); 197 } catch (OutOfMemoryError e) { 198 Log.wtf(TAG, "problem completely reading network stats", e); 199 recoverFromWtf(); 200 } 201 return res; 202 } 203 204 /** 205 * Record any delta that occurred since last {@link NetworkStats} snapshot, 206 * using the given {@link Map} to identify network interfaces. First 207 * snapshot is considered bootstrap, and is not counted as delta. 208 * 209 * @param vpnArray Optional info about the currently active VPN, if any. This is used to 210 * redistribute traffic from the VPN app to the underlying responsible apps. 211 * This should always be set to null if the provided snapshot is aggregated 212 * across all UIDs (e.g. contains UID_ALL buckets), regardless of VPN state. 213 */ 214 public void recordSnapshotLocked(NetworkStats snapshot, 215 Map<String, NetworkIdentitySet> ifaceIdent, @Nullable VpnInfo[] vpnArray, 216 long currentTimeMillis) { 217 final HashSet<String> unknownIfaces = Sets.newHashSet(); 218 219 // skip recording when snapshot missing 220 if (snapshot == null) return; 221 222 // assume first snapshot is bootstrap and don't record 223 if (mLastSnapshot == null) { 224 mLastSnapshot = snapshot; 225 return; 226 } 227 228 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 229 230 final NetworkStats delta = NetworkStats.subtract( 231 snapshot, mLastSnapshot, mObserver, mCookie); 232 final long end = currentTimeMillis; 233 final long start = end - delta.getElapsedRealtime(); 234 235 if (vpnArray != null) { 236 for (VpnInfo info : vpnArray) { 237 delta.migrateTun(info.ownerUid, info.vpnIface, info.primaryUnderlyingIface); 238 } 239 } 240 241 NetworkStats.Entry entry = null; 242 for (int i = 0; i < delta.size(); i++) { 243 entry = delta.getValues(i, entry); 244 final NetworkIdentitySet ident = ifaceIdent.get(entry.iface); 245 if (ident == null) { 246 unknownIfaces.add(entry.iface); 247 continue; 248 } 249 250 // skip when no delta occurred 251 if (entry.isEmpty()) continue; 252 253 // only record tag data when requested 254 if ((entry.tag == TAG_NONE) != mOnlyTags) { 255 if (mPending != null) { 256 mPending.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 257 } 258 259 // also record against boot stats when present 260 if (mSinceBoot != null) { 261 mSinceBoot.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 262 } 263 264 // also record against complete dataset when present 265 if (complete != null) { 266 complete.recordData(ident, entry.uid, entry.set, entry.tag, start, end, entry); 267 } 268 } 269 } 270 271 mLastSnapshot = snapshot; 272 273 if (LOGV && unknownIfaces.size() > 0) { 274 Slog.w(TAG, "unknown interfaces " + unknownIfaces + ", ignoring those stats"); 275 } 276 } 277 278 /** 279 * Consider persisting any pending deltas, if they are beyond 280 * {@link #mPersistThresholdBytes}. 281 */ 282 public void maybePersistLocked(long currentTimeMillis) { 283 checkNotNull(mRotator, "missing FileRotator"); 284 final long pendingBytes = mPending.getTotalBytes(); 285 if (pendingBytes >= mPersistThresholdBytes) { 286 forcePersistLocked(currentTimeMillis); 287 } else { 288 mRotator.maybeRotate(currentTimeMillis); 289 } 290 } 291 292 /** 293 * Force persisting any pending deltas. 294 */ 295 public void forcePersistLocked(long currentTimeMillis) { 296 checkNotNull(mRotator, "missing FileRotator"); 297 if (mPending.isDirty()) { 298 if (LOGD) Slog.d(TAG, "forcePersistLocked() writing for " + mCookie); 299 try { 300 mRotator.rewriteActive(mPendingRewriter, currentTimeMillis); 301 mRotator.maybeRotate(currentTimeMillis); 302 mPending.reset(); 303 } catch (IOException e) { 304 Log.wtf(TAG, "problem persisting pending stats", e); 305 recoverFromWtf(); 306 } catch (OutOfMemoryError e) { 307 Log.wtf(TAG, "problem persisting pending stats", e); 308 recoverFromWtf(); 309 } 310 } 311 } 312 313 /** 314 * Remove the given UID from all {@link FileRotator} history, migrating it 315 * to {@link TrafficStats#UID_REMOVED}. 316 */ 317 public void removeUidsLocked(int[] uids) { 318 if (mRotator != null) { 319 try { 320 // Rewrite all persisted data to migrate UID stats 321 mRotator.rewriteAll(new RemoveUidRewriter(mBucketDuration, uids)); 322 } catch (IOException e) { 323 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 324 recoverFromWtf(); 325 } catch (OutOfMemoryError e) { 326 Log.wtf(TAG, "problem removing UIDs " + Arrays.toString(uids), e); 327 recoverFromWtf(); 328 } 329 } 330 331 // Remove any pending stats 332 if (mPending != null) { 333 mPending.removeUids(uids); 334 } 335 if (mSinceBoot != null) { 336 mSinceBoot.removeUids(uids); 337 } 338 339 // Clear UID from current stats snapshot 340 if (mLastSnapshot != null) { 341 mLastSnapshot = mLastSnapshot.withoutUids(uids); 342 } 343 344 final NetworkStatsCollection complete = mComplete != null ? mComplete.get() : null; 345 if (complete != null) { 346 complete.removeUids(uids); 347 } 348 } 349 350 /** 351 * Rewriter that will combine current {@link NetworkStatsCollection} values 352 * with anything read from disk, and write combined set to disk. Clears the 353 * original {@link NetworkStatsCollection} when finished writing. 354 */ 355 private static class CombiningRewriter implements FileRotator.Rewriter { 356 private final NetworkStatsCollection mCollection; 357 358 public CombiningRewriter(NetworkStatsCollection collection) { 359 mCollection = checkNotNull(collection, "missing NetworkStatsCollection"); 360 } 361 362 @Override 363 public void reset() { 364 // ignored 365 } 366 367 @Override 368 public void read(InputStream in) throws IOException { 369 mCollection.read(in); 370 } 371 372 @Override 373 public boolean shouldWrite() { 374 return true; 375 } 376 377 @Override 378 public void write(OutputStream out) throws IOException { 379 mCollection.write(new DataOutputStream(out)); 380 mCollection.reset(); 381 } 382 } 383 384 /** 385 * Rewriter that will remove any {@link NetworkStatsHistory} attributed to 386 * the requested UID, only writing data back when modified. 387 */ 388 public static class RemoveUidRewriter implements FileRotator.Rewriter { 389 private final NetworkStatsCollection mTemp; 390 private final int[] mUids; 391 392 public RemoveUidRewriter(long bucketDuration, int[] uids) { 393 mTemp = new NetworkStatsCollection(bucketDuration); 394 mUids = uids; 395 } 396 397 @Override 398 public void reset() { 399 mTemp.reset(); 400 } 401 402 @Override 403 public void read(InputStream in) throws IOException { 404 mTemp.read(in); 405 mTemp.clearDirty(); 406 mTemp.removeUids(mUids); 407 } 408 409 @Override 410 public boolean shouldWrite() { 411 return mTemp.isDirty(); 412 } 413 414 @Override 415 public void write(OutputStream out) throws IOException { 416 mTemp.write(new DataOutputStream(out)); 417 } 418 } 419 420 public void importLegacyNetworkLocked(File file) throws IOException { 421 checkNotNull(mRotator, "missing FileRotator"); 422 423 // legacy file still exists; start empty to avoid double importing 424 mRotator.deleteAll(); 425 426 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 427 collection.readLegacyNetwork(file); 428 429 final long startMillis = collection.getStartMillis(); 430 final long endMillis = collection.getEndMillis(); 431 432 if (!collection.isEmpty()) { 433 // process legacy data, creating active file at starting time, then 434 // using end time to possibly trigger rotation. 435 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 436 mRotator.maybeRotate(endMillis); 437 } 438 } 439 440 public void importLegacyUidLocked(File file) throws IOException { 441 checkNotNull(mRotator, "missing FileRotator"); 442 443 // legacy file still exists; start empty to avoid double importing 444 mRotator.deleteAll(); 445 446 final NetworkStatsCollection collection = new NetworkStatsCollection(mBucketDuration); 447 collection.readLegacyUid(file, mOnlyTags); 448 449 final long startMillis = collection.getStartMillis(); 450 final long endMillis = collection.getEndMillis(); 451 452 if (!collection.isEmpty()) { 453 // process legacy data, creating active file at starting time, then 454 // using end time to possibly trigger rotation. 455 mRotator.rewriteActive(new CombiningRewriter(collection), startMillis); 456 mRotator.maybeRotate(endMillis); 457 } 458 } 459 460 public void dumpLocked(IndentingPrintWriter pw, boolean fullHistory) { 461 if (mPending != null) { 462 pw.print("Pending bytes: "); pw.println(mPending.getTotalBytes()); 463 } 464 if (fullHistory) { 465 pw.println("Complete history:"); 466 getOrLoadCompleteLocked().dump(pw); 467 } else { 468 pw.println("History since boot:"); 469 mSinceBoot.dump(pw); 470 } 471 } 472 473 public void writeToProtoLocked(ProtoOutputStream proto, long tag) { 474 final long start = proto.start(tag); 475 if (mPending != null) { 476 proto.write(NetworkStatsRecorderProto.PENDING_TOTAL_BYTES, mPending.getTotalBytes()); 477 } 478 getOrLoadCompleteLocked().writeToProto(proto, NetworkStatsRecorderProto.COMPLETE_HISTORY); 479 proto.end(start); 480 } 481 482 public void dumpCheckin(PrintWriter pw, long start, long end) { 483 // Only load and dump stats from the requested window 484 getOrLoadPartialLocked(start, end).dumpCheckin(pw, start, end); 485 } 486 487 /** 488 * Recover from {@link FileRotator} failure by dumping state to 489 * {@link DropBoxManager} and deleting contents. 490 */ 491 private void recoverFromWtf() { 492 if (DUMP_BEFORE_DELETE) { 493 final ByteArrayOutputStream os = new ByteArrayOutputStream(); 494 try { 495 mRotator.dumpAll(os); 496 } catch (IOException e) { 497 // ignore partial contents 498 os.reset(); 499 } finally { 500 IoUtils.closeQuietly(os); 501 } 502 mDropBox.addData(TAG_NETSTATS_DUMP, os.toByteArray(), 0); 503 } 504 505 mRotator.deleteAll(); 506 } 507 } 508