Home | History | Annotate | Download | only in procstatlog
      1 #!/usr/bin/python
      2 #
      3 # Copyright (C) 2010 The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #      http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 import cgi
     18 import csv
     19 import json
     20 import math
     21 import os
     22 import re
     23 import sys
     24 import time
     25 import urllib
     26 
     27 """Interpret output from procstatlog and write an HTML report file."""
     28 
     29 
     30 # TODO: Rethink dygraph-combined.js source URL?
     31 PAGE_BEGIN = """
     32 <html><head>
     33 <title>%(filename)s</title>
     34 <script type="text/javascript" src="http://www.corp.google.com/~egnor/no_crawl/dygraph-combined.js"></script>
     35 <script>
     36 var allCharts = [];
     37 var inDrawCallback = false;
     38 
     39 OnDraw = function(me, initial) {
     40     if (inDrawCallback || initial) return;
     41     inDrawCallback = true;
     42     var range = me.xAxisRange();
     43     for (var j = 0; j < allCharts.length; j++) {
     44         if (allCharts[j] == me) continue;
     45         allCharts[j].updateOptions({dateWindow: range});
     46     }
     47     inDrawCallback = false;
     48 }
     49 
     50 MakeChart = function(id, filename, options) {
     51     options.width = "75%%";
     52     options.xTicker = Dygraph.dateTicker;
     53     options.xValueFormatter = Dygraph.dateString_;
     54     options.xAxisLabelFormatter = Dygraph.dateAxisFormatter;
     55     options.drawCallback = OnDraw;
     56     allCharts.push(new Dygraph(document.getElementById(id), filename, options));
     57 }
     58 </script>
     59 </head><body>
     60 <p>
     61 <span style="font-size: 150%%">%(filename)s</span>
     62 - stat report generated by %(user)s on %(date)s</p>
     63 <table cellpadding=0 cellspacing=0 margin=0 border=0>
     64 """
     65 
     66 CHART = """
     67 <tr>
     68 <td valign=top width=25%%>%(label_html)s</td>
     69 <td id="%(id)s"> </td>
     70 </tr>
     71 <script>
     72 MakeChart(%(id_js)s, %(filename_js)s, %(options_js)s)
     73 
     74 </script>
     75 """
     76 
     77 SPACER = """
     78 <tr><td colspan=2 height=20> </td></tr>
     79 """
     80 
     81 TOTAL_CPU_LABEL = """
     82 <b style="font-size: 150%%">Total CPU</b><br>
     83 jiffies: <nobr>%(sys)d sys</nobr>, <nobr>%(user)d user</nobr>
     84 """
     85 
     86 CPU_SPEED_LABEL = """
     87 <nobr>average CPU speed</nobr>
     88 """
     89 
     90 CONTEXT_LABEL = """
     91 context: <nobr>%(switches)d switches</nobr>
     92 """
     93 
     94 FAULTS_LABEL = """
     95 <nobr>page faults:</nobr> <nobr>%(major)d major</nobr>
     96 """
     97 
     98 BINDER_LABEL = """
     99 binder: <nobr>%(calls)d calls</nobr>
    100 """
    101 
    102 PROC_CPU_LABEL = """
    103 <span style="font-size: 150%%">%(process)s</span> (%(pid)d)<br>
    104 jiffies: <nobr>%(sys)d sys</nobr>, <nobr>%(user)d user</nobr>
    105 </div>
    106 """
    107 
    108 YAFFS_LABEL = """
    109 <span style="font-size: 150%%">yaffs: %(partition)s</span><br>
    110 pages: <nobr>%(nPageReads)d read</nobr>,
    111 <nobr>%(nPageWrites)d written</nobr><br>
    112 blocks: <nobr>%(nBlockErasures)d erased</nobr>
    113 """
    114 
    115 DISK_LABEL = """
    116 <span style="font-size: 150%%">disk: %(device)s</span><br>
    117 sectors: <nobr>%(reads)d read</nobr>, <nobr>%(writes)d written</nobr>
    118 """
    119 
    120 DISK_TIME_LABEL = """
    121 msec: <nobr>%(msec)d waiting</nobr>
    122 """
    123 
    124 NET_LABEL = """
    125 <span style="font-size: 150%%">net: %(interface)s</span><br>
    126 bytes: <nobr>%(tx)d tx</nobr>,
    127 <nobr>%(rx)d rx</nobr>
    128 """
    129 
    130 PAGE_END = """
    131 </table></body></html>
    132 """
    133 
    134 
    135 def WriteChartData(titles, datasets, filename):
    136     writer = csv.writer(file(filename, "w"))
    137     writer.writerow(["Time"] + titles)
    138 
    139     merged_rows = {}
    140     for set_num, data in enumerate(datasets):
    141         for when, datum in data.iteritems():
    142             if type(datum) == tuple: datum = "%d/%d" % datum
    143             merged_rows.setdefault(when, {})[set_num] = datum
    144 
    145     num_cols = len(datasets)
    146     for when, values in sorted(merged_rows.iteritems()):
    147         msec = "%d" % (when * 1000)
    148         writer.writerow([msec] + [values.get(n, "") for n in range(num_cols)])
    149 
    150 
    151 def WriteOutput(history, log_filename, filename):
    152     out = []
    153 
    154     out.append(PAGE_BEGIN % {
    155         "filename": cgi.escape(log_filename),
    156         "user": cgi.escape(os.environ.get("USER", "unknown")),
    157         "date": cgi.escape(time.ctime()),
    158     })
    159 
    160     files_dir = "%s_files" % os.path.splitext(filename)[0]
    161     files_url = os.path.basename(files_dir)
    162     if not os.path.isdir(files_dir): os.makedirs(files_dir)
    163 
    164     sorted_history = sorted(history.iteritems())
    165     date_window = [1000 * sorted_history[1][0], 1000 * sorted_history[-1][0]]
    166 
    167     #
    168     # Output total CPU statistics
    169     #
    170 
    171     sys_jiffies = {}
    172     sys_user_jiffies = {}
    173     all_jiffies = {}
    174     total_sys = total_user = 0
    175 
    176     last_state = {}
    177     for when, state in sorted_history:
    178         last = last_state.get("/proc/stat:cpu", "").split()
    179         next = state.get("/proc/stat:cpu", "").split()
    180         if last and next:
    181             stime = sum([int(next[x]) - int(last[x]) for x in [2, 5, 6]])
    182             utime = sum([int(next[x]) - int(last[x]) for x in [0, 1]])
    183             idle = sum([int(next[x]) - int(last[x]) for x in [3, 4]])
    184             all = stime + utime + idle
    185             total_sys += stime
    186             total_user += utime
    187 
    188             sys_jiffies[when] = (stime, all)
    189             sys_user_jiffies[when] = (stime + utime, all)
    190             all_jiffies[when] = all
    191 
    192         last_state = state
    193 
    194     WriteChartData(
    195         ["sys", "sys+user"],
    196         [sys_jiffies, sys_user_jiffies],
    197         os.path.join(files_dir, "total_cpu.csv"))
    198 
    199     out.append(CHART % {
    200         "id": cgi.escape("total_cpu"),
    201         "id_js": json.write("total_cpu"),
    202         "label_html": TOTAL_CPU_LABEL % {"sys": total_sys, "user": total_user},
    203         "filename_js": json.write(files_url + "/total_cpu.csv"),
    204         "options_js": json.write({
    205             "colors": ["blue", "green"],
    206             "dateWindow": date_window,
    207             "fillGraph": True,
    208             "fractions": True,
    209             "height": 100,
    210             "valueRange": [0, 110],
    211         }),
    212     })
    213 
    214     #
    215     # Output CPU speed statistics
    216     #
    217 
    218     cpu_speed = {}
    219     speed_key = "/sys/devices/system/cpu/cpu0/cpufreq/stats/time_in_state:"
    220 
    221     last_state = {}
    222     for when, state in sorted_history:
    223         total_time = total_cycles = 0
    224         for key in state:
    225             if not key.startswith(speed_key): continue
    226 
    227             last = int(last_state.get(key, -1))
    228             next = int(state.get(key, -1))
    229             if last != -1 and next != -1:
    230                 speed = int(key[len(speed_key):])
    231                 total_time += next - last
    232                 total_cycles += (next - last) * speed
    233 
    234         if total_time > 0: cpu_speed[when] = total_cycles / total_time
    235         last_state = state
    236 
    237     WriteChartData(
    238         ["kHz"], [cpu_speed],
    239         os.path.join(files_dir, "cpu_speed.csv"))
    240 
    241     out.append(CHART % {
    242         "id": cgi.escape("cpu_speed"),
    243         "id_js": json.write("cpu_speed"),
    244         "label_html": CPU_SPEED_LABEL,
    245         "filename_js": json.write(files_url + "/cpu_speed.csv"),
    246         "options_js": json.write({
    247             "colors": ["navy"],
    248             "dateWindow": date_window,
    249             "fillGraph": True,
    250             "height": 50,
    251             "includeZero": True,
    252         }),
    253     })
    254 
    255     #
    256     # Output total context switch statistics
    257     #
    258 
    259     context_switches = {}
    260 
    261     last_state = {}
    262     for when, state in sorted_history:
    263         last = int(last_state.get("/proc/stat:ctxt", -1))
    264         next = int(state.get("/proc/stat:ctxt", -1))
    265         if last != -1 and next != -1: context_switches[when] = next - last
    266         last_state = state
    267 
    268     WriteChartData(
    269         ["switches"], [context_switches],
    270         os.path.join(files_dir, "context_switches.csv"))
    271 
    272     total_switches = sum(context_switches.values())
    273     out.append(CHART % {
    274         "id": cgi.escape("context_switches"),
    275         "id_js": json.write("context_switches"),
    276         "label_html": CONTEXT_LABEL % {"switches": total_switches},
    277         "filename_js": json.write(files_url + "/context_switches.csv"),
    278         "options_js": json.write({
    279             "colors": ["blue"],
    280             "dateWindow": date_window,
    281             "fillGraph": True,
    282             "height": 50,
    283             "includeZero": True,
    284         }),
    285     })
    286 
    287     #
    288     # Collect (no output yet) per-process CPU and major faults
    289     #
    290 
    291     process_name = {}
    292     process_start = {}
    293     process_sys = {}
    294     process_sys_user = {}
    295 
    296     process_faults = {}
    297     total_faults = {}
    298     max_faults = 0
    299 
    300     last_state = {}
    301     zero_stat = "0 (zero) Z 0 0 0 0 0 0 0 0 0 0 0 0"
    302     for when, state in sorted_history:
    303         for key in state:
    304             if not key.endswith("/stat"): continue
    305 
    306             last = last_state.get(key, zero_stat).split()
    307             next = state.get(key, "").split()
    308             if not next: continue
    309 
    310             pid = int(next[0])
    311             process_start.setdefault(pid, when)
    312             process_name[pid] = next[1][1:-1]
    313 
    314             all = all_jiffies.get(when, 0)
    315             if not all: continue
    316 
    317             faults = int(next[11]) - int(last[11])
    318             process_faults.setdefault(pid, {})[when] = faults
    319             tf = total_faults[when] = total_faults.get(when, 0) + faults
    320             max_faults = max(max_faults, tf)
    321 
    322             stime = int(next[14]) - int(last[14])
    323             utime = int(next[13]) - int(last[13])
    324             process_sys.setdefault(pid, {})[when] = (stime, all)
    325             process_sys_user.setdefault(pid, {})[when] = (stime + utime, all)
    326 
    327         last_state = state
    328 
    329     #
    330     # Output total major faults (sum over all processes)
    331     #
    332 
    333     WriteChartData(
    334         ["major"], [total_faults],
    335         os.path.join(files_dir, "total_faults.csv"))
    336 
    337     out.append(CHART % {
    338         "id": cgi.escape("total_faults"),
    339         "id_js": json.write("total_faults"),
    340         "label_html": FAULTS_LABEL % {"major": sum(total_faults.values())},
    341         "filename_js": json.write(files_url + "/total_faults.csv"),
    342         "options_js": json.write({
    343             "colors": ["gray"],
    344             "dateWindow": date_window,
    345             "fillGraph": True,
    346             "height": 50,
    347             "valueRange": [0, max_faults * 11 / 10],
    348         }),
    349     })
    350 
    351     #
    352     # Output binder transaactions
    353     #
    354 
    355     binder_calls = {}
    356 
    357     last_state = {}
    358     for when, state in sorted_history:
    359         last = int(last_state.get("/proc/binder/stats:BC_TRANSACTION", -1))
    360         next = int(state.get("/proc/binder/stats:BC_TRANSACTION", -1))
    361         if last != -1 and next != -1: binder_calls[when] = next - last
    362         last_state = state
    363 
    364     WriteChartData(
    365         ["calls"], [binder_calls],
    366         os.path.join(files_dir, "binder_calls.csv"))
    367 
    368     out.append(CHART % {
    369         "id": cgi.escape("binder_calls"),
    370         "id_js": json.write("binder_calls"),
    371         "label_html": BINDER_LABEL % {"calls": sum(binder_calls.values())},
    372         "filename_js": json.write(files_url + "/binder_calls.csv"),
    373         "options_js": json.write({
    374             "colors": ["green"],
    375             "dateWindow": date_window,
    376             "fillGraph": True,
    377             "height": 50,
    378             "includeZero": True,
    379         })
    380     })
    381 
    382     #
    383     # Output network interface statistics
    384     #
    385 
    386     if out[-1] != SPACER: out.append(SPACER)
    387 
    388     interface_rx = {}
    389     interface_tx = {}
    390     max_bytes = 0
    391 
    392     last_state = {}
    393     for when, state in sorted_history:
    394         for key in state:
    395             if not key.startswith("/proc/net/dev:"): continue
    396 
    397             last = last_state.get(key, "").split()
    398             next = state.get(key, "").split()
    399             if not (last and next): continue
    400 
    401             rx = int(next[0]) - int(last[0])
    402             tx = int(next[8]) - int(last[8])
    403             max_bytes = max(max_bytes, rx, tx)
    404 
    405             net, interface = key.split(":", 1)
    406             interface_rx.setdefault(interface, {})[when] = rx
    407             interface_tx.setdefault(interface, {})[when] = tx
    408 
    409         last_state = state
    410 
    411     for num, interface in enumerate(sorted(interface_rx.keys())):
    412         rx, tx = interface_rx[interface], interface_tx[interface]
    413         total_rx, total_tx = sum(rx.values()), sum(tx.values())
    414         if not (total_rx or total_tx): continue
    415 
    416         WriteChartData(
    417             ["rx", "tx"], [rx, tx],
    418             os.path.join(files_dir, "net%d.csv" % num))
    419 
    420         out.append(CHART % {
    421             "id": cgi.escape("net%d" % num),
    422             "id_js": json.write("net%d" % num),
    423             "label_html": NET_LABEL % {
    424                 "interface": cgi.escape(interface),
    425                 "rx": total_rx,
    426                 "tx": total_tx
    427             },
    428             "filename_js": json.write("%s/net%d.csv" % (files_url, num)),
    429             "options_js": json.write({
    430                 "colors": ["black", "purple"],
    431                 "dateWindow": date_window,
    432                 "fillGraph": True,
    433                 "height": 75,
    434                 "valueRange": [0, max_bytes * 11 / 10],
    435             })
    436         })
    437 
    438     #
    439     # Output YAFFS statistics
    440     #
    441 
    442     if out[-1] != SPACER: out.append(SPACER)
    443 
    444     yaffs_vars = ["nBlockErasures", "nPageReads", "nPageWrites"]
    445     partition_ops = {}
    446 
    447     last_state = {}
    448     for when, state in sorted_history:
    449         for key in state:
    450             if not key.startswith("/proc/yaffs:"): continue
    451 
    452             last = int(last_state.get(key, -1))
    453             next = int(state.get(key, -1))
    454             if last == -1 or next == -1: continue
    455 
    456             value = next - last
    457             yaffs, partition, var = key.split(":", 2)
    458             ops = partition_ops.setdefault(partition, {})
    459             if var in yaffs_vars:
    460                 ops.setdefault(var, {})[when] = value
    461 
    462         last_state = state
    463 
    464     for num, (partition, ops) in enumerate(sorted(partition_ops.iteritems())):
    465         totals = [sum(ops.get(var, {}).values()) for var in yaffs_vars]
    466         if not sum(totals): continue
    467 
    468         WriteChartData(
    469             yaffs_vars,
    470             [ops.get(var, {}) for var in yaffs_vars],
    471             os.path.join(files_dir, "yaffs%d.csv" % num))
    472 
    473         values = {"partition": partition}
    474         values.update(zip(yaffs_vars, totals))
    475         out.append(CHART % {
    476             "id": cgi.escape("yaffs%d" % num),
    477             "id_js": json.write("yaffs%d" % num),
    478             "label_html": YAFFS_LABEL % values,
    479             "filename_js": json.write("%s/yaffs%d.csv" % (files_url, num)),
    480             "options_js": json.write({
    481                 "colors": ["maroon", "gray", "teal"],
    482                 "dateWindow": date_window,
    483                 "fillGraph": True,
    484                 "height": 75,
    485                 "includeZero": True,
    486             })
    487         })
    488 
    489     #
    490     # Output non-YAFFS statistics
    491     #
    492 
    493     disk_reads = {}
    494     disk_writes = {}
    495     disk_msec = {}
    496     total_io = max_io = max_msec = 0
    497 
    498     last_state = {}
    499     for when, state in sorted_history:
    500         for key in state:
    501             if not key.startswith("/proc/diskstats:"): continue
    502 
    503             last = last_state.get(key, "").split()
    504             next = state.get(key, "").split()
    505             if not (last and next): continue
    506 
    507             reads = int(next[2]) - int(last[2])
    508             writes = int(next[6]) - int(last[6])
    509             msec = int(next[10]) - int(last[10])
    510             total_io += reads + writes
    511             max_io = max(max_io, reads, writes)
    512             max_msec = max(max_msec, msec)
    513 
    514             diskstats, device = key.split(":", 1)
    515             disk_reads.setdefault(device, {})[when] = reads
    516             disk_writes.setdefault(device, {})[when] = writes
    517             disk_msec.setdefault(device, {})[when] = msec
    518 
    519         last_state = state
    520 
    521     io_cutoff = total_io / 100
    522     for num, device in enumerate(sorted(disk_reads.keys())):
    523         if [d for d in disk_reads.keys()
    524             if d.startswith(device) and d != device]: continue
    525 
    526         reads, writes = disk_reads[device], disk_writes[device]
    527         total_reads, total_writes = sum(reads.values()), sum(writes.values())
    528         if total_reads + total_writes <= io_cutoff: continue
    529 
    530         WriteChartData(
    531             ["reads", "writes"], [reads, writes],
    532             os.path.join(files_dir, "disk%d.csv" % num))
    533 
    534         out.append(CHART % {
    535             "id": cgi.escape("disk%d" % num),
    536             "id_js": json.write("disk%d" % num),
    537             "label_html": DISK_LABEL % {
    538                 "device": cgi.escape(device),
    539                 "reads": total_reads,
    540                 "writes": total_writes,
    541             },
    542             "filename_js": json.write("%s/disk%d.csv" % (files_url, num)),
    543             "options_js": json.write({
    544                 "colors": ["gray", "teal"],
    545                 "dateWindow": date_window,
    546                 "fillGraph": True,
    547                 "height": 75,
    548                 "valueRange": [0, max_io * 11 / 10],
    549             }),
    550         })
    551 
    552         msec = disk_msec[device]
    553 
    554         WriteChartData(
    555             ["msec"], [msec],
    556             os.path.join(files_dir, "disk%d_time.csv" % num))
    557 
    558         out.append(CHART % {
    559             "id": cgi.escape("disk%d_time" % num),
    560             "id_js": json.write("disk%d_time" % num),
    561             "label_html": DISK_TIME_LABEL % {"msec": sum(msec.values())},
    562             "filename_js": json.write("%s/disk%d_time.csv" % (files_url, num)),
    563             "options_js": json.write({
    564                 "colors": ["blue"],
    565                 "dateWindow": date_window,
    566                 "fillGraph": True,
    567                 "height": 50,
    568                 "valueRange": [0, max_msec * 11 / 10],
    569             }),
    570         })
    571 
    572     #
    573     # Output per-process CPU and page faults collected earlier
    574     #
    575 
    576     cpu_cutoff = (total_sys + total_user) / 200
    577     faults_cutoff = sum(total_faults.values()) / 100
    578     for start, pid in sorted([(s, p) for p, s in process_start.iteritems()]):
    579         sys = sum([n for n, d in process_sys.get(pid, {}).values()])
    580         sys_user = sum([n for n, d in process_sys_user.get(pid, {}).values()])
    581         if sys_user <= cpu_cutoff: continue
    582 
    583         if out[-1] != SPACER: out.append(SPACER)
    584 
    585         WriteChartData(
    586             ["sys", "sys+user"],
    587             [process_sys.get(pid, {}), process_sys_user.get(pid, {})],
    588             os.path.join(files_dir, "proc%d.csv" % pid))
    589 
    590         out.append(CHART % {
    591             "id": cgi.escape("proc%d" % pid),
    592             "id_js": json.write("proc%d" % pid),
    593             "label_html": PROC_CPU_LABEL % {
    594                 "pid": pid,
    595                 "process": cgi.escape(process_name.get(pid, "(unknown)")),
    596                 "sys": sys,
    597                 "user": sys_user - sys,
    598             },
    599             "filename_js": json.write("%s/proc%d.csv" % (files_url, pid)),
    600             "options_js": json.write({
    601                 "colors": ["blue", "green"],
    602                 "dateWindow": date_window,
    603                 "fillGraph": True,
    604                 "fractions": True,
    605                 "height": 75,
    606                 "valueRange": [0, 110],
    607             }),
    608         })
    609 
    610         faults = sum(process_faults.get(pid, {}).values())
    611         if faults <= faults_cutoff: continue
    612 
    613         WriteChartData(
    614             ["major"], [process_faults.get(pid, {})],
    615             os.path.join(files_dir, "proc%d_faults.csv" % pid))
    616 
    617         out.append(CHART % {
    618             "id": cgi.escape("proc%d_faults" % pid),
    619             "id_js": json.write("proc%d_faults" % pid),
    620             "label_html": FAULTS_LABEL % {"major": faults},
    621             "filename_js": json.write("%s/proc%d_faults.csv" % (files_url, pid)),
    622             "options_js": json.write({
    623                 "colors": ["gray"],
    624                 "dateWindow": date_window,
    625                 "fillGraph": True,
    626                 "height": 50,
    627                 "valueRange": [0, max_faults * 11 / 10],
    628             }),
    629         })
    630 
    631     out.append(PAGE_END)
    632     file(filename, "w").write("\n".join(out))
    633 
    634 
    635 def main(argv):
    636     if len(argv) != 3:
    637         print >>sys.stderr, "usage: procstatreport.py procstat.log output.html"
    638         return 2
    639 
    640     history = {}
    641     current_state = {}
    642     scan_time = 0.0
    643 
    644     for line in file(argv[1]):
    645         if not line.endswith("\n"): continue
    646 
    647         parts = line.split(None, 2)
    648         if len(parts) < 2 or parts[1] not in "+-=":
    649             print >>sys.stderr, "Invalid input:", line
    650             sys.exit(1)
    651 
    652         name, op = parts[:2]
    653 
    654         if name == "T" and op == "+":  # timestamp: scan about to begin
    655             scan_time = float(line[4:])
    656             continue
    657 
    658         if name == "T" and op == "-":  # timestamp: scan complete
    659             time = (scan_time + float(line[4:])) / 2.0
    660             history[time] = dict(current_state)
    661 
    662         elif op == "-":
    663             if name in current_state: del current_state[name]
    664 
    665         else:
    666             current_state[name] = "".join(parts[2:]).strip()
    667 
    668     if len(history) < 2:
    669         print >>sys.stderr, "error: insufficient history to chart"
    670         return 1
    671 
    672     WriteOutput(history, argv[1], argv[2])
    673 
    674 
    675 if __name__ == "__main__":
    676     sys.exit(main(sys.argv))
    677