Home | History | Annotate | Download | only in result_tools 
      1 # Copyright 2017 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """
      6 This is a utility to build an html page based on the directory summaries
      7 collected during the test.
      8 """
      9 
     10 import os
     11 import re
     12 
     13 import common
     14 from autotest_lib.client.bin.result_tools import utils_lib
     15 from autotest_lib.client.common_lib import global_config
     16 
     17 
     18 CONFIG = global_config.global_config
     19 # Base url to open a file from Google Storage
     20 GS_FILE_BASE_URL = CONFIG.get_config_value('CROS', 'gs_file_base_url')
     21 
     22 # Default width of `size_trimmed_width`. If throttle is not applied, the block
     23 # of `size_trimmed_width` will be set to minimum to make the view more compact.
     24 DEFAULT_SIZE_TRIMMED_WIDTH = 50
     25 
     26 DEFAULT_RESULT_SUMMARY_NAME = 'result_summary.html'
     27 
     28 DIR_SUMMARY_PATTERN = 'dir_summary_\d+.json'
     29 
     30 # ==================================================
     31 # Following are key names used in the html templates:
     32 
     33 CSS = 'css'
     34 DIRS = 'dirs'
     35 GS_FILE_BASE_URL_KEY = 'gs_file_base_url'
     36 INDENTATION_KEY = 'indentation'
     37 JAVASCRIPT = 'javascript'
     38 JOB_DIR = 'job_dir'
     39 NAME = 'name'
     40 PATH = 'path'
     41 
     42 SIZE_CLIENT_COLLECTED = 'size_client_collected'
     43 
     44 SIZE_INFO = 'size_info'
     45 SIZE_ORIGINAL = 'size_original'
     46 SIZE_PERCENT = 'size_percent'
     47 SIZE_PERCENT_CLASS = 'size_percent_class'
     48 SIZE_PERCENT_CLASS_REGULAR = 'size_percent'
     49 SIZE_PERCENT_CLASS_TOP = 'top_size_percent'
     50 SIZE_SUMMARY = 'size_summary'
     51 SIZE_TRIMMED = 'size_trimmed'
     52 
     53 # Width of `size_trimmed` block`
     54 SIZE_TRIMMED_WIDTH = 'size_trimmed_width'
     55 
     56 SUBDIRS = 'subdirs'
     57 SUMMARY_TREE = 'summary_tree'
     58 # ==================================================
     59 
     60 # Text to show when test result is not throttled.
     61 NOT_THROTTLED = '(Not throttled)'
     62 
     63 
     64 PAGE_TEMPLATE = """
     65 <!DOCTYPE html>
     66   <html>
     67     <body onload="init()">
     68       <h3>Summary of test results</h3>
     69 %(size_summary)s
     70       <p>
     71       <b>
     72         Display format of a file or directory:
     73       </b>
     74       </p>
     75       <p>
     76         <span class="size_percent" style="width:auto">
     77           [percentage of size in the parent directory]
     78         </span>
     79         <span class="size_original" style="width:auto">
     80           [original size]
     81         </span>
     82         <span class="size_trimmed" style="width:auto">
     83           [size after throttling (empty if not throttled)]
     84         </span>
     85         [file name (<strike>strikethrough</strike> if file was deleted due to
     86             throttling)]
     87       </p>
     88 
     89       <button onclick="expandAll();">Expand All</button>
     90       <button onclick="collapseAll();">Collapse All</button>
     91 
     92 %(summary_tree)s
     93 
     94 %(css)s
     95 %(javascript)s
     96 
     97     </body>
     98 </html>
     99 """
    100 
    101 CSS_TEMPLATE = """
    102 <style>
    103   body {
    104       font-family: Arial;
    105   }
    106 
    107   td.table_header {
    108       font-weight: normal;
    109   }
    110 
    111   span.size_percent {
    112       color: #e8773e;
    113       display: inline-block;
    114       font-size: 75%%;
    115       text-align: right;
    116       width: 35px;
    117   }
    118 
    119   span.top_size_percent {
    120       color: #e8773e;
    121       background-color: yellow;
    122       display: inline-block;
    123       font-size: 75%%;
    124       fount-weight: bold;
    125       text-align: right;
    126       width: 35px;
    127   }
    128 
    129   span.size_original {
    130       color: sienna;
    131       display: inline-block;
    132       font-size: 75%%;
    133       text-align: right;
    134       width: 50px;
    135   }
    136 
    137   span.size_trimmed {
    138       color: green;
    139       display: inline-block;
    140       font-size: 75%%;
    141       text-align: right;
    142       width: %(size_trimmed_width)dpx;
    143   }
    144 
    145   ul.tree li {
    146       list-style-type: none;
    147       position: relative;
    148   }
    149 
    150   ul.tree li ul {
    151       display: none;
    152   }
    153 
    154   ul.tree li.open > ul {
    155       display: block;
    156   }
    157 
    158   ul.tree li a {
    159     color: black;
    160     text-decoration: none;
    161   }
    162 
    163   ul.tree li a.file {
    164     color: blue;
    165     text-decoration: underline;
    166   }
    167 
    168   ul.tree li a:before {
    169       height: 1em;
    170       padding:0 .1em;
    171       font-size: .8em;
    172       display: block;
    173       position: absolute;
    174       left: -1.3em;
    175       top: .2em;
    176   }
    177 
    178   ul.tree li > a:not(:last-child):before {
    179       content: '+';
    180   }
    181 
    182   ul.tree li.open > a:not(:last-child):before {
    183       content: '-';
    184   }
    185 </style>
    186 """
    187 
    188 JAVASCRIPT_TEMPLATE = """
    189 <script>
    190 function init() {
    191     var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
    192     for(var i = 0; i < tree.length; i++){
    193         tree[i].addEventListener('click', function(e) {
    194             var parent = e.target.parentElement;
    195             var classList = parent.classList;
    196             if(classList.contains("open")) {
    197                 classList.remove('open');
    198                 var opensubs = parent.querySelectorAll(':scope .open');
    199                 for(var i = 0; i < opensubs.length; i++){
    200                     opensubs[i].classList.remove('open');
    201                 }
    202             } else {
    203                 classList.add('open');
    204             }
    205         });
    206     }
    207 }
    208 
    209 function expandAll() {
    210     var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
    211     for(var i = 0; i < tree.length; i++){
    212         var classList = tree[i].parentElement.classList;
    213         if(classList.contains("close")) {
    214             classList.remove('close');
    215         }
    216         classList.add('open');
    217     }
    218 }
    219 
    220 function collapseAll() {
    221     var tree = document.querySelectorAll('ul.tree a:not(:last-child)');
    222     for(var i = 0; i < tree.length; i++){
    223         var classList = tree[i].parentElement.classList;
    224         if(classList.contains("open")) {
    225             classList.remove('open');
    226         }
    227         classList.add('close');
    228     }
    229 }
    230 
    231 // If the current url has `gs_url`, it means the file is opened from Google
    232 // Storage.
    233 var gs_url = 'apidata.googleusercontent.com';
    234 // Base url to open a file from Google Storage
    235 var gs_file_base_url = '%(gs_file_base_url)s'
    236 // Path to the result.
    237 var job_dir = '%(job_dir)s'
    238 
    239 function openFile(path) {
    240     if(window.location.href.includes(gs_url)) {
    241         url = gs_file_base_url + job_dir + '/' + path.substring(3);
    242     } else {
    243         url = window.location.href + '/' + path;
    244     }
    245     window.open(url, '_blank');
    246 }
    247 </script>
    248 """
    249 
    250 SIZE_SUMMARY_TEMPLATE = """
    251 <table>
    252   <tr>
    253     <td class="table_header">Results collected from test device: </td>
    254     <td><span>%(size_client_collected)s</span> </td>
    255   </tr>
    256   <tr>
    257     <td class="table_header">Original size of test results:</td>
    258     <td>
    259       <span class="size_original" style="font-size:100%%;width:auto">
    260         %(size_original)s
    261       </span>
    262     </td>
    263   </tr>
    264   <tr>
    265     <td class="table_header">Size of test results after throttling:</td>
    266     <td>
    267       <span class="size_trimmed" style="font-size:100%%;width:auto">
    268         %(size_trimmed)s
    269       </span>
    270     </td>
    271   </tr>
    272 </table>
    273 """
    274 
    275 SIZE_INFO_TEMPLATE = """
    276 %(indentation)s<span class="%(size_percent_class)s">%(size_percent)s</span>
    277 %(indentation)s<span class="size_original">%(size_original)s</span>
    278 %(indentation)s<span class="size_trimmed">%(size_trimmed)s</span> """
    279 
    280 FILE_ENTRY_TEMPLATE = """
    281 %(indentation)s<li>
    282 %(indentation)s\t<div>
    283 %(size_info)s
    284 %(indentation)s\t\t<a class="file" href="javascript:openFile('%(path)s');" >
    285 %(indentation)s\t\t\t%(name)s
    286 %(indentation)s\t\t</a>
    287 %(indentation)s\t</div>
    288 %(indentation)s</li>"""
    289 
    290 DELETED_FILE_ENTRY_TEMPLATE = """
    291 %(indentation)s<li>
    292 %(indentation)s\t<div>
    293 %(size_info)s
    294 %(indentation)s\t\t<strike>%(name)s</strike>
    295 %(indentation)s\t</div>
    296 %(indentation)s</li>"""
    297 
    298 DIR_ENTRY_TEMPLATE = """
    299 %(indentation)s<li><a>%(size_info)s %(name)s</a>
    300 %(subdirs)s
    301 %(indentation)s</li>"""
    302 
    303 SUBDIRS_WRAPPER_TEMPLATE = """
    304 %(indentation)s<ul class="tree">
    305 %(dirs)s
    306 %(indentation)s</ul>"""
    307 
    308 INDENTATION = '\t'
    309 
    310 def _get_size_percent(size_original, total_bytes):
    311     """Get the percentage of file size in the parent directory before throttled.
    312 
    313     @param size_original: Original size of the file, in bytes.
    314     @param total_bytes: Total size of all files under the parent directory, in
    315             bytes.
    316     @return: A formatted string of the percentage of file size in the parent
    317             directory before throttled.
    318     """
    319     if total_bytes == 0:
    320         return '0%'
    321     return '%.1f%%' % (100*float(size_original)/total_bytes)
    322 
    323 
    324 def _get_dirs_html(dirs, parent_path, total_bytes, indentation):
    325     """Get the html string for the given directory.
    326 
    327     @param dirs: A list of ResultInfo.
    328     @param parent_path: Path to the parent directory.
    329     @param total_bytes: Total of the original size of files in the given
    330             directories in bytes.
    331     @param indentation: Indentation to be used for the html.
    332     """
    333     if not dirs:
    334         return ''
    335     summary_html = ''
    336     top_size_limit = max([entry.original_size for entry in dirs])
    337     # A map between file name to ResultInfo that contains the summary of the
    338     # file.
    339     entries = dict((entry.keys()[0], entry) for entry in dirs)
    340     for name in sorted(entries.keys()):
    341         entry = entries[name]
    342         if not entry.is_dir and re.match(DIR_SUMMARY_PATTERN, name):
    343             # Do not include directory summary json files in the html, as they
    344             # will be deleted.
    345             continue
    346 
    347         size_data = {SIZE_PERCENT: _get_size_percent(entry.original_size,
    348                                                      total_bytes),
    349                      SIZE_ORIGINAL:
    350                         utils_lib.get_size_string(entry.original_size),
    351                      SIZE_TRIMMED:
    352                         utils_lib.get_size_string(entry.trimmed_size),
    353                      INDENTATION_KEY: indentation + 2*INDENTATION}
    354         if entry.original_size < top_size_limit:
    355             size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_REGULAR
    356         else:
    357             size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_TOP
    358         if entry.trimmed_size == entry.original_size:
    359             size_data[SIZE_TRIMMED] = ''
    360 
    361         entry_path = '%s/%s' % (parent_path, name)
    362         if not entry.is_dir:
    363             # This is a file
    364             data = {NAME: name,
    365                     PATH: entry_path,
    366                     SIZE_INFO: SIZE_INFO_TEMPLATE % size_data,
    367                     INDENTATION_KEY: indentation}
    368             if entry.original_size > 0 and entry.trimmed_size == 0:
    369                 summary_html += DELETED_FILE_ENTRY_TEMPLATE % data
    370             else:
    371                 summary_html += FILE_ENTRY_TEMPLATE % data
    372         else:
    373             subdir_total_size = entry.original_size
    374             sub_indentation = indentation + INDENTATION
    375             subdirs_html = (
    376                     SUBDIRS_WRAPPER_TEMPLATE %
    377                     {DIRS: _get_dirs_html(
    378                             entry.files, entry_path, subdir_total_size,
    379                             sub_indentation),
    380                      INDENTATION_KEY: indentation})
    381             data = {NAME: entry.name,
    382                     SIZE_INFO: SIZE_INFO_TEMPLATE % size_data,
    383                     SUBDIRS: subdirs_html,
    384                     INDENTATION_KEY: indentation}
    385             summary_html += DIR_ENTRY_TEMPLATE % data
    386     return summary_html
    387 
    388 
    389 def build(client_collected_bytes, summary, html_file):
    390     """Generate an HTML file to visualize the given directory summary.
    391 
    392     @param client_collected_bytes: The total size of results collected from
    393             the DUT. The number can be larger than the total file size of the
    394             given path, as files can be overwritten or removed.
    395     @param summary: A ResultInfo instance containing the directory summary.
    396     @param html_file: Path to save the html file to.
    397     """
    398     size_original = summary.original_size
    399     size_trimmed = summary.trimmed_size
    400     size_summary_data = {SIZE_CLIENT_COLLECTED:
    401                              utils_lib.get_size_string(client_collected_bytes),
    402                          SIZE_ORIGINAL:
    403                              utils_lib.get_size_string(size_original),
    404                          SIZE_TRIMMED:
    405                              utils_lib.get_size_string(size_trimmed)}
    406     size_trimmed_width = DEFAULT_SIZE_TRIMMED_WIDTH
    407     if size_original == size_trimmed:
    408         size_summary_data[SIZE_TRIMMED] = NOT_THROTTLED
    409         size_trimmed_width = 0
    410 
    411     size_summary = SIZE_SUMMARY_TEMPLATE % size_summary_data
    412 
    413     indentation = INDENTATION
    414     dirs_html = _get_dirs_html(
    415             summary.files, '..', size_original, indentation + INDENTATION)
    416     summary_tree = SUBDIRS_WRAPPER_TEMPLATE % {DIRS: dirs_html,
    417                                                INDENTATION_KEY: indentation}
    418 
    419     # job_dir is the path between Autotest `results` folder and the summary html
    420     # file, e.g., 123-debug_user/host1. Assume it always contains 2 levels.
    421     job_dir_sections = html_file.split(os.sep)[:-1]
    422     try:
    423         job_dir = '/'.join(job_dir_sections[
    424                 (job_dir_sections.index('results')+1):])
    425     except ValueError:
    426         # 'results' is not in the path, default to two levels up of the summary
    427         # file.
    428         job_dir = '/'.join(job_dir_sections[-2:])
    429 
    430     javascript = (JAVASCRIPT_TEMPLATE %
    431                   {GS_FILE_BASE_URL_KEY: GS_FILE_BASE_URL,
    432                    JOB_DIR: job_dir})
    433     css = CSS_TEMPLATE % {SIZE_TRIMMED_WIDTH: size_trimmed_width}
    434     html = PAGE_TEMPLATE % {SIZE_SUMMARY: size_summary,
    435                             SUMMARY_TREE: summary_tree,
    436                             CSS: css,
    437                             JAVASCRIPT: javascript}
    438     with open(html_file, 'w') as f:
    439         f.write(html)
    440