1 # Copyright 2017 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """ 6 This is a utility to build an html page based on the directory summaries 7 collected during the test. 8 """ 9 10 import os 11 import re 12 13 import common 14 from autotest_lib.client.bin.result_tools import utils_lib 15 from autotest_lib.client.common_lib import global_config 16 17 18 CONFIG = global_config.global_config 19 # Base url to open a file from Google Storage 20 GS_FILE_BASE_URL = CONFIG.get_config_value('CROS', 'gs_file_base_url') 21 22 # Default width of `size_trimmed_width`. If throttle is not applied, the block 23 # of `size_trimmed_width` will be set to minimum to make the view more compact. 24 DEFAULT_SIZE_TRIMMED_WIDTH = 50 25 26 DEFAULT_RESULT_SUMMARY_NAME = 'result_summary.html' 27 28 DIR_SUMMARY_PATTERN = 'dir_summary_\d+.json' 29 30 # ================================================== 31 # Following are key names used in the html templates: 32 33 CSS = 'css' 34 DIRS = 'dirs' 35 GS_FILE_BASE_URL_KEY = 'gs_file_base_url' 36 INDENTATION_KEY = 'indentation' 37 JAVASCRIPT = 'javascript' 38 JOB_DIR = 'job_dir' 39 NAME = 'name' 40 PATH = 'path' 41 42 SIZE_CLIENT_COLLECTED = 'size_client_collected' 43 44 SIZE_INFO = 'size_info' 45 SIZE_ORIGINAL = 'size_original' 46 SIZE_PERCENT = 'size_percent' 47 SIZE_PERCENT_CLASS = 'size_percent_class' 48 SIZE_PERCENT_CLASS_REGULAR = 'size_percent' 49 SIZE_PERCENT_CLASS_TOP = 'top_size_percent' 50 SIZE_SUMMARY = 'size_summary' 51 SIZE_TRIMMED = 'size_trimmed' 52 53 # Width of `size_trimmed` block` 54 SIZE_TRIMMED_WIDTH = 'size_trimmed_width' 55 56 SUBDIRS = 'subdirs' 57 SUMMARY_TREE = 'summary_tree' 58 # ================================================== 59 60 # Text to show when test result is not throttled. 61 NOT_THROTTLED = '(Not throttled)' 62 63 64 PAGE_TEMPLATE = """ 65 <!DOCTYPE html> 66 <html> 67 <body onload="init()"> 68 <h3>Summary of test results</h3> 69 %(size_summary)s 70 <p> 71 <b> 72 Display format of a file or directory: 73 </b> 74 </p> 75 <p> 76 <span class="size_percent" style="width:auto"> 77 [percentage of size in the parent directory] 78 </span> 79 <span class="size_original" style="width:auto"> 80 [original size] 81 </span> 82 <span class="size_trimmed" style="width:auto"> 83 [size after throttling (empty if not throttled)] 84 </span> 85 [file name (<strike>strikethrough</strike> if file was deleted due to 86 throttling)] 87 </p> 88 89 <button onclick="expandAll();">Expand All</button> 90 <button onclick="collapseAll();">Collapse All</button> 91 92 %(summary_tree)s 93 94 %(css)s 95 %(javascript)s 96 97 </body> 98 </html> 99 """ 100 101 CSS_TEMPLATE = """ 102 <style> 103 body { 104 font-family: Arial; 105 } 106 107 td.table_header { 108 font-weight: normal; 109 } 110 111 span.size_percent { 112 color: #e8773e; 113 display: inline-block; 114 font-size: 75%%; 115 text-align: right; 116 width: 35px; 117 } 118 119 span.top_size_percent { 120 color: #e8773e; 121 background-color: yellow; 122 display: inline-block; 123 font-size: 75%%; 124 fount-weight: bold; 125 text-align: right; 126 width: 35px; 127 } 128 129 span.size_original { 130 color: sienna; 131 display: inline-block; 132 font-size: 75%%; 133 text-align: right; 134 width: 50px; 135 } 136 137 span.size_trimmed { 138 color: green; 139 display: inline-block; 140 font-size: 75%%; 141 text-align: right; 142 width: %(size_trimmed_width)dpx; 143 } 144 145 ul.tree li { 146 list-style-type: none; 147 position: relative; 148 } 149 150 ul.tree li ul { 151 display: none; 152 } 153 154 ul.tree li.open > ul { 155 display: block; 156 } 157 158 ul.tree li a { 159 color: black; 160 text-decoration: none; 161 } 162 163 ul.tree li a.file { 164 color: blue; 165 text-decoration: underline; 166 } 167 168 ul.tree li a:before { 169 height: 1em; 170 padding:0 .1em; 171 font-size: .8em; 172 display: block; 173 position: absolute; 174 left: -1.3em; 175 top: .2em; 176 } 177 178 ul.tree li > a:not(:last-child):before { 179 content: '+'; 180 } 181 182 ul.tree li.open > a:not(:last-child):before { 183 content: '-'; 184 } 185 </style> 186 """ 187 188 JAVASCRIPT_TEMPLATE = """ 189 <script> 190 function init() { 191 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 192 for(var i = 0; i < tree.length; i++){ 193 tree[i].addEventListener('click', function(e) { 194 var parent = e.target.parentElement; 195 var classList = parent.classList; 196 if(classList.contains("open")) { 197 classList.remove('open'); 198 var opensubs = parent.querySelectorAll(':scope .open'); 199 for(var i = 0; i < opensubs.length; i++){ 200 opensubs[i].classList.remove('open'); 201 } 202 } else { 203 classList.add('open'); 204 } 205 }); 206 } 207 } 208 209 function expandAll() { 210 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 211 for(var i = 0; i < tree.length; i++){ 212 var classList = tree[i].parentElement.classList; 213 if(classList.contains("close")) { 214 classList.remove('close'); 215 } 216 classList.add('open'); 217 } 218 } 219 220 function collapseAll() { 221 var tree = document.querySelectorAll('ul.tree a:not(:last-child)'); 222 for(var i = 0; i < tree.length; i++){ 223 var classList = tree[i].parentElement.classList; 224 if(classList.contains("open")) { 225 classList.remove('open'); 226 } 227 classList.add('close'); 228 } 229 } 230 231 // If the current url has `gs_url`, it means the file is opened from Google 232 // Storage. 233 var gs_url = 'apidata.googleusercontent.com'; 234 // Base url to open a file from Google Storage 235 var gs_file_base_url = '%(gs_file_base_url)s' 236 // Path to the result. 237 var job_dir = '%(job_dir)s' 238 239 function openFile(path) { 240 if(window.location.href.includes(gs_url)) { 241 url = gs_file_base_url + job_dir + '/' + path.substring(3); 242 } else { 243 url = window.location.href + '/' + path; 244 } 245 window.open(url, '_blank'); 246 } 247 </script> 248 """ 249 250 SIZE_SUMMARY_TEMPLATE = """ 251 <table> 252 <tr> 253 <td class="table_header">Results collected from test device: </td> 254 <td><span>%(size_client_collected)s</span> </td> 255 </tr> 256 <tr> 257 <td class="table_header">Original size of test results:</td> 258 <td> 259 <span class="size_original" style="font-size:100%%;width:auto"> 260 %(size_original)s 261 </span> 262 </td> 263 </tr> 264 <tr> 265 <td class="table_header">Size of test results after throttling:</td> 266 <td> 267 <span class="size_trimmed" style="font-size:100%%;width:auto"> 268 %(size_trimmed)s 269 </span> 270 </td> 271 </tr> 272 </table> 273 """ 274 275 SIZE_INFO_TEMPLATE = """ 276 %(indentation)s<span class="%(size_percent_class)s">%(size_percent)s</span> 277 %(indentation)s<span class="size_original">%(size_original)s</span> 278 %(indentation)s<span class="size_trimmed">%(size_trimmed)s</span> """ 279 280 FILE_ENTRY_TEMPLATE = """ 281 %(indentation)s<li> 282 %(indentation)s\t<div> 283 %(size_info)s 284 %(indentation)s\t\t<a class="file" href="javascript:openFile('%(path)s');" > 285 %(indentation)s\t\t\t%(name)s 286 %(indentation)s\t\t</a> 287 %(indentation)s\t</div> 288 %(indentation)s</li>""" 289 290 DELETED_FILE_ENTRY_TEMPLATE = """ 291 %(indentation)s<li> 292 %(indentation)s\t<div> 293 %(size_info)s 294 %(indentation)s\t\t<strike>%(name)s</strike> 295 %(indentation)s\t</div> 296 %(indentation)s</li>""" 297 298 DIR_ENTRY_TEMPLATE = """ 299 %(indentation)s<li><a>%(size_info)s %(name)s</a> 300 %(subdirs)s 301 %(indentation)s</li>""" 302 303 SUBDIRS_WRAPPER_TEMPLATE = """ 304 %(indentation)s<ul class="tree"> 305 %(dirs)s 306 %(indentation)s</ul>""" 307 308 INDENTATION = '\t' 309 310 def _get_size_percent(size_original, total_bytes): 311 """Get the percentage of file size in the parent directory before throttled. 312 313 @param size_original: Original size of the file, in bytes. 314 @param total_bytes: Total size of all files under the parent directory, in 315 bytes. 316 @return: A formatted string of the percentage of file size in the parent 317 directory before throttled. 318 """ 319 if total_bytes == 0: 320 return '0%' 321 return '%.1f%%' % (100*float(size_original)/total_bytes) 322 323 324 def _get_dirs_html(dirs, parent_path, total_bytes, indentation): 325 """Get the html string for the given directory. 326 327 @param dirs: A list of ResultInfo. 328 @param parent_path: Path to the parent directory. 329 @param total_bytes: Total of the original size of files in the given 330 directories in bytes. 331 @param indentation: Indentation to be used for the html. 332 """ 333 if not dirs: 334 return '' 335 summary_html = '' 336 top_size_limit = max([entry.original_size for entry in dirs]) 337 # A map between file name to ResultInfo that contains the summary of the 338 # file. 339 entries = dict((entry.keys()[0], entry) for entry in dirs) 340 for name in sorted(entries.keys()): 341 entry = entries[name] 342 if not entry.is_dir and re.match(DIR_SUMMARY_PATTERN, name): 343 # Do not include directory summary json files in the html, as they 344 # will be deleted. 345 continue 346 347 size_data = {SIZE_PERCENT: _get_size_percent(entry.original_size, 348 total_bytes), 349 SIZE_ORIGINAL: 350 utils_lib.get_size_string(entry.original_size), 351 SIZE_TRIMMED: 352 utils_lib.get_size_string(entry.trimmed_size), 353 INDENTATION_KEY: indentation + 2*INDENTATION} 354 if entry.original_size < top_size_limit: 355 size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_REGULAR 356 else: 357 size_data[SIZE_PERCENT_CLASS] = SIZE_PERCENT_CLASS_TOP 358 if entry.trimmed_size == entry.original_size: 359 size_data[SIZE_TRIMMED] = '' 360 361 entry_path = '%s/%s' % (parent_path, name) 362 if not entry.is_dir: 363 # This is a file 364 data = {NAME: name, 365 PATH: entry_path, 366 SIZE_INFO: SIZE_INFO_TEMPLATE % size_data, 367 INDENTATION_KEY: indentation} 368 if entry.original_size > 0 and entry.trimmed_size == 0: 369 summary_html += DELETED_FILE_ENTRY_TEMPLATE % data 370 else: 371 summary_html += FILE_ENTRY_TEMPLATE % data 372 else: 373 subdir_total_size = entry.original_size 374 sub_indentation = indentation + INDENTATION 375 subdirs_html = ( 376 SUBDIRS_WRAPPER_TEMPLATE % 377 {DIRS: _get_dirs_html( 378 entry.files, entry_path, subdir_total_size, 379 sub_indentation), 380 INDENTATION_KEY: indentation}) 381 data = {NAME: entry.name, 382 SIZE_INFO: SIZE_INFO_TEMPLATE % size_data, 383 SUBDIRS: subdirs_html, 384 INDENTATION_KEY: indentation} 385 summary_html += DIR_ENTRY_TEMPLATE % data 386 return summary_html 387 388 389 def build(client_collected_bytes, summary, html_file): 390 """Generate an HTML file to visualize the given directory summary. 391 392 @param client_collected_bytes: The total size of results collected from 393 the DUT. The number can be larger than the total file size of the 394 given path, as files can be overwritten or removed. 395 @param summary: A ResultInfo instance containing the directory summary. 396 @param html_file: Path to save the html file to. 397 """ 398 size_original = summary.original_size 399 size_trimmed = summary.trimmed_size 400 size_summary_data = {SIZE_CLIENT_COLLECTED: 401 utils_lib.get_size_string(client_collected_bytes), 402 SIZE_ORIGINAL: 403 utils_lib.get_size_string(size_original), 404 SIZE_TRIMMED: 405 utils_lib.get_size_string(size_trimmed)} 406 size_trimmed_width = DEFAULT_SIZE_TRIMMED_WIDTH 407 if size_original == size_trimmed: 408 size_summary_data[SIZE_TRIMMED] = NOT_THROTTLED 409 size_trimmed_width = 0 410 411 size_summary = SIZE_SUMMARY_TEMPLATE % size_summary_data 412 413 indentation = INDENTATION 414 dirs_html = _get_dirs_html( 415 summary.files, '..', size_original, indentation + INDENTATION) 416 summary_tree = SUBDIRS_WRAPPER_TEMPLATE % {DIRS: dirs_html, 417 INDENTATION_KEY: indentation} 418 419 # job_dir is the path between Autotest `results` folder and the summary html 420 # file, e.g., 123-debug_user/host1. Assume it always contains 2 levels. 421 job_dir_sections = html_file.split(os.sep)[:-1] 422 try: 423 job_dir = '/'.join(job_dir_sections[ 424 (job_dir_sections.index('results')+1):]) 425 except ValueError: 426 # 'results' is not in the path, default to two levels up of the summary 427 # file. 428 job_dir = '/'.join(job_dir_sections[-2:]) 429 430 javascript = (JAVASCRIPT_TEMPLATE % 431 {GS_FILE_BASE_URL_KEY: GS_FILE_BASE_URL, 432 JOB_DIR: job_dir}) 433 css = CSS_TEMPLATE % {SIZE_TRIMMED_WIDTH: size_trimmed_width} 434 html = PAGE_TEMPLATE % {SIZE_SUMMARY: size_summary, 435 SUMMARY_TREE: summary_tree, 436 CSS: css, 437 JAVASCRIPT: javascript} 438 with open(html_file, 'w') as f: 439 f.write(html) 440