1 # pylint: disable-msg=C0111 2 3 import base64, os, tempfile, pickle, datetime, django.db 4 import os.path, getpass 5 from math import sqrt 6 7 # When you import matplotlib, it tries to write some temp files for better 8 # performance, and it does that to the directory in MPLCONFIGDIR, or, if that 9 # doesn't exist, the home directory. Problem is, the home directory is not 10 # writable when running under Apache, and matplotlib's not smart enough to 11 # handle that. It does appear smart enough to handle the files going 12 # away after they are written, though. 13 14 temp_dir = os.path.join(tempfile.gettempdir(), 15 '.matplotlib-%s' % getpass.getuser()) 16 if not os.path.exists(temp_dir): 17 os.mkdir(temp_dir) 18 os.environ['MPLCONFIGDIR'] = temp_dir 19 20 try: 21 import matplotlib 22 matplotlib.use('Agg') 23 import matplotlib.figure, matplotlib.backends.backend_agg 24 import StringIO, colorsys, PIL.Image, PIL.ImageChops 25 except ImportError: 26 # Do nothing, in case this is part of a unit test, so the unit test 27 # can proceed. 28 pass 29 30 from autotest_lib.frontend.afe import readonly_connection 31 from autotest_lib.frontend.afe.model_logic import ValidationError 32 from json import encoder 33 from autotest_lib.client.common_lib import global_config 34 from autotest_lib.frontend.tko import models, tko_rpc_utils 35 36 _FIGURE_DPI = 100 37 _FIGURE_WIDTH_IN = 10 38 _FIGURE_BOTTOM_PADDING_IN = 2 # for x-axis labels 39 40 _SINGLE_PLOT_HEIGHT = 6 41 _MULTIPLE_PLOT_HEIGHT_PER_PLOT = 4 42 43 _MULTIPLE_PLOT_MARKER_TYPE = 'o' 44 _MULTIPLE_PLOT_MARKER_SIZE = 4 45 _SINGLE_PLOT_STYLE = 'bs-' # blue squares with lines connecting 46 _SINGLE_PLOT_ERROR_BAR_COLOR = 'r' 47 48 _LEGEND_FONT_SIZE = 'xx-small' 49 _LEGEND_HANDLE_LENGTH = 0.03 50 _LEGEND_NUM_POINTS = 3 51 _LEGEND_MARKER_TYPE = 'o' 52 53 _LINE_XTICK_LABELS_SIZE = 'x-small' 54 _BAR_XTICK_LABELS_SIZE = 8 55 56 _json_encoder = encoder.JSONEncoder() 57 58 class NoDataError(Exception): 59 """\ 60 Exception to raise if the graphing query returned an empty resultset. 61 """ 62 63 64 def _colors(n): 65 """\ 66 Generator function for creating n colors. The return value is a tuple 67 representing the RGB of the color. 68 """ 69 for i in xrange(n): 70 yield colorsys.hsv_to_rgb(float(i) / n, 1.0, 1.0) 71 72 73 def _resort(kernel_labels, list_to_sort): 74 """\ 75 Resorts a list, using a list of kernel strings as the keys. Returns the 76 resorted list. 77 """ 78 79 labels = [tko_rpc_utils.KernelString(label) for label in kernel_labels] 80 resorted_pairs = sorted(zip(labels, list_to_sort)) 81 82 # We only want the resorted list; we are not interested in the kernel 83 # strings. 84 return [pair[1] for pair in resorted_pairs] 85 86 87 def _quote(string): 88 return "%s%s%s" % ("'", string.replace("'", r"\'"), "'") 89 90 91 _HTML_TEMPLATE = """\ 92 <html><head></head><body> 93 <img src="data:image/png;base64,%s" usemap="#%s" 94 border="0" alt="graph"> 95 <map name="%s">%s</map> 96 </body></html>""" 97 98 _AREA_TEMPLATE = """\ 99 <area shape="rect" coords="%i,%i,%i,%i" title="%s" 100 href="#" 101 onclick="%s(%s); return false;">""" 102 103 104 class MetricsPlot(object): 105 def __init__(self, query_dict, plot_type, inverted_series, normalize_to, 106 drilldown_callback): 107 """ 108 query_dict: dictionary containing the main query and the drilldown 109 queries. The main query returns a row for each x value. The first 110 column contains the x-axis label. Subsequent columns contain data 111 for each series, named by the column names. A column named 112 'errors-<x>' will be interpreted as errors for the series named <x>. 113 114 plot_type: 'Line' or 'Bar', depending on the plot type the user wants 115 116 inverted_series: list of series that should be plotted on an inverted 117 y-axis 118 119 normalize_to: 120 None - do not normalize 121 'first' - normalize against the first data point 122 'x__%s' - normalize against the x-axis value %s 123 'series__%s' - normalize against the series %s 124 125 drilldown_callback: name of drilldown callback method. 126 """ 127 self.query_dict = query_dict 128 if plot_type == 'Line': 129 self.is_line = True 130 elif plot_type == 'Bar': 131 self.is_line = False 132 else: 133 raise ValidationError({'plot' : 'Plot must be either Line or Bar'}) 134 self.plot_type = plot_type 135 self.inverted_series = inverted_series 136 self.normalize_to = normalize_to 137 if self.normalize_to is None: 138 self.normalize_to = '' 139 self.drilldown_callback = drilldown_callback 140 141 142 class QualificationHistogram(object): 143 def __init__(self, query, filter_string, interval, drilldown_callback): 144 """ 145 query: the main query to retrieve the pass rate information. The first 146 column contains the hostnames of all the machines that satisfied the 147 global filter. The second column (titled 'total') contains the total 148 number of tests that ran on that machine and satisfied the global 149 filter. The third column (titled 'good') contains the number of 150 those tests that passed on that machine. 151 152 filter_string: filter to apply to the common global filter to show the 153 Table View drilldown of a histogram bucket 154 155 interval: interval for each bucket. E.g., 10 means that buckets should 156 be 0-10%, 10%-20%, ... 157 158 """ 159 self.query = query 160 self.filter_string = filter_string 161 self.interval = interval 162 self.drilldown_callback = drilldown_callback 163 164 165 def _create_figure(height_inches): 166 """\ 167 Creates an instance of matplotlib.figure.Figure, given the height in inches. 168 Returns the figure and the height in pixels. 169 """ 170 171 fig = matplotlib.figure.Figure( 172 figsize=(_FIGURE_WIDTH_IN, height_inches + _FIGURE_BOTTOM_PADDING_IN), 173 dpi=_FIGURE_DPI, facecolor='white') 174 fig.subplots_adjust(bottom=float(_FIGURE_BOTTOM_PADDING_IN) / height_inches) 175 return (fig, fig.get_figheight() * _FIGURE_DPI) 176 177 178 def _create_line(plots, labels, plot_info): 179 """\ 180 Given all the data for the metrics, create a line plot. 181 182 plots: list of dicts containing the plot data. Each dict contains: 183 x: list of x-values for the plot 184 y: list of corresponding y-values 185 errors: errors for each data point, or None if no error information 186 available 187 label: plot title 188 labels: list of x-tick labels 189 plot_info: a MetricsPlot 190 """ 191 # when we're doing any kind of normalization, all series get put into a 192 # single plot 193 single = bool(plot_info.normalize_to) 194 195 area_data = [] 196 lines = [] 197 if single: 198 plot_height = _SINGLE_PLOT_HEIGHT 199 else: 200 plot_height = _MULTIPLE_PLOT_HEIGHT_PER_PLOT * len(plots) 201 figure, height = _create_figure(plot_height) 202 203 if single: 204 subplot = figure.add_subplot(1, 1, 1) 205 206 # Plot all the data 207 for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))): 208 needs_invert = (plot['label'] in plot_info.inverted_series) 209 210 # Add a new subplot, if user wants multiple subplots 211 # Also handle axis inversion for subplots here 212 if not single: 213 subplot = figure.add_subplot(len(plots), 1, plot_index + 1) 214 subplot.set_title(plot['label']) 215 if needs_invert: 216 # for separate plots, just invert the y-axis 217 subplot.set_ylim(1, 0) 218 elif needs_invert: 219 # for a shared plot (normalized data), need to invert the y values 220 # manually, since all plots share a y-axis 221 plot['y'] = [-y for y in plot['y']] 222 223 # Plot the series 224 subplot.set_xticks(range(0, len(labels))) 225 subplot.set_xlim(-1, len(labels)) 226 if single: 227 lines += subplot.plot(plot['x'], plot['y'], label=plot['label'], 228 marker=_MULTIPLE_PLOT_MARKER_TYPE, 229 markersize=_MULTIPLE_PLOT_MARKER_SIZE) 230 error_bar_color = lines[-1].get_color() 231 else: 232 lines += subplot.plot(plot['x'], plot['y'], _SINGLE_PLOT_STYLE, 233 label=plot['label']) 234 error_bar_color = _SINGLE_PLOT_ERROR_BAR_COLOR 235 if plot['errors']: 236 subplot.errorbar(plot['x'], plot['y'], linestyle='None', 237 yerr=plot['errors'], color=error_bar_color) 238 subplot.set_xticklabels([]) 239 240 # Construct the information for the drilldowns. 241 # We need to do this in a separate loop so that all the data is in 242 # matplotlib before we start calling transform(); otherwise, it will return 243 # incorrect data because it hasn't finished adjusting axis limits. 244 for line in lines: 245 246 # Get the pixel coordinates of each point on the figure 247 x = line.get_xdata() 248 y = line.get_ydata() 249 label = line.get_label() 250 icoords = line.get_transform().transform(zip(x,y)) 251 252 # Get the appropriate drilldown query 253 drill = plot_info.query_dict['__' + label + '__'] 254 255 # Set the title attributes (hover-over tool-tips) 256 x_labels = [labels[x_val] for x_val in x] 257 titles = ['%s - %s: %f' % (label, x_label, y_val) 258 for x_label, y_val in zip(x_labels, y)] 259 260 # Get the appropriate parameters for the drilldown query 261 params = [dict(query=drill, series=line.get_label(), param=x_label) 262 for x_label in x_labels] 263 264 area_data += [dict(left=ix - 5, top=height - iy - 5, 265 right=ix + 5, bottom=height - iy + 5, 266 title= title, 267 callback=plot_info.drilldown_callback, 268 callback_arguments=param_dict) 269 for (ix, iy), title, param_dict 270 in zip(icoords, titles, params)] 271 272 subplot.set_xticklabels(labels, rotation=90, size=_LINE_XTICK_LABELS_SIZE) 273 274 # Show the legend if there are not multiple subplots 275 if single: 276 font_properties = matplotlib.font_manager.FontProperties( 277 size=_LEGEND_FONT_SIZE) 278 legend = figure.legend(lines, [plot['label'] for plot in plots], 279 prop=font_properties, 280 handlelen=_LEGEND_HANDLE_LENGTH, 281 numpoints=_LEGEND_NUM_POINTS) 282 # Workaround for matplotlib not keeping all line markers in the legend - 283 # it seems if we don't do this, matplotlib won't keep all the line 284 # markers in the legend. 285 for line in legend.get_lines(): 286 line.set_marker(_LEGEND_MARKER_TYPE) 287 288 return (figure, area_data) 289 290 291 def _get_adjusted_bar(x, bar_width, series_index, num_plots): 292 """\ 293 Adjust the list 'x' to take the multiple series into account. Each series 294 should be shifted such that the middle series lies at the appropriate x-axis 295 tick with the other bars around it. For example, if we had four series 296 (i.e. four bars per x value), we want to shift the left edges of the bars as 297 such: 298 Bar 1: -2 * width 299 Bar 2: -width 300 Bar 3: none 301 Bar 4: width 302 """ 303 adjust = (-0.5 * num_plots - 1 + series_index) * bar_width 304 return [x_val + adjust for x_val in x] 305 306 307 # TODO(showard): merge much of this function with _create_line by extracting and 308 # parameterizing methods 309 def _create_bar(plots, labels, plot_info): 310 """\ 311 Given all the data for the metrics, create a line plot. 312 313 plots: list of dicts containing the plot data. 314 x: list of x-values for the plot 315 y: list of corresponding y-values 316 errors: errors for each data point, or None if no error information 317 available 318 label: plot title 319 labels: list of x-tick labels 320 plot_info: a MetricsPlot 321 """ 322 323 area_data = [] 324 bars = [] 325 figure, height = _create_figure(_SINGLE_PLOT_HEIGHT) 326 327 # Set up the plot 328 subplot = figure.add_subplot(1, 1, 1) 329 subplot.set_xticks(range(0, len(labels))) 330 subplot.set_xlim(-1, len(labels)) 331 subplot.set_xticklabels(labels, rotation=90, size=_BAR_XTICK_LABELS_SIZE) 332 # draw a bold line at y=0, making it easier to tell if bars are dipping 333 # below the axis or not. 334 subplot.axhline(linewidth=2, color='black') 335 336 # width here is the width for each bar in the plot. Matplotlib default is 337 # 0.8. 338 width = 0.8 / len(plots) 339 340 # Plot the data 341 for plot_index, (plot, color) in enumerate(zip(plots, _colors(len(plots)))): 342 # Invert the y-axis if needed 343 if plot['label'] in plot_info.inverted_series: 344 plot['y'] = [-y for y in plot['y']] 345 346 adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1, 347 len(plots)) 348 bar_data = subplot.bar(adjusted_x, plot['y'], 349 width=width, yerr=plot['errors'], 350 facecolor=color, 351 label=plot['label']) 352 bars.append(bar_data[0]) 353 354 # Construct the information for the drilldowns. 355 # See comment in _create_line for why we need a separate loop to do this. 356 for plot_index, plot in enumerate(plots): 357 adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index + 1, 358 len(plots)) 359 360 # Let matplotlib plot the data, so that we can get the data-to-image 361 # coordinate transforms 362 line = subplot.plot(adjusted_x, plot['y'], linestyle='None')[0] 363 label = plot['label'] 364 upper_left_coords = line.get_transform().transform(zip(adjusted_x, 365 plot['y'])) 366 bottom_right_coords = line.get_transform().transform( 367 [(x + width, 0) for x in adjusted_x]) 368 369 # Get the drilldown query 370 drill = plot_info.query_dict['__' + label + '__'] 371 372 # Set the title attributes 373 x_labels = [labels[x] for x in plot['x']] 374 titles = ['%s - %s: %f' % (plot['label'], label, y) 375 for label, y in zip(x_labels, plot['y'])] 376 params = [dict(query=drill, series=plot['label'], param=x_label) 377 for x_label in x_labels] 378 area_data += [dict(left=ulx, top=height - uly, 379 right=brx, bottom=height - bry, 380 title=title, 381 callback=plot_info.drilldown_callback, 382 callback_arguments=param_dict) 383 for (ulx, uly), (brx, bry), title, param_dict 384 in zip(upper_left_coords, bottom_right_coords, titles, 385 params)] 386 387 figure.legend(bars, [plot['label'] for plot in plots]) 388 return (figure, area_data) 389 390 391 def _normalize(data_values, data_errors, base_values, base_errors): 392 """\ 393 Normalize the data against a baseline. 394 395 data_values: y-values for the to-be-normalized data 396 data_errors: standard deviations for the to-be-normalized data 397 base_values: list of values normalize against 398 base_errors: list of standard deviations for those base values 399 """ 400 values = [] 401 for value, base in zip(data_values, base_values): 402 try: 403 values.append(100 * (value - base) / base) 404 except ZeroDivisionError: 405 # Base is 0.0 so just simplify: 406 # If value < base: append -100.0; 407 # If value == base: append 0.0 (obvious); and 408 # If value > base: append 100.0. 409 values.append(100 * float(cmp(value, base))) 410 411 # Based on error for f(x,y) = 100 * (x - y) / y 412 if data_errors: 413 if not base_errors: 414 base_errors = [0] * len(data_errors) 415 errors = [] 416 for data, error, base_value, base_error in zip( 417 data_values, data_errors, base_values, base_errors): 418 try: 419 errors.append(sqrt(error**2 * (100 / base_value)**2 420 + base_error**2 * (100 * data / base_value**2)**2 421 + error * base_error * (100 / base_value**2)**2)) 422 except ZeroDivisionError: 423 # Again, base is 0.0 so do the simple thing. 424 errors.append(100 * abs(error)) 425 else: 426 errors = None 427 428 return (values, errors) 429 430 431 def _create_png(figure): 432 """\ 433 Given the matplotlib figure, generate the PNG data for it. 434 """ 435 436 # Draw the image 437 canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(figure) 438 canvas.draw() 439 size = canvas.get_renderer().get_canvas_width_height() 440 image_as_string = canvas.tostring_rgb() 441 image = PIL.Image.fromstring('RGB', size, image_as_string, 'raw', 'RGB', 0, 442 1) 443 image_background = PIL.Image.new(image.mode, image.size, 444 figure.get_facecolor()) 445 446 # Crop the image to remove surrounding whitespace 447 non_whitespace = PIL.ImageChops.difference(image, image_background) 448 bounding_box = non_whitespace.getbbox() 449 image = image.crop(bounding_box) 450 451 image_data = StringIO.StringIO() 452 image.save(image_data, format='PNG') 453 454 return image_data.getvalue(), bounding_box 455 456 457 def _create_image_html(figure, area_data, plot_info): 458 """\ 459 Given the figure and drilldown data, construct the HTML that will render the 460 graph as a PNG image, and attach the image map to that image. 461 462 figure: figure containing the drawn plot(s) 463 area_data: list of parameters for each area of the image map. See the 464 definition of the template string '_AREA_TEMPLATE' 465 plot_info: a MetricsPlot or QualHistogram 466 """ 467 468 png, bbox = _create_png(figure) 469 470 # Construct the list of image map areas 471 areas = [_AREA_TEMPLATE % 472 (data['left'] - bbox[0], data['top'] - bbox[1], 473 data['right'] - bbox[0], data['bottom'] - bbox[1], 474 data['title'], data['callback'], 475 _json_encoder.encode(data['callback_arguments']) 476 .replace('"', '"')) 477 for data in area_data] 478 479 map_name = plot_info.drilldown_callback + '_map' 480 return _HTML_TEMPLATE % (base64.b64encode(png), map_name, map_name, 481 '\n'.join(areas)) 482 483 484 def _find_plot_by_label(plots, label): 485 for index, plot in enumerate(plots): 486 if plot['label'] == label: 487 return index 488 raise ValueError('no plot labeled "%s" found' % label) 489 490 491 def _normalize_to_series(plots, base_series): 492 base_series_index = _find_plot_by_label(plots, base_series) 493 base_plot = plots[base_series_index] 494 base_xs = base_plot['x'] 495 base_values = base_plot['y'] 496 base_errors = base_plot['errors'] 497 del plots[base_series_index] 498 499 for plot in plots: 500 old_xs, old_values, old_errors = plot['x'], plot['y'], plot['errors'] 501 new_xs, new_values, new_errors = [], [], [] 502 new_base_values, new_base_errors = [], [] 503 # Select only points in the to-be-normalized data that have a 504 # corresponding baseline value 505 for index, x_value in enumerate(old_xs): 506 try: 507 base_index = base_xs.index(x_value) 508 except ValueError: 509 continue 510 511 new_xs.append(x_value) 512 new_values.append(old_values[index]) 513 new_base_values.append(base_values[base_index]) 514 if old_errors: 515 new_errors.append(old_errors[index]) 516 new_base_errors.append(base_errors[base_index]) 517 518 if not new_xs: 519 raise NoDataError('No normalizable data for series ' + 520 plot['label']) 521 plot['x'] = new_xs 522 plot['y'] = new_values 523 if old_errors: 524 plot['errors'] = new_errors 525 526 plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'], 527 new_base_values, 528 new_base_errors) 529 530 531 def _create_metrics_plot_helper(plot_info, extra_text=None): 532 """ 533 Create a metrics plot of the given plot data. 534 plot_info: a MetricsPlot object. 535 extra_text: text to show at the uppper-left of the graph 536 537 TODO(showard): move some/all of this logic into methods on MetricsPlot 538 """ 539 query = plot_info.query_dict['__main__'] 540 cursor = readonly_connection.cursor() 541 cursor.execute(query) 542 543 if not cursor.rowcount: 544 raise NoDataError('query did not return any data') 545 rows = cursor.fetchall() 546 # "transpose" rows, so columns[0] is all the values from the first column, 547 # etc. 548 columns = zip(*rows) 549 550 plots = [] 551 labels = [str(label) for label in columns[0]] 552 needs_resort = (cursor.description[0][0] == 'kernel') 553 554 # Collect all the data for the plot 555 col = 1 556 while col < len(cursor.description): 557 y = columns[col] 558 label = cursor.description[col][0] 559 col += 1 560 if (col < len(cursor.description) and 561 'errors-' + label == cursor.description[col][0]): 562 errors = columns[col] 563 col += 1 564 else: 565 errors = None 566 if needs_resort: 567 y = _resort(labels, y) 568 if errors: 569 errors = _resort(labels, errors) 570 571 x = [index for index, value in enumerate(y) if value is not None] 572 if not x: 573 raise NoDataError('No data for series ' + label) 574 y = [y[i] for i in x] 575 if errors: 576 errors = [errors[i] for i in x] 577 plots.append({ 578 'label': label, 579 'x': x, 580 'y': y, 581 'errors': errors 582 }) 583 584 if needs_resort: 585 labels = _resort(labels, labels) 586 587 # Normalize the data if necessary 588 normalize_to = plot_info.normalize_to 589 if normalize_to == 'first' or normalize_to.startswith('x__'): 590 if normalize_to != 'first': 591 baseline = normalize_to[3:] 592 try: 593 baseline_index = labels.index(baseline) 594 except ValueError: 595 raise ValidationError({ 596 'Normalize' : 'Invalid baseline %s' % baseline 597 }) 598 for plot in plots: 599 if normalize_to == 'first': 600 plot_index = 0 601 else: 602 try: 603 plot_index = plot['x'].index(baseline_index) 604 # if the value is not found, then we cannot normalize 605 except ValueError: 606 raise ValidationError({ 607 'Normalize' : ('%s does not have a value for %s' 608 % (plot['label'], normalize_to[3:])) 609 }) 610 base_values = [plot['y'][plot_index]] * len(plot['y']) 611 if plot['errors']: 612 base_errors = [plot['errors'][plot_index]] * len(plot['errors']) 613 plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'], 614 base_values, 615 None or base_errors) 616 617 elif normalize_to.startswith('series__'): 618 base_series = normalize_to[8:] 619 _normalize_to_series(plots, base_series) 620 621 # Call the appropriate function to draw the line or bar plot 622 if plot_info.is_line: 623 figure, area_data = _create_line(plots, labels, plot_info) 624 else: 625 figure, area_data = _create_bar(plots, labels, plot_info) 626 627 # TODO(showard): extract these magic numbers to named constants 628 if extra_text: 629 text_y = .95 - .0075 * len(plots) 630 figure.text(.1, text_y, extra_text, size='xx-small') 631 632 return (figure, area_data) 633 634 635 def create_metrics_plot(query_dict, plot_type, inverted_series, normalize_to, 636 drilldown_callback, extra_text=None): 637 plot_info = MetricsPlot(query_dict, plot_type, inverted_series, 638 normalize_to, drilldown_callback) 639 figure, area_data = _create_metrics_plot_helper(plot_info, extra_text) 640 return _create_image_html(figure, area_data, plot_info) 641 642 643 def _get_hostnames_in_bucket(hist_data, bucket): 644 """\ 645 Get all the hostnames that constitute a particular bucket in the histogram. 646 647 hist_data: list containing tuples of (hostname, pass_rate) 648 bucket: tuple containing the (low, high) values of the target bucket 649 """ 650 651 return [hostname for hostname, pass_rate in hist_data 652 if bucket[0] <= pass_rate < bucket[1]] 653 654 655 def _create_qual_histogram_helper(plot_info, extra_text=None): 656 """\ 657 Create a machine qualification histogram of the given data. 658 659 plot_info: a QualificationHistogram 660 extra_text: text to show at the upper-left of the graph 661 662 TODO(showard): move much or all of this into methods on 663 QualificationHistogram 664 """ 665 cursor = readonly_connection.cursor() 666 cursor.execute(plot_info.query) 667 668 if not cursor.rowcount: 669 raise NoDataError('query did not return any data') 670 671 # Lists to store the plot data. 672 # hist_data store tuples of (hostname, pass_rate) for machines that have 673 # pass rates between 0 and 100%, exclusive. 674 # no_tests is a list of machines that have run none of the selected tests 675 # no_pass is a list of machines with 0% pass rate 676 # perfect is a list of machines with a 100% pass rate 677 hist_data = [] 678 no_tests = [] 679 no_pass = [] 680 perfect = [] 681 682 # Construct the lists of data to plot 683 for hostname, total, good in cursor.fetchall(): 684 if total == 0: 685 no_tests.append(hostname) 686 continue 687 688 if good == 0: 689 no_pass.append(hostname) 690 elif good == total: 691 perfect.append(hostname) 692 else: 693 percentage = 100.0 * good / total 694 hist_data.append((hostname, percentage)) 695 696 interval = plot_info.interval 697 bins = range(0, 100, interval) 698 if bins[-1] != 100: 699 bins.append(bins[-1] + interval) 700 701 figure, height = _create_figure(_SINGLE_PLOT_HEIGHT) 702 subplot = figure.add_subplot(1, 1, 1) 703 704 # Plot the data and get all the bars plotted 705 _,_, bars = subplot.hist([data[1] for data in hist_data], 706 bins=bins, align='left') 707 bars += subplot.bar([-interval], len(no_pass), 708 width=interval, align='center') 709 bars += subplot.bar([bins[-1]], len(perfect), 710 width=interval, align='center') 711 bars += subplot.bar([-3 * interval], len(no_tests), 712 width=interval, align='center') 713 714 buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]] 715 # set the x-axis range to cover all the normal bins plus the three "special" 716 # ones - N/A (3 intervals left), 0% (1 interval left) ,and 100% (far right) 717 subplot.set_xlim(-4 * interval, bins[-1] + interval) 718 subplot.set_xticks([-3 * interval, -interval] + bins + [100 + interval]) 719 subplot.set_xticklabels(['N/A', '0%'] + 720 ['%d%% - <%d%%' % bucket for bucket in buckets] + 721 ['100%'], rotation=90, size='small') 722 723 # Find the coordinates on the image for each bar 724 x = [] 725 y = [] 726 for bar in bars: 727 x.append(bar.get_x()) 728 y.append(bar.get_height()) 729 f = subplot.plot(x, y, linestyle='None')[0] 730 upper_left_coords = f.get_transform().transform(zip(x, y)) 731 bottom_right_coords = f.get_transform().transform( 732 [(x_val + interval, 0) for x_val in x]) 733 734 # Set the title attributes 735 titles = ['%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val) 736 for bucket, y_val in zip(buckets, y)] 737 titles.append('0%%: %d machines' % len(no_pass)) 738 titles.append('100%%: %d machines' % len(perfect)) 739 titles.append('N/A: %d machines' % len(no_tests)) 740 741 # Get the hostnames for each bucket in the histogram 742 names_list = [_get_hostnames_in_bucket(hist_data, bucket) 743 for bucket in buckets] 744 names_list += [no_pass, perfect] 745 746 if plot_info.filter_string: 747 plot_info.filter_string += ' AND ' 748 749 # Construct the list of drilldown parameters to be passed when the user 750 # clicks on the bar. 751 params = [] 752 for names in names_list: 753 if names: 754 hostnames = ','.join(_quote(hostname) for hostname in names) 755 hostname_filter = 'hostname IN (%s)' % hostnames 756 full_filter = plot_info.filter_string + hostname_filter 757 params.append({'type': 'normal', 758 'filterString': full_filter}) 759 else: 760 params.append({'type': 'empty'}) 761 762 params.append({'type': 'not_applicable', 763 'hosts': '<br />'.join(no_tests)}) 764 765 area_data = [dict(left=ulx, top=height - uly, 766 right=brx, bottom=height - bry, 767 title=title, callback=plot_info.drilldown_callback, 768 callback_arguments=param_dict) 769 for (ulx, uly), (brx, bry), title, param_dict 770 in zip(upper_left_coords, bottom_right_coords, titles, params)] 771 772 # TODO(showard): extract these magic numbers to named constants 773 if extra_text: 774 figure.text(.1, .95, extra_text, size='xx-small') 775 776 return (figure, area_data) 777 778 779 def create_qual_histogram(query, filter_string, interval, drilldown_callback, 780 extra_text=None): 781 plot_info = QualificationHistogram(query, filter_string, interval, 782 drilldown_callback) 783 figure, area_data = _create_qual_histogram_helper(plot_info, extra_text) 784 return _create_image_html(figure, area_data, plot_info) 785 786 787 def create_embedded_plot(model, update_time): 788 """\ 789 Given an EmbeddedGraphingQuery object, generate the PNG image for it. 790 791 model: EmbeddedGraphingQuery object 792 update_time: 'Last updated' time 793 """ 794 795 params = pickle.loads(model.params) 796 extra_text = 'Last updated: %s' % update_time 797 798 if model.graph_type == 'metrics': 799 plot_info = MetricsPlot(query_dict=params['queries'], 800 plot_type=params['plot'], 801 inverted_series=params['invert'], 802 normalize_to=None, 803 drilldown_callback='') 804 figure, areas_unused = _create_metrics_plot_helper(plot_info, 805 extra_text) 806 elif model.graph_type == 'qual': 807 plot_info = QualificationHistogram( 808 query=params['query'], filter_string=params['filter_string'], 809 interval=params['interval'], drilldown_callback='') 810 figure, areas_unused = _create_qual_histogram_helper(plot_info, 811 extra_text) 812 else: 813 raise ValueError('Invalid graph_type %s' % model.graph_type) 814 815 image, bounding_box_unused = _create_png(figure) 816 return image 817 818 819 _cache_timeout = global_config.global_config.get_config_value( 820 'AUTOTEST_WEB', 'graph_cache_creation_timeout_minutes') 821 822 823 def handle_plot_request(id, max_age): 824 """\ 825 Given the embedding id of a graph, generate a PNG of the embedded graph 826 associated with that id. 827 828 id: id of the embedded graph 829 max_age: maximum age, in minutes, that a cached version should be held 830 """ 831 model = models.EmbeddedGraphingQuery.objects.get(id=id) 832 833 # Check if the cached image needs to be updated 834 now = datetime.datetime.now() 835 update_time = model.last_updated + datetime.timedelta(minutes=int(max_age)) 836 if now > update_time: 837 cursor = django.db.connection.cursor() 838 839 # We want this query to update the refresh_time only once, even if 840 # multiple threads are running it at the same time. That is, only the 841 # first thread will win the race, and it will be the one to update the 842 # cached image; all other threads will show that they updated 0 rows 843 query = """ 844 UPDATE embedded_graphing_queries 845 SET refresh_time = NOW() 846 WHERE id = %s AND ( 847 refresh_time IS NULL OR 848 refresh_time + INTERVAL %s MINUTE < NOW() 849 ) 850 """ 851 cursor.execute(query, (id, _cache_timeout)) 852 853 # Only refresh the cached image if we were successful in updating the 854 # refresh time 855 if cursor.rowcount: 856 model.cached_png = create_embedded_plot(model, now.ctime()) 857 model.last_updated = now 858 model.refresh_time = None 859 model.save() 860 861 return model.cached_png 862
