1 /******************************************************************************* 2 * Copyright (c) 2000, 2009 IBM Corporation and others. 3 * All rights reserved. This program and the accompanying materials 4 * are made available under the terms of the Eclipse Public License v1.0 5 * which accompanies this distribution, and is available at 6 * http://www.eclipse.org/legal/epl-v10.html 7 * 8 * Contributors: 9 * IBM Corporation - initial API and implementation 10 *******************************************************************************/ 11 package org.eclipse.test.internal.performance.results.model; 12 13 import java.util.ArrayList; 14 import java.util.List; 15 import java.util.Vector; 16 17 import org.eclipse.core.runtime.IAdaptable; 18 import org.eclipse.core.runtime.preferences.IEclipsePreferences; 19 import org.eclipse.core.runtime.preferences.InstanceScope; 20 import org.eclipse.jface.resource.ImageDescriptor; 21 import org.eclipse.swt.graphics.Image; 22 import org.eclipse.test.internal.performance.eval.StatisticsUtil; 23 import org.eclipse.test.internal.performance.results.db.AbstractResults; 24 import org.eclipse.test.internal.performance.results.db.BuildResults; 25 import org.eclipse.test.internal.performance.results.db.ConfigResults; 26 import org.eclipse.test.internal.performance.results.db.DB_Results; 27 import org.eclipse.test.internal.performance.results.utils.IPerformancesConstants; 28 import org.eclipse.test.internal.performance.results.utils.Util; 29 import org.eclipse.ui.ISharedImages; 30 import org.eclipse.ui.PlatformUI; 31 import org.eclipse.ui.model.IWorkbenchAdapter; 32 import org.eclipse.ui.views.properties.ComboBoxPropertyDescriptor; 33 import org.eclipse.ui.views.properties.IPropertyDescriptor; 34 import org.eclipse.ui.views.properties.IPropertySource; 35 import org.eclipse.ui.views.properties.PropertyDescriptor; 36 import org.eclipse.ui.views.properties.TextPropertyDescriptor; 37 import org.osgi.service.prefs.BackingStoreException; 38 39 /** 40 * An Organization Element 41 */ 42 public abstract class ResultsElement implements IAdaptable, IPropertySource, IWorkbenchAdapter, Comparable { 43 44 // Image descriptors 45 private static final ISharedImages WORKBENCH_SHARED_IMAGES = PlatformUI.getWorkbench().getSharedImages(); 46 public static final Image ERROR_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_ERROR_TSK); 47 public static final ImageDescriptor ERROR_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_ERROR_TSK); 48 public static final Image WARN_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_WARN_TSK); 49 public static final ImageDescriptor WARN_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_WARN_TSK); 50 public static final Image INFO_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_OBJS_INFO_TSK); 51 public static final ImageDescriptor INFO_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJS_INFO_TSK); 52 public static final Image HELP_IMAGE = WORKBENCH_SHARED_IMAGES.getImage(ISharedImages.IMG_LCL_LINKTO_HELP); 53 public static final ImageDescriptor HELP_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_LCL_LINKTO_HELP); 54 public static final ImageDescriptor FOLDER_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_OBJ_FOLDER); 55 public static final ImageDescriptor CONNECT_IMAGE_DESCRIPTOR = WORKBENCH_SHARED_IMAGES.getImageDescriptor(ISharedImages.IMG_ELCL_SYNCED); 56 57 // Model 58 ResultsElement parent; 59 AbstractResults results; 60 ResultsElement[] children; 61 String name; 62 int status = -1; 63 64 // Stats 65 double[] statistics; 66 67 // Status constants 68 // state 69 static final int UNKNOWN = 0x01; 70 static final int UNREAD = 0x02; 71 static final int READ = 0x04; 72 static final int MISSING = 0x08; 73 public static final int STATE_MASK = 0x0F; 74 // info 75 static final int SMALL_VALUE = 0x0010; 76 static final int STUDENT_TTEST = 0x0020; 77 public static final int INFO_MASK = 0x0030; 78 // warning 79 static final int NO_BASELINE = 0x0040; 80 static final int SINGLE_RUN = 0x0080; 81 static final int BIG_ERROR = 0x0100; 82 static final int NOT_STABLE = 0x0200; 83 static final int NOT_RELIABLE = 0x0400; 84 public static final int WARNING_MASK = 0x0FC0; 85 // error 86 static final int BIG_DELTA = 0x1000; 87 public static final int ERROR_MASK = 0xF000; 88 89 // Property descriptors 90 static final String P_ID_STATUS_INFO = "ResultsElement.status_info"; //$NON-NLS-1$ 91 static final String P_ID_STATUS_WARNING = "ResultsElement.status_warning"; //$NON-NLS-1$ 92 static final String P_ID_STATUS_ERROR = "ResultsElement.status_error"; //$NON-NLS-1$ 93 static final String P_ID_STATUS_COMMENT = "ResultsElement.status_comment"; //$NON-NLS-1$ 94 95 static final String P_STR_STATUS_INFO = " info"; //$NON-NLS-1$ 96 static final String P_STR_STATUS_WARNING = "warning"; //$NON-NLS-1$ 97 static final String P_STR_STATUS_ERROR = "error"; //$NON-NLS-1$ 98 static final String P_STR_STATUS_COMMENT = "comment"; //$NON-NLS-1$ 99 static final String[] NO_VALUES = new String[0]; 100 101 private static Vector DESCRIPTORS; 102 static final TextPropertyDescriptor COMMENT_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_COMMENT, P_STR_STATUS_COMMENT); 103 static final TextPropertyDescriptor ERROR_DESCRIPTOR = new TextPropertyDescriptor(P_ID_STATUS_ERROR, P_STR_STATUS_ERROR); 104 static Vector initDescriptors(int status) { 105 DESCRIPTORS = new Vector(); 106 // Status category 107 DESCRIPTORS.add(getInfosDescriptor(status)); 108 DESCRIPTORS.add(getWarningsDescriptor(status)); 109 DESCRIPTORS.add(ERROR_DESCRIPTOR); 110 ERROR_DESCRIPTOR.setCategory("Status"); 111 // Survey category 112 DESCRIPTORS.add(COMMENT_DESCRIPTOR); 113 COMMENT_DESCRIPTOR.setCategory("Survey"); 114 return DESCRIPTORS; 115 } 116 static Vector getDescriptors() { 117 return DESCRIPTORS; 118 } 119 static ComboBoxPropertyDescriptor getInfosDescriptor(int status) { 120 List list = new ArrayList(); 121 if ((status & SMALL_VALUE) != 0) { 122 list.add("Some builds have tests with small values"); 123 } 124 if ((status & STUDENT_TTEST) != 0) { 125 list.add("Some builds have student-t test error over the threshold"); 126 } 127 String[] infos = new String[list.size()]; 128 if (list.size() > 0) { 129 list.toArray(infos); 130 } 131 ComboBoxPropertyDescriptor infoDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_INFO, P_STR_STATUS_INFO, infos); 132 infoDescriptor.setCategory("Status"); 133 return infoDescriptor; 134 } 135 static PropertyDescriptor getWarningsDescriptor(int status) { 136 List list = new ArrayList(); 137 if ((status & BIG_ERROR) != 0) { 138 list.add("Some builds have tests with error over 3%"); 139 } 140 if ((status & NOT_RELIABLE) != 0) { 141 list.add("Some builds have no reliable tests"); 142 } 143 if ((status & NOT_STABLE) != 0) { 144 list.add("Some builds have no stable tests"); 145 } 146 if ((status & NO_BASELINE) != 0) { 147 list.add("Some builds have no baseline to compare with"); 148 } 149 if ((status & SINGLE_RUN) != 0) { 150 list.add("Some builds have single run tests"); 151 } 152 String[] warnings = new String[list.size()]; 153 if (list.size() > 0) { 154 list.toArray(warnings); 155 } 156 ComboBoxPropertyDescriptor warningDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_WARNING, P_STR_STATUS_WARNING, warnings); 157 warningDescriptor.setCategory("Status"); 158 return warningDescriptor; 159 } 160 161 ResultsElement() { 162 } 163 164 ResultsElement(AbstractResults results, ResultsElement parent) { 165 this.parent = parent; 166 this.results = results; 167 } 168 169 ResultsElement(String name, ResultsElement parent) { 170 this.parent = parent; 171 this.name = name; 172 } 173 174 public int compareTo(Object o) { 175 if (this.results == null) { 176 if (o instanceof ResultsElement && this.name != null) { 177 ResultsElement element = (ResultsElement) o; 178 return this.name.compareTo(element.getName()); 179 } 180 return -1; 181 } 182 if (o instanceof ResultsElement) { 183 return this.results.compareTo(((ResultsElement)o).results); 184 } 185 return -1; 186 } 187 188 abstract ResultsElement createChild(AbstractResults testResults); 189 190 /* (non-Javadoc) 191 * Method declared on IAdaptable 192 */ 193 public Object getAdapter(Class adapter) { 194 if (adapter == IPropertySource.class) { 195 return this; 196 } 197 if (adapter == IWorkbenchAdapter.class) { 198 return this; 199 } 200 return null; 201 } 202 203 /** 204 * Iterate the element children. 205 */ 206 public ResultsElement[] getChildren() { 207 if (this.results == null) { 208 return new ResultsElement[0]; 209 } 210 if (this.children == null) { 211 initChildren(); 212 } 213 return this.children; 214 } 215 216 /* (non-Javadoc) 217 * Method declared on IWorkbenchAdapter 218 */ 219 public Object[] getChildren(Object o) { 220 if (this.results == null) { 221 return new Object[0]; 222 } 223 if (this.children == null) { 224 initChildren(); 225 } 226 return this.children; 227 } 228 229 /* (non-Javadoc) 230 * Method declared on IPropertySource 231 */ 232 public Object getEditableValue() { 233 return this; 234 } 235 236 final String getId() { 237 return getId(new StringBuffer()).toString(); 238 } 239 240 private StringBuffer getId(StringBuffer buffer) { 241 if (this.parent != null) { 242 return this.parent.getId(buffer).append('/').append(getName()); 243 } 244 return buffer.append(DB_Results.getDbName()); 245 } 246 247 /* (non-Javadoc) 248 * Method declared on IWorkbenchAdapter 249 */ 250 public ImageDescriptor getImageDescriptor(Object object) { 251 if (object instanceof ResultsElement) { 252 ResultsElement resultsElement = (ResultsElement) object; 253 // DEBUG 254 // if (resultsElement.getName().equals("I20090806-0100")) { 255 // if (resultsElement.results != null) { 256 // String toString = resultsElement.results.getParent().toString(); 257 // String toString = resultsElement.results.toString(); 258 // if (toString.indexOf("testStoreExists")>0 && toString.indexOf("eplnx2")>0) { 259 // System.out.println("stop"); 260 // } 261 // } 262 // } 263 int elementStatus = resultsElement.getStatus(); 264 if (elementStatus == MISSING) { 265 return HELP_IMAGE_DESCRIPTOR; 266 } 267 if ((elementStatus & ResultsElement.ERROR_MASK) != 0) { 268 return ERROR_IMAGE_DESCRIPTOR; 269 } 270 if ((elementStatus & ResultsElement.WARNING_MASK) != 0) { 271 return WARN_IMAGE_DESCRIPTOR; 272 } 273 if ((elementStatus & ResultsElement.INFO_MASK) != 0) { 274 return INFO_IMAGE_DESCRIPTOR; 275 } 276 } 277 return null; 278 } 279 280 /* (non-Javadoc) 281 * Method declared on IWorkbenchAdapter 282 */ 283 public String getLabel(Object o) { 284 return getName(); 285 } 286 287 /** 288 * Returns the name 289 */ 290 public String getName() { 291 if (this.name == null && this.results != null) { 292 this.name = this.results.getName(); 293 } 294 return this.name; 295 } 296 297 /** 298 * Returns the parent 299 */ 300 public Object getParent(Object o) { 301 return this.parent; 302 } 303 304 /* (non-Javadoc) 305 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyDescriptors() 306 */ 307 public IPropertyDescriptor[] getPropertyDescriptors() { 308 Vector descriptors = getDescriptors(); 309 if (descriptors == null) { 310 descriptors = initDescriptors(getStatus()); 311 } 312 int size = descriptors.size(); 313 IPropertyDescriptor[] descriptorsArray = new IPropertyDescriptor[size]; 314 descriptorsArray[0] = getInfosDescriptor(getStatus()); 315 descriptorsArray[1] = getWarningsDescriptor(getStatus()); 316 for (int i=2; i<size; i++) { 317 descriptorsArray[i] = (IPropertyDescriptor) descriptors.get(i); 318 } 319 return descriptorsArray; 320 } 321 322 /* (non-Javadoc) 323 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyValue(java.lang.Object) 324 */ 325 public Object getPropertyValue(Object propKey) { 326 if (propKey.equals(P_ID_STATUS_INFO)) { 327 if ((getStatus() & INFO_MASK) != 0) { 328 return new Integer(0); 329 } 330 } 331 if (propKey.equals(P_ID_STATUS_WARNING)) { 332 if ((getStatus() & WARNING_MASK) != 0) { 333 return new Integer(0); 334 } 335 } 336 if (propKey.equals(P_ID_STATUS_ERROR)) { 337 if ((getStatus() & BIG_DELTA) != 0) { 338 return "Some builds have tests with regression"; 339 } 340 } 341 if (propKey.equals(P_ID_STATUS_COMMENT)) { 342 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID); 343 return preferences.get(getId(), ""); 344 } 345 return null; 346 } 347 348 public ResultsElement getResultsElement(String resultName) { 349 int length = getChildren(null).length; 350 for (int i=0; i<length; i++) { 351 ResultsElement searchedResults = this.children[i]; 352 if (searchedResults.getName().equals(resultName)) { 353 return searchedResults; 354 } 355 } 356 return null; 357 } 358 359 /** 360 * Return the status of the element. 361 * 362 * The status is a bit mask pattern where digits are 363 * allowed as follow: 364 * <ul> 365 * <li>0-3: bits for state showing whether the element is 366 * <ul> 367 * <li>{@link #UNKNOWN} : not connected to a db</li> 368 * <li>{@link #UNREAD} : is not valid (e.g. NaN results)</li> 369 * <li>{@link #MISSING} : no results (e.g. the perf machine crashed and didn't store any results)</li> 370 * <li>{@link #READ} : has valid results</li> 371 * </ul> 372 * </li> 373 * <li>4-5: bits for information. Current possible information are 374 * <ul> 375 * <li>{@link #SMALL_VALUE} : build results or delta with baseline value is under 100ms</li> 376 * <li>{@link #STUDENT_TTEST} : the Student T-test is over the threshold (old yellow color for test results)</li> 377 * </ul> 378 * </li> 379 * <li>6-11: bits for warnings. Current possible warnings are 380 * <ul> 381 * <li>{@link #NO_BASELINE} : no baseline for the current build</li> 382 * <li>{@link #SINGLE_RUN} : the test has only one run (i.e. no error could be computed), hence its reliability cannot be evaluated</li> 383 * <li>{@link #BIG_ERROR} : the test result is over the 3% threshold</li> 384 * <li>{@link #NOT_STABLE} : the test history shows a deviation between 10% and 20% (may mean that this test is not so reliable)</li> 385 * <li>{@link #NOT_RELIABLE} : the test history shows a deviation over 20% (surely means that this test is too erratic to be reliable)</li> 386 * </ul> 387 * </li> 388 * <li>12-15: bits for errors. Current possible errors are 389 * <ul> 390 * <li>{@link #BIG_DELTA} : the delta for the test is over the 10% threshold</li> 391 * </ul> 392 * </li> 393 * </ul> 394 * 395 * Note that these explanation applied to {@link BuildResultsElement}, and {@link DimResultsElement}. 396 * For {@link ComponentResultsElement}, and {@link ScenarioResultsElement}, it's the merge of all the children status 397 * and means "Some tests have..." instead of "The test has...". For {@link ConfigResultsElement}, it means the status 398 * of the most recent build compared to its most recent baseline. 399 * 400 * @return An int with each bit set when the corresponding symptom applies. 401 */ 402 public final int getStatus() { 403 if (this.status < 0) { 404 initStatus(); 405 } 406 return this.status; 407 } 408 409 /** 410 * Return the statistics of the build along its history. 411 * 412 * @return An array of double built as follows: 413 * <ul> 414 * <li>0: numbers of values</li> 415 * <li>1: mean of values</li> 416 * <li>2: standard deviation of these values</li> 417 * <li>3: coefficient of variation of these values</li> 418 * </ul> 419 */ 420 double[] getStatistics() { 421 return this.statistics; 422 } 423 424 /** 425 * Returns whether the element (or one in its hierarchy) has an error. 426 * 427 * @return <code> true</code> if the element or one in its hierarchy has an error, 428 * <code> false</code> otherwise 429 */ 430 public final boolean hasError() { 431 return (getStatus() & ERROR_MASK) != 0; 432 } 433 434 void initChildren() { 435 AbstractResults[] resultsChildren = this.results.getChildren(); 436 int length = resultsChildren.length; 437 this.children = new ResultsElement[length]; 438 int count = 0; 439 for (int i=0; i<length; i++) { 440 ResultsElement childElement = createChild(resultsChildren[i]); 441 if (childElement != null) { 442 this.children[count++] = childElement; 443 } 444 } 445 if (count < length) { 446 System.arraycopy(this.children, 0, this.children = new ResultsElement[count], 0, count); 447 } 448 } 449 void initStatus() { 450 this.status = READ; 451 if (this.results != null) { 452 if (this.children == null) initChildren(); 453 int length = this.children.length; 454 for (int i=0; i<length; i++) { 455 this.status |= this.children[i].getStatus(); 456 } 457 } 458 } 459 460 int initStatus(BuildResults buildResults) { 461 this.status = READ; 462 463 // Get values 464 double buildValue = buildResults.getValue(); 465 ConfigResults configResults = (ConfigResults) buildResults.getParent(); 466 BuildResults baselineResults = configResults.getBaselineBuildResults(buildResults.getName()); 467 double baselineValue = baselineResults.getValue(); 468 double delta = (baselineValue - buildValue) / baselineValue; 469 470 // Store if there's no baseline 471 if (Double.isNaN(delta)) { 472 this.status |= NO_BASELINE; 473 } 474 475 // Store if there's only one run 476 long baselineCount = baselineResults.getCount(); 477 long currentCount = buildResults.getCount(); 478 double error = Double.NaN; 479 if (baselineCount == 1 || currentCount == 1) { 480 this.status |= SINGLE_RUN; 481 } 482 483 // Store if the T-test is not good 484 double ttestValue = Util.computeTTest(baselineResults, buildResults); 485 int degreeOfFreedom = (int) (baselineResults.getCount()+buildResults.getCount()-2); 486 if (ttestValue >= 0 && StatisticsUtil.getStudentsT(degreeOfFreedom, StatisticsUtil.T90) >= ttestValue) { 487 this.status |= STUDENT_TTEST; 488 } 489 490 // Store if there's a big error (over 3%) 491 double baselineError = baselineResults.getError(); 492 double currentError = buildResults.getError(); 493 error = Double.isNaN(baselineError) 494 ? currentError / baselineValue 495 : Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue; 496 if (error > 0.03) { 497 this.status |= BIG_ERROR; 498 } 499 500 // Store if there's a big delta (over 10%) 501 if (delta <= -0.1) { 502 this.status |= BIG_DELTA; 503 double currentBuildValue = buildResults.getValue(); 504 double diff = Math.abs(baselineValue - currentBuildValue); 505 if (currentBuildValue < 100 || diff < 100) { // moderate the status when 506 // diff is less than 100ms 507 this.status |= SMALL_VALUE; 508 } else { 509 double[] stats = getStatistics(); 510 if (stats != null) { 511 if (stats[3] > 0.2) { // invalidate the status when the test 512 // historical deviation is over 20% 513 this.status |= NOT_RELIABLE; 514 } else if (stats[3] > 0.1) { // moderate the status when the test 515 // historical deviation is between 10% 516 // and 20% 517 this.status |= NOT_STABLE; 518 } 519 } 520 } 521 } 522 523 return this.status; 524 } 525 526 public boolean isInitialized() { 527 return this.results != null; 528 } 529 530 /* (non-Javadoc) 531 * Method declared on IPropertySource 532 */ 533 public boolean isPropertySet(Object property) { 534 return false; 535 } 536 537 boolean onlyFingerprints() { 538 if (this.parent != null) { 539 return this.parent.onlyFingerprints(); 540 } 541 return ((PerformanceResultsElement)this).fingerprints; 542 } 543 544 /* (non-Javadoc) 545 * Method declared on IPropertySource 546 */ 547 public void resetPropertyValue(Object property) { 548 } 549 550 void resetStatus() { 551 this.status = -1; 552 if (this.results != null) { 553 if (this.children == null) initChildren(); 554 int length = this.children.length; 555 for (int i=0; i<length; i++) { 556 this.children[i].resetStatus(); 557 } 558 } 559 } 560 561 public void setPropertyValue(Object name, Object value) { 562 if (name.equals(P_ID_STATUS_COMMENT)) { 563 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID); 564 preferences.put(getId(), (String) value); 565 try { 566 preferences.flush(); 567 } catch (BackingStoreException e) { 568 // skip 569 } 570 } 571 } 572 573 /** 574 * Sets the image descriptor 575 */ 576 void setImageDescriptor(ImageDescriptor desc) { 577 // this.imageDescriptor = desc; 578 } 579 580 public String toString() { 581 if (this.results == null) { 582 return getName(); 583 } 584 return this.results.toString(); 585 } 586 587 /* 588 * Write the element status in the given stream 589 */ 590 StringBuffer writableStatus(StringBuffer buffer, int kind, StringBuffer excluded) { 591 int length = this.children.length; 592 for (int i=0; i<length; i++) { 593 this.children[i].writableStatus(buffer, kind, excluded); 594 } 595 return buffer; 596 } 597 598 599 } 600
