1 /******************************************************************************* 2 * Copyright (c) 2000, 2009 IBM Corporation and others. 3 * All rights reserved. This program and the accompanying materials 4 * are made available under the terms of the Eclipse Public License v1.0 5 * which accompanies this distribution, and is available at 6 * http://www.eclipse.org/legal/epl-v10.html 7 * 8 * Contributors: 9 * IBM Corporation - initial API and implementation 10 *******************************************************************************/ 11 package org.eclipse.test.internal.performance.results.model; 12 13 import java.util.ArrayList; 14 import java.util.List; 15 import java.util.Vector; 16 17 import org.eclipse.test.internal.performance.results.db.*; 18 import org.eclipse.test.internal.performance.results.utils.IPerformancesConstants; 19 import org.eclipse.test.internal.performance.results.utils.Util; 20 import org.eclipse.ui.views.properties.ComboBoxPropertyDescriptor; 21 import org.eclipse.ui.views.properties.IPropertyDescriptor; 22 import org.eclipse.ui.views.properties.PropertyDescriptor; 23 import org.eclipse.ui.views.properties.TextPropertyDescriptor; 24 25 public class ConfigResultsElement extends ResultsElement { 26 27 // Elements 28 BuildResultsElement currentBuild, baselineBuild; 29 30 // Property descriptors 31 static final String P_ID_CONFIG_NAME = "ConfigResultsElement.name"; //$NON-NLS-1$ 32 static final String P_ID_CONFIG_DESCRIPTION = "ConfigResultsElement.description"; //$NON-NLS-1$ 33 static final String P_ID_CONFIG_CURRENT_BUILD = "ConfigResultsElement.currentbuild"; //$NON-NLS-1$ 34 static final String P_ID_CONFIG_BASELINE_BUILD = "ConfigResultsElement.baselinebuild"; //$NON-NLS-1$ 35 static final String P_ID_CONFIG_BASELINED = "ConfigResultsElement.baselined"; //$NON-NLS-1$ 36 static final String P_ID_CONFIG_VALID = "ConfigResultsElement.valid"; //$NON-NLS-1$ 37 static final String P_ID_CONFIG_DELTA = "ConfigResultsElement.delta"; //$NON-NLS-1$ 38 static final String P_ID_CONFIG_ERROR = "ConfigResultsElement.error"; //$NON-NLS-1$ 39 40 static final String P_STR_CONFIG_NAME = "internal name"; //$NON-NLS-1$ 41 static final String P_STR_CONFIG_DESCRIPTION = "description"; //$NON-NLS-1$ 42 static final String P_STR_CONFIG_CURRENT_BUILD = "current build"; //$NON-NLS-1$ 43 static final String P_STR_CONFIG_BASELINE_BUILD = "baseline build"; //$NON-NLS-1$ 44 static final String P_STR_CONFIG_BASELINED = "has baseline"; //$NON-NLS-1$ 45 static final String P_STR_CONFIG_VALID = "is valid"; //$NON-NLS-1$ 46 static final String P_STR_CONFIG_DELTA = "delta with baseline"; //$NON-NLS-1$ 47 static final String P_STR_CONFIG_ERROR = "delta error"; //$NON-NLS-1$ 48 49 private static final TextPropertyDescriptor CONFIG_NAME_DESCRIPTOR = new TextPropertyDescriptor(P_ID_CONFIG_NAME, P_STR_CONFIG_NAME); 50 private static final TextPropertyDescriptor CONFIG_DESCRIPTION_DESCRIPTOR = new TextPropertyDescriptor(P_ID_CONFIG_DESCRIPTION, P_STR_CONFIG_DESCRIPTION); 51 private static final PropertyDescriptor CONFIG_CURRENT_BUILD_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_CURRENT_BUILD, P_STR_CONFIG_CURRENT_BUILD); 52 private static final PropertyDescriptor CONFIG_BASELINE_BUILD_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_BASELINE_BUILD, P_STR_CONFIG_BASELINE_BUILD); 53 private static final PropertyDescriptor CONFIG_BASELINED_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_BASELINED, P_STR_CONFIG_BASELINED); 54 private static final PropertyDescriptor CONFIG_VALID_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_VALID, P_STR_CONFIG_VALID); 55 private static final PropertyDescriptor CONFIG_DELTA_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_DELTA, P_STR_CONFIG_DELTA); 56 private static final PropertyDescriptor CONFIG_ERROR_DESCRIPTOR = new PropertyDescriptor(P_ID_CONFIG_ERROR, P_STR_CONFIG_ERROR); 57 58 private static Vector DESCRIPTORS; 59 static Vector initDescriptors(int status) { 60 DESCRIPTORS = new Vector(); 61 // Status category 62 DESCRIPTORS.add(getInfosDescriptor(status)); 63 DESCRIPTORS.add(getWarningsDescriptor(status)); 64 DESCRIPTORS.add(ERROR_DESCRIPTOR); 65 ERROR_DESCRIPTOR.setCategory("Status"); 66 // Results category 67 DESCRIPTORS.addElement(CONFIG_NAME_DESCRIPTOR); 68 CONFIG_NAME_DESCRIPTOR.setCategory("Results"); 69 DESCRIPTORS.addElement(CONFIG_DESCRIPTION_DESCRIPTOR); 70 CONFIG_DESCRIPTION_DESCRIPTOR.setCategory("Results"); 71 DESCRIPTORS.addElement(CONFIG_CURRENT_BUILD_DESCRIPTOR); 72 CONFIG_CURRENT_BUILD_DESCRIPTOR.setCategory("Results"); 73 DESCRIPTORS.addElement(CONFIG_BASELINE_BUILD_DESCRIPTOR); 74 CONFIG_BASELINE_BUILD_DESCRIPTOR.setCategory("Results"); 75 DESCRIPTORS.addElement(CONFIG_BASELINED_DESCRIPTOR); 76 CONFIG_BASELINED_DESCRIPTOR.setCategory("Results"); 77 DESCRIPTORS.addElement(CONFIG_VALID_DESCRIPTOR); 78 CONFIG_VALID_DESCRIPTOR.setCategory("Results"); 79 DESCRIPTORS.addElement(CONFIG_DELTA_DESCRIPTOR); 80 CONFIG_DELTA_DESCRIPTOR.setCategory("Results"); 81 DESCRIPTORS.addElement(CONFIG_ERROR_DESCRIPTOR); 82 CONFIG_ERROR_DESCRIPTOR.setCategory("Results"); 83 // Survey category 84 DESCRIPTORS.add(COMMENT_DESCRIPTOR); 85 COMMENT_DESCRIPTOR.setCategory("Survey"); 86 return DESCRIPTORS; 87 } 88 static ComboBoxPropertyDescriptor getInfosDescriptor(int status) { 89 List list = new ArrayList(); 90 if ((status & SMALL_VALUE) != 0) { 91 list.add("This test and/or its variation has a small value on this machine, hence it may not be necessary to spend time on fixing it if a regression occurs"); 92 } 93 if ((status & STUDENT_TTEST) != 0) { 94 list.add("The student-t test error on this machine is over the threshold"); 95 } 96 String[] infos = new String[list.size()]; 97 if (list.size() > 0) { 98 list.toArray(infos); 99 } 100 ComboBoxPropertyDescriptor infoDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_INFO, P_STR_STATUS_INFO, infos); 101 infoDescriptor.setCategory("Status"); 102 return infoDescriptor; 103 } 104 static PropertyDescriptor getWarningsDescriptor(int status) { 105 List list = new ArrayList(); 106 if ((status & BIG_ERROR) != 0) { 107 list.add("The error on this machine is over the 3% threshold, hence its result may not be really reliable"); 108 } 109 if ((status & NOT_RELIABLE) != 0) { 110 list.add("The results history for this machine shows that the variation of its delta is over 20%, hence its result is surely not reliable"); 111 } 112 if ((status & NOT_STABLE) != 0) { 113 list.add("The results history for this machine shows that the variation of its delta is between 10% and 20%, hence its result may not be really reliable"); 114 } 115 if ((status & NO_BASELINE) != 0) { 116 list.add("There's no baseline for this machine to compare with"); 117 } 118 if ((status & SINGLE_RUN) != 0) { 119 list.add("This test has only one run on this machine, hence no error can be computed to verify if it's stable enough to be reliable"); 120 } 121 if ((status & STUDENT_TTEST) != 0) { 122 list.add("The student-t test error on this machine is over the threshold"); 123 } 124 String[] warnings = new String[list.size()]; 125 if (list.size() > 0) { 126 list.toArray(warnings); 127 } 128 ComboBoxPropertyDescriptor warningDescriptor = new ComboBoxPropertyDescriptor(P_ID_STATUS_WARNING, P_STR_STATUS_WARNING, warnings); 129 warningDescriptor.setCategory("Status"); 130 return warningDescriptor; 131 } 132 static Vector getDescriptors() { 133 return DESCRIPTORS; 134 } 135 136 public ConfigResultsElement(AbstractResults results, ResultsElement parent) { 137 super(results, parent); 138 } 139 140 public int compareTo(Object o) { 141 // TODO Auto-generated method stub 142 return super.compareTo(o); 143 } 144 ResultsElement createChild(AbstractResults testResults) { 145 return new BuildResultsElement(testResults, this); 146 } 147 148 BuildResultsElement getBaselineBuild() { 149 if (this.baselineBuild == null) { 150 this.baselineBuild = new BuildResultsElement(getConfigResults().getBaselineBuildResults(), this); 151 } 152 return this.baselineBuild; 153 } 154 155 /** 156 * Get the baseline build used for this configuration. 157 * 158 * @param buildName The name of the build to have the baseline 159 * @return The baseline build as {@link BuildResultsElement}. 160 */ 161 public String getBaselineBuildName(String buildName) { 162 return getConfigResults().getBaselineBuildResults(buildName).getName(); 163 } 164 165 private ConfigResults getConfigResults() { 166 return (ConfigResults) this.results; 167 } 168 169 BuildResultsElement getCurrentBuild() { 170 if (this.currentBuild == null) { 171 this.currentBuild = new BuildResultsElement(getConfigResults().getCurrentBuildResults(), this); 172 } 173 return this.currentBuild; 174 } 175 176 public String getLabel(Object o) { 177 String description = getConfigResults().getDescription(); 178 int index = description.indexOf(" ("); 179 if (index <= 0) { 180 return description; 181 } 182 return description.substring(0, index); 183 } 184 185 /* 186 * (non-Javadoc) 187 * 188 * @see org.eclipse.ui.views.properties.IPropertySource#getPropertyDescriptors() 189 */ 190 public IPropertyDescriptor[] getPropertyDescriptors() { 191 Vector descriptors = getDescriptors(); 192 if (descriptors == null) { 193 descriptors = initDescriptors(getStatus()); 194 } 195 int size = descriptors.size(); 196 IPropertyDescriptor[] descriptorsArray = new IPropertyDescriptor[size]; 197 descriptorsArray[0] = getInfosDescriptor(getStatus()); 198 descriptorsArray[1] = getWarningsDescriptor(getStatus()); 199 for (int i=2; i<size; i++) { 200 descriptorsArray[i] = (IPropertyDescriptor) descriptors.get(i); 201 } 202 return descriptorsArray; 203 } 204 205 /* 206 * (non-Javadoc) 207 * 208 * @see 209 * org.eclipse.ui.views.properties.IPropertySource#getPropertyValue(java.lang 210 * .Object) 211 */ 212 public Object getPropertyValue(Object propKey) { 213 ConfigResults configResults = getConfigResults(); 214 if (propKey.equals(P_ID_CONFIG_NAME)) { 215 return configResults.getName(); 216 } 217 if (propKey.equals(P_ID_CONFIG_DESCRIPTION)) { 218 return configResults.getDescription(); 219 } 220 if (propKey.equals(P_ID_CONFIG_CURRENT_BUILD)) { 221 return getCurrentBuild(); 222 } 223 if (propKey.equals(P_ID_CONFIG_BASELINE_BUILD)) { 224 return getBaselineBuild(); 225 } 226 if (propKey.equals(P_ID_CONFIG_BASELINED)) { 227 return new Boolean(configResults.isBaselined()); 228 } 229 if (propKey.equals(P_ID_CONFIG_VALID)) { 230 return new Boolean(configResults.isValid()); 231 } 232 if (propKey.equals(P_ID_CONFIG_DELTA)) { 233 return new Double(configResults.getDelta()); 234 } 235 if (propKey.equals(P_ID_CONFIG_ERROR)) { 236 return new Double(configResults.getError()); 237 } 238 if (propKey.equals(P_ID_STATUS_ERROR)) { 239 if (getStatus() == MISSING) { 240 PerformanceResultsElement performanceResultsElement = (PerformanceResultsElement) ((ResultsElement)((ResultsElement)getParent(null)).getParent(null)).getParent(null); 241 return "No result for build "+performanceResultsElement.getName()+" on this machine!"; 242 } 243 if ((getStatus() & BIG_DELTA) != 0) { 244 return "The delta on this machine is over the 10% threshold, hence may indicate a possible regression"; 245 } 246 } 247 return super.getPropertyValue(propKey); 248 } 249 250 /** 251 * Return the statistics of the build along its history. 252 * 253 * @return An array of double built as follows: 254 * <ul> 255 * <li>0: numbers of values</li> 256 * <li>1: mean of values</li> 257 * <li>2: standard deviation of these values</li> 258 * <li>3: coefficient of variation of these values</li> 259 * </ul> 260 */ 261 public double[] getStatistics() { 262 if (this.statistics == null) { 263 this.statistics = getConfigResults().getStatistics(Util.BASELINE_BUILD_PREFIXES); 264 } 265 return this.statistics; 266 } 267 268 void initStatus() { 269 ConfigResults configResults = getConfigResults(); 270 if (configResults.isValid()) { 271 initStatus(configResults.getCurrentBuildResults()); 272 } else { 273 this.status = MISSING; 274 } 275 } 276 277 /* 278 * Write the element status in the given stream 279 */ 280 StringBuffer writableStatus(StringBuffer buffer, int kind, StringBuffer excluded) { 281 if ((this.status & BIG_DELTA) != 0) { // there's a failure on this config 282 283 // Get numbers 284 int buildsNumber = kind & IPerformancesConstants.STATUS_BUILDS_NUMBER_MASK; 285 ConfigResults configResults = getConfigResults(); 286 double[][] numbers = configResults.getLastNumbers(buildsNumber); 287 int numbersLength = numbers.length; 288 289 // if there are several builds to confirm the regression, then verify all deltas 290 if (numbersLength > 1) { 291 if (numbersLength < buildsNumber) { 292 // there's not enough builds to wee whether there's a real regression, hence skip result 293 if (excluded != null) { 294 excluded.append(configResults+" excluded from status because there's only "+numbersLength+" builds available although "+buildsNumber+" is required to decide a regression is confirmed or not!"); 295 excluded.append(Util.LINE_SEPARATOR); 296 } 297 return buffer; 298 } 299 int confirmed = 1; 300 for (int i=1; i<numbersLength; i++) { 301 if (numbers[i][AbstractResults.DELTA_VALUE_INDEX] < -0.1) { 302 confirmed++; 303 } 304 } 305 float ratio = ((float) confirmed) / numbersLength; 306 if (ratio < 0.8) { 307 // more than 20% of previous build didn't fail, hence skip result 308 if (excluded != null) { 309 excluded.append(configResults+" excluded from status because only "+confirmed+" builds failed on last "+buildsNumber+" ones!"); 310 excluded.append(Util.LINE_SEPARATOR); 311 } 312 return buffer; 313 } 314 } 315 316 // Add values 317 double[] values = numbers[0]; 318 double buildValue = values[AbstractResults.BUILD_VALUE_INDEX]; 319 double baselineValue = values[AbstractResults.BASELINE_VALUE_INDEX]; 320 double delta = values[AbstractResults.DELTA_VALUE_INDEX]; 321 double error = values[AbstractResults.DELTA_ERROR_INDEX]; 322 StringBuffer localBuffer = new StringBuffer(" "); 323 localBuffer.append(configResults.getName()); 324 double[] stats = null; 325 boolean printValues = (kind & IPerformancesConstants.STATUS_VALUES) != 0; 326 if (printValues) { 327 localBuffer.append(" "); 328 localBuffer.append(buildValue); 329 localBuffer.append(" "); 330 localBuffer.append(baselineValue); 331 localBuffer.append(" "); 332 localBuffer.append(buildValue-baselineValue); 333 localBuffer.append(" "); 334 localBuffer.append(Util.PERCENTAGE_FORMAT.format(delta)); 335 localBuffer.append(" "); 336 localBuffer.append(Util.PERCENTAGE_FORMAT.format(error)); 337 stats = getStatistics(); 338 if (stats != null) { 339 localBuffer.append(" "); 340 localBuffer.append((int) stats[0]); 341 localBuffer.append(" "); 342 localBuffer.append(Util.DOUBLE_FORMAT.format(stats[1])); 343 localBuffer.append(" "); 344 localBuffer.append(Util.DOUBLE_FORMAT.format(stats[2])); 345 localBuffer.append(" "); 346 localBuffer.append(Util.PERCENTAGE_FORMAT.format(stats[3])); 347 } 348 } 349 350 /* Add comment 351 IEclipsePreferences preferences = new InstanceScope().getNode(IPerformancesConstants.PLUGIN_ID); 352 String comment = preferences.get(getId(), null); 353 if (comment != null) { 354 if (stats == null && printValues) { 355 buffer.append(" "); 356 } 357 buffer.append(" "); 358 buffer.append(comment); 359 } 360 */ 361 362 // Add status info 363 if (this.status != BIG_DELTA) { // there's some other info in the status 364 // if (comment == null) { 365 if (stats == null && printValues) { 366 localBuffer.append(" "); 367 } 368 // } 369 localBuffer.append(" "); 370 String separator = ""; 371 372 // Error 373 if ((this.status & BIG_ERROR) != 0) { 374 int statusErrorLevel = kind & IPerformancesConstants.STATUS_ERROR_LEVEL_MASK; 375 if (statusErrorLevel == IPerformancesConstants.STATUS_ERROR_NOTICEABLE) { 376 // Skip result 377 if (excluded != null) { 378 excluded.append(configResults+" excluded from status due to a noticeable error!"); 379 excluded.append(Util.LINE_SEPARATOR); 380 } 381 return buffer; 382 } 383 localBuffer.append(separator); 384 localBuffer.append("error ("); 385 localBuffer.append(Util.PERCENTAGE_FORMAT.format(error)); 386 localBuffer.append(")"); 387 separator = "+"; 388 double ratio = -(error/delta); 389 if (ratio > 1) { 390 switch (statusErrorLevel) { 391 case IPerformancesConstants.STATUS_ERROR_INVALID: 392 case IPerformancesConstants.STATUS_ERROR_WEIRD: 393 case IPerformancesConstants.STATUS_ERROR_SUSPICIOUS: 394 // Skip result 395 if (excluded != null) { 396 excluded.append(configResults+" excluded from status due to an invalid error!"); 397 excluded.append(Util.LINE_SEPARATOR); 398 } 399 return buffer; 400 } 401 localBuffer.append(": invalid measure!"); 402 } else if (ratio > 0.5) { 403 switch (statusErrorLevel) { 404 case IPerformancesConstants.STATUS_ERROR_WEIRD: 405 case IPerformancesConstants.STATUS_ERROR_SUSPICIOUS: 406 // Skip result 407 if (excluded != null) { 408 excluded.append(configResults+" excluded from status due to a weird error!"); 409 excluded.append(Util.LINE_SEPARATOR); 410 } 411 return buffer; 412 } 413 localBuffer.append(": weird measure!"); 414 } else if (ratio > 0.25) { 415 if (statusErrorLevel == IPerformancesConstants.STATUS_ERROR_SUSPICIOUS) { 416 // Skip result 417 if (excluded != null) { 418 excluded.append(configResults+" excluded from status due to a suspicious error!"); 419 excluded.append(Util.LINE_SEPARATOR); 420 } 421 return buffer; 422 } 423 localBuffer.append(": suspicious measure!"); 424 } 425 } 426 427 // Small value 428 if ((this.status & SMALL_VALUE) != 0) { 429 int statusSmallValue = kind & IPerformancesConstants.STATUS_SMALL_VALUE_MASK; 430 localBuffer.append(separator); 431 if (buildValue < 100) { 432 if (statusSmallValue == IPerformancesConstants.STATUS_SMALL_VALUE_BUILD) { 433 // Skip result 434 if (excluded != null) { 435 excluded.append(configResults+" excluded from status due to a small build value!"); 436 excluded.append(Util.LINE_SEPARATOR); 437 } 438 return buffer; 439 } 440 localBuffer.append("small build value ("); 441 localBuffer.append((int)buildValue); 442 localBuffer.append("ms)"); 443 } 444 int diff = (int) Math.abs(baselineValue - buildValue); 445 if (diff < 100) { 446 if (statusSmallValue == IPerformancesConstants.STATUS_SMALL_VALUE_DELTA) { 447 // Skip result 448 if (excluded != null) { 449 excluded.append(configResults+" excluded from status due to a small delta value!"); 450 excluded.append(Util.LINE_SEPARATOR); 451 } 452 return buffer; 453 } 454 localBuffer.append("small delta value ("); 455 localBuffer.append(diff); 456 localBuffer.append("ms)"); 457 } 458 separator = "+"; 459 } 460 461 // Statistics 462 if ((this.status & NOT_RELIABLE) != 0) { 463 switch (kind & IPerformancesConstants.STATUS_STATISTICS_MASK) { 464 case IPerformancesConstants.STATUS_STATISTICS_UNSTABLE: 465 case IPerformancesConstants.STATUS_STATISTICS_ERRATIC: 466 // Skip result 467 if (excluded != null) { 468 excluded.append(configResults+" excluded from status due to erratic statistics!"); 469 excluded.append(Util.LINE_SEPARATOR); 470 } 471 return buffer; 472 } 473 localBuffer.append(separator); 474 localBuffer.append("erratic"); 475 separator = "+"; 476 } else if ((this.status & NOT_STABLE) != 0) { 477 if ((kind & IPerformancesConstants.STATUS_STATISTICS_UNSTABLE) != 0) { 478 // Skip result 479 if (excluded != null) { 480 excluded.append(configResults+" excluded from status due to unstable statistics!"); 481 excluded.append(Util.LINE_SEPARATOR); 482 } 483 return buffer; 484 } 485 localBuffer.append(separator); 486 localBuffer.append("unstable"); 487 separator = "+"; 488 } 489 } 490 491 // Write status 492 buffer.append(localBuffer); 493 buffer.append(Util.LINE_SEPARATOR); 494 } 495 return buffer; 496 } 497 498 } 499
