results/db/ConfigResults.java

/*******************************************************************************
 * Copyright (c) 2000, 2009 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/
package org.eclipse.test.internal.performance.results.db;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.test.internal.performance.InternalDimensions;
import org.eclipse.test.internal.performance.results.utils.Util;

/**
 * Class to handle results for an Eclipse performance test box
 * (called a <i>configuration</i>).
 *
 * It gives access to results for each build on which this configuration has been run.
 *
 * @see BuildResults
 */
public class ConfigResults extends AbstractResults {
	BuildResults baseline, current;
	boolean baselined = false, valid = false;
	double delta, error;

public ConfigResults(AbstractResults parent, int id) {
	super(parent, id);
	this.name = parent.getPerformance().sortedConfigNames[id];
	this.printStream = parent.printStream;
}

/*
 * Complete results with additional database information.
 */
void completeResults(String[] builds) {
	/*if (this.baseline == null || this.current == null) */initialize();
	ScenarioResults scenarioResults = (ScenarioResults) this.parent;
	DB_Results.queryScenarioSummaries(scenarioResults, this.name, builds);
}

/**
 * Returns the baseline build name used to compare results with.
 *
 * @return The name of the baseline build
 * @see #getBaselineBuildResults()
 */
public String getBaselineBuildName() {
	if (this.baseline == null) {
	    initialize();
    }
	return this.baseline.getName();
}

/**
 * Returns the most recent baseline build results.
 *
 * @return The {@link BuildResults baseline build results}.
 * @see BuildResults
 */
public BuildResults getBaselineBuildResults() {
	if (this.baseline == null) {
	    initialize();
    }
	return this.baseline;
}

/**
 * Return the baseline build results run just before the given build name.
 *
 * @param buildName The build name
 * @return The {@link BuildResults baseline results} preceding the given build name
 * 	or <code>null</code> if none was found.
 */
public BuildResults getBaselineBuildResults(String buildName) {
	if (this.baseline == null) {
	    initialize();
    }
	int size = this.children.size();
	String buildDate = Util.getBuildDate(buildName);
	for (int i=size-1; i>=0; i--) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (buildResults.isBaseline() && buildResults.getDate().compareTo(buildDate) < 0) {
			return buildResults;
		}
	}
	return this.baseline;

}

/**
 * Returns the most recent baseline build result value.
 *
 * @return The value of the most recent baseline build results.
 */
public double getBaselineBuildValue() {
	if (this.baseline == null) {
	    initialize();
    }
	return this.baseline.getValue();
}

/**
 * Returns the configuration description (currently the box name).
 *
 * @return The configuration description (currently the box name).
 */
public String getDescription() {
	return getPerformance().sortedConfigDescriptions[this.id];
}

/**
 * Return the results for the given build name.
 *
 * @param buildName The build name
 * @return The {@link BuildResults results} for the given build name
 * 	or <code>null</code> if none was found.
 */
public BuildResults getBuildResults(String buildName) {
	return (BuildResults) getResults(buildName);
}

/**
 * Returns the build results matching a given pattern.
 *
 * @param buildPattern The pattern of searched builds
 * @return The list of the builds which names match the given pattern.
 * 	The list is ordered by build results date.
 */
public List getBuilds(String buildPattern) {
	List builds = new ArrayList();
	int size = size();
	for (int i=0; i<size; i++) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (buildPattern == null || buildResults.match(buildPattern)) {
			builds.add(buildResults);
		}
	}
	return builds;
}

/**
 * Returns the build results before a given build name.
 *
 * @param buildName Name of the last build (included)
 * @return The list of the builds which precedes the given build name.
 */
public List getBuildsBefore(String buildName) {
	String buildDate = Util.getBuildDate(buildName);
	List builds = new ArrayList();
	int size = size();
	for (int i=0; i<size; i++) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (buildName == null || buildResults.getDate().compareTo(buildDate) <= 0) {
			builds.add(buildResults);
		}
	}
	return builds;
}

/**
 * Returns a list of build results which names starts with one of the given prefixes.
 *
 * @param prefixes List of expected prefixes
 * @return A list of builds which names start with one of the given patterns.
 */
public List getBuildsMatchingPrefixes(List prefixes) {
	List builds = new ArrayList();
	int size = size();
	int length = prefixes.size();
	for (int i=0; i<size; i++) {
		AbstractResults buildResults = (AbstractResults) this.children.get(i);
		String buildName = buildResults.getName();
		for (int j=0; j<length; j++) {
			if (buildName.startsWith((String)prefixes.get(j))) {
				builds.add(buildResults);
			}
		}
	}
	return builds;
}

/**
 * Get all results numbers for the max last builds.
 *
 * @param max The number of last builds to get numbers.
 * @return An 2 dimensions array of doubles. At the first level of the array each slot
 * 		represents one build. That means that the dimension of the array matches
 * 		the given numbers as soon as there are enough builds in the database.
 * <p>
 * 		The slots of the second level are the numbers values:
 * 	<ul>
 * 		<li>{@link #BUILD_VALUE_INDEX}: the build value in milliseconds</li>
 * 		<li>{@link #BASELINE_VALUE_INDEX}: the baseline value in milliseconds</li>
 * 		<li>{@link #DELTA_VALUE_INDEX}: the difference between the build value and its more recent baseline</li>
 * 		<li>{@link #DELTA_ERROR_INDEX}: the error made while computing the difference</li>
 * 		<li>{@link #BUILD_ERROR_INDEX}: the error made while measuring the build value</li>
 * 		<li>{@link #BASELINE_ERROR_INDEX}: the error made while measuring the baseline value</li>
 * 	</ul>
*/
public double[][] getLastNumbers(int max) {

	// Return null if no previous builds are expected
	if (max <= 0) return null;

	// Add numbers for each previous build
	int size = size();
	double[][] numbers = new double[Math.min(max, size)][];
	int n = 0;
	for (int i=size-1; i>=0 && n<max; i--) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (!buildResults.isBaseline()) {
			numbers[n] = getNumbers(buildResults, getBaselineBuildResults(buildResults.getName()));
			n++;
		}
	}

	// Return the numbers
	return numbers;
}

/**
 * Returns interesting numbers for the current configuration.
 *
 * @return Values in an array of double:
 * 	<ul>
 * 		<li>{@link AbstractResults#BUILD_VALUE_INDEX}: the build value in milliseconds</li>
 * 		<li>{@link AbstractResults#BASELINE_VALUE_INDEX}: the baseline value in milliseconds</li>
 * 		<li>{@link AbstractResults#DELTA_VALUE_INDEX}: the difference between the build value and its more recent baseline</li>
 * 		<li>{@link AbstractResults#DELTA_ERROR_INDEX}: the error made while computing the difference</li>
 * 		<li>{@link AbstractResults#BUILD_ERROR_INDEX}: the error made while measuring the build value</li>
 * 		<li>{@link AbstractResults#BASELINE_ERROR_INDEX}: the error made while measuring the baseline value</li>
 * 	</ul>
 */
double[] getNumbers(BuildResults buildResults, BuildResults baselineResults) {
	if (baselineResults == null) {
		return null;
	}
	double[] values = new double[NUMBERS_LENGTH];
	for (int i=0 ;i<NUMBERS_LENGTH; i++) {
		values[i] = Double.NaN;
	}
	double buildValue = buildResults.getValue();
	values[BUILD_VALUE_INDEX] = buildValue;
	double baselineValue = baselineResults.getValue();
	values[BASELINE_VALUE_INDEX] = baselineValue;
	double buildDelta = (baselineValue - buildValue) / baselineValue;
	values[DELTA_VALUE_INDEX] = buildDelta;
	if (Double.isNaN(buildDelta)) {
		return values;
	}
	long baselineCount = baselineResults.getCount();
	long currentCount = buildResults.getCount();
	if (baselineCount > 1 && currentCount > 1) {
		double baselineError = baselineResults.getError();
		double currentError = buildResults.getError();
		values[BASELINE_ERROR_INDEX] = baselineError;
		values[BUILD_ERROR_INDEX] = currentError;
		values[DELTA_ERROR_INDEX] = Double.isNaN(baselineError)
				? currentError / baselineValue
				: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
	}
	return values;
}

/**
 * Return the deviation value and its associated standard error for the default dimension
 * (currently {@link InternalDimensions#ELAPSED_PROCESS}).
 *
 * @return an array of double. First number is the deviation itself and
 * 	the second is the standard error.
 */
public double[] getCurrentBuildDeltaInfo() {
	if (this.baseline == null || this.current == null) {
		initialize();
	}
	return new double[] { this.delta, this.error };
}

/**
 * Returns the error of the current build results
 *
 * @return the error made during the current build measure
 */
public double getCurrentBuildError() {
	if (this.current == null) {
	    initialize();
    }
	return this.current.getError();
}

/**
 * Returns the current build name.
 *
 * @return The name of the current build
 * @see #getCurrentBuildResults()
 */
public String getCurrentBuildName() {
	if (this.current == null) {
	    initialize();
    }
	return this.current.getName();
}

/**
 * Returns the current build results.
 * <p>
 * This build is currently the last integration or nightly
 * build which has performance results in the database.
 * It may differ from the {@link PerformanceResults#getName()}.
 *
 * @return The current build results.
 * @see BuildResults
 */
public BuildResults getCurrentBuildResults() {
	if (this.current == null) {
	    initialize();
    }
	return this.current;
}

/**
 * Returns the current build result value.
 *
 * @return The value of the current build results.
 */
public double getCurrentBuildValue() {
	if (this.current == null) {
	    initialize();
    }
	return this.current.getValue();
}

/**
 * Returns the delta between current and baseline builds results.
 *
 * @return the delta
 */
public double getDelta() {
	if (this.baseline == null || this.current == null) {
		initialize();
	}
	return this.delta;
}

/**
 * Returns the standard error of the delta between current and baseline builds results.
 *
 * @return the delta
 * @see #getDelta()
 */
public double getError() {
	if (this.baseline == null || this.current == null) {
		initialize();
	}
	return this.error;
}

/**
 * Return the name of the machine associated with the current config.
 *
 * @return The name of the machine.
 */
public String getLabel() {
	return this.parent.getPerformance().sortedConfigDescriptions[this.id];
}

/**
 * Get all dimension builds default dimension statistics for all builds.
 *
 * @return An array of double built as follows:
 * <ul>
 * <li>0:	numbers of values</li>
 * <li>1:	mean of values</li>
 * <li>2:	standard deviation of these values</li>
 * <li>3:	coefficient of variation of these values</li>
 * </ul>
 */
public double[] getStatistics() {
	return getStatistics(Util.ALL_BUILD_PREFIXES, DB_Results.getDefaultDimension().getId());
}

/**
 * Get all dimension builds default dimension statistics for a given list of build
 * prefixes.
 *
 * @param prefixes List of prefixes to filter builds. If <code>null</code>
 * 	then all the builds are taken to compute statistics.
 * @return An array of double built as follows:
 * <ul>
 * <li>0:	numbers of values</li>
 * <li>1:	mean of values</li>
 * <li>2:	standard deviation of these values</li>
 * <li>3:	coefficient of variation of these values</li>
 * </ul>
 */
public double[] getStatistics(List prefixes) {
	return getStatistics(prefixes, DB_Results.getDefaultDimension().getId());
}

/**
 * Get all dimension builds statistics for a given list of build prefixes
 * and a given dimension.
 *
 * @param prefixes List of prefixes to filter builds. If <code>null</code>
 * 	then all the builds are taken to compute statistics.
 * @param dim_id The id of the dimension on which the statistics must be computed
 * @return An array of double built as follows:
 * <ul>
 * <li>0:	numbers of values</li>
 * <li>1:	mean of values</li>
 * <li>2:	standard deviation of these values</li>
 * <li>3:	coefficient of variation of these values</li>
 * </ul>
 */
public double[] getStatistics(List prefixes, int dim_id) {
	int size = size();
	int length = prefixes == null ? 0 : prefixes.size();
	int count = 0;
	double mean=0, stddev=0, variation=0;
	double[] values = new double[size];
	count = 0;
	mean = 0.0;
	for (int i=0; i<size; i++) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		String buildName = buildResults.getName();
		if (isBuildConcerned(buildResults)) {
			if (prefixes == null) {
				double value = buildResults.getValue(dim_id);
				values[count] = value;
				mean += value;
				count++;
			} else {
				for (int j=0; j<length; j++) {
					if (buildName.startsWith((String)prefixes.get(j))) {
						double value = buildResults.getValue(dim_id);
						values[count] = value;
						mean += value;
						count++;
					}
				}
			}
		}
	}
	mean /= count;
	for (int i=0; i<count; i++) {
		stddev += Math.pow(values[i] - mean, 2);
	}
	stddev = Math.sqrt((stddev / (count - 1)));
	variation = stddev / mean;
	return new double[] { count, mean, stddev, variation };
}

private void initialize() {
	reset();
	// Get performance results builds name
	PerformanceResults perfResults = getPerformance();
	String baselineBuildName = perfResults.getBaselineName();
	String baselineBuildDate = baselineBuildName == null ? null : Util.getBuildDate(baselineBuildName);
	String currentBuildName = perfResults.name;
	String currentBuildDate = currentBuildName == null ? null : Util.getBuildDate(currentBuildName);

	// Set baseline and current builds
	BuildResults lastBaseline = null;
	int size = size();
	if (size == 0) return;
	for (int i=0; i<size; i++) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (buildResults.values != null) {
			buildResults.cleanValues();
		}
		if (buildResults.isBaseline()) {
			if (lastBaseline == null || baselineBuildDate == null || baselineBuildDate.compareTo(buildResults.getDate()) >= 0) {
				lastBaseline = buildResults;
			}
			if (baselineBuildName != null && buildResults.getName().equals(baselineBuildName)) {
				this.baseline = buildResults;
				this.baselined = true;
			}
		} else if (currentBuildName == null || currentBuildDate == null || (this.current == null && buildResults.getDate().compareTo(currentBuildDate) >= 0)) {
			this.current = buildResults;
			this.valid = true;
		}
	}
	if (this.baseline == null) {
		this.baseline = (lastBaseline == null) ? (BuildResults) this.children.get(0) : lastBaseline;
	}
	if (this.current == null) {
		int idx = size() - 1;
		BuildResults previous = (BuildResults) this.children.get(idx--);
		while (idx >= 0 && previous.isBaseline()) {
			previous = (BuildResults) this.children.get(idx--);
		}
		this.current = previous;
	}

	// Set delta between current vs. baseline and the corresponding error
	int dim_id = DB_Results.getDefaultDimension().getId();
	double baselineValue = this.baseline.getValue();
	double currentValue = this.current.getValue();
	this.delta = (currentValue - baselineValue) / baselineValue;
	if (Double.isNaN(this.delta)) {
		this.error = Double.NaN;
	} else {
		long baselineCount = this.baseline.getCount(dim_id);
		long currentCount = this.current.getCount(dim_id);
		if (baselineCount == 1 || currentCount == 1) {
			this.error = Double.NaN;
		} else {
			double baselineError = this.baseline.getError(dim_id);
			double currentError = this.current.getError(dim_id);
			this.error = Double.isNaN(baselineError)
					? currentError / baselineValue
					: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
		}
	}

	// Set the failure on the current build if necessary
	int failure_threshold = getPerformance().failure_threshold;
	if (this.delta >= (failure_threshold/100.0)) {
		StringBuffer buffer = new StringBuffer("Performance criteria not met when compared to '"); //$NON-NLS-1$
		buffer.append(this.baseline.getName());
		buffer.append("': "); //$NON-NLS-1$
		buffer.append(DB_Results.getDefaultDimension().getName());
		buffer.append("= "); //$NON-NLS-1$
		buffer.append(Util.timeString((long)this.current.getValue()));
		buffer.append(" is not within [0%, "); //$NON-NLS-1$
		buffer.append(100+failure_threshold);
		buffer.append("'%] of "); //$NON-NLS-1$
		buffer.append(Util.timeString((long)this.baseline.getValue()));
		this.current.setFailure(buffer.toString());
	}
}

/**
 * Returns whether the configuration has results for the performance
 * baseline build or not.
 *
 * @return <code>true</code> if the configuration has results
 * 	for the performance baseline build, <code>false</code> otherwise.
 */
public boolean isBaselined() {
	if (this.baseline == null || this.current == null) {
	    initialize();
    }
	return this.baselined;
}

boolean isBuildConcerned(BuildResults buildResults) {
	String buildDate = buildResults.getDate();
	String currentBuildDate = getCurrentBuildResults() == null ? null : getCurrentBuildResults().getDate();
	String baselineBuildDate = getBaselineBuildResults() == null ? null : getBaselineBuildResults().getDate();
	return ((currentBuildDate == null || buildDate.compareTo(currentBuildDate) <= 0) &&
		(baselineBuildDate == null || buildDate.compareTo(baselineBuildDate) <= 0));
}
/**
 * Returns whether the configuration has results for the performance
 * current build or not.
 *
 * @return <code>true</code> if the configuration has results
 * 	for the performance current build, <code>false</code> otherwise.
 */
public boolean isValid() {
	if (this.baseline == null || this.current == null) {
	    initialize();
    }
	return this.valid;
}

/**
 * Returns the 'n' last nightly build names.
 *
 * @param n Number of last nightly builds to return
 * @return Last n nightly build names preceding current.
 */
public List lastNightlyBuildNames(int n) {
	List labels = new ArrayList();
	for (int i=size()-2; i>=0; i--) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		if (isBuildConcerned(buildResults)) {
			String buildName = buildResults.getName();
			if (buildName.startsWith("N")) { //$NON-NLS-1$
				labels.add(buildName);
				if (labels.size() >= n) {
	                break;
                }
			}
		}
	}
	return labels;
}

/*
 * Read all configuration builds results data from the given stream.
 */
void readData(DataInputStream stream) throws IOException {
	int size = stream.readInt();
	for (int i=0; i<size; i++) {
		BuildResults buildResults = new BuildResults(this);
		buildResults.readData(stream);
		String lastBuildName = getPerformance().lastBuildName;
		if (lastBuildName == null || buildResults.getDate().compareTo(Util.getBuildDate(lastBuildName)) <= 0) {
			addChild(buildResults, true);
		}
	}
}

private void reset() {
	this.current = null;
	this.baseline = null;
	this.baselined = false;
	this.valid = false;
	this.delta = 0;
	this.error = -1;
}

/*
 * Set the configuration value from database information
 */
void setInfos(int build_id, int summaryKind, String comment) {
	BuildResults buildResults = (BuildResults) getResults(build_id);
	if (buildResults == null) {
		buildResults = new BuildResults(this, build_id);
		addChild(buildResults, true);
	}
	buildResults.summaryKind = summaryKind;
	buildResults.comment = comment;
}

/*
 * Set the configuration value from database information
 */
void setValue(int build_id, int dim_id, int step, long value) {
//	reset();
	BuildResults buildResults = (BuildResults) getResults(build_id);
	if (buildResults == null) {
		buildResults = new BuildResults(this, build_id);
		addChild(buildResults, true);
	}
	buildResults.setValue(dim_id, step, value);
}

/*
 * Write all configuration builds results data into the given stream.
 */
void write(DataOutputStream stream) throws IOException {
	int size = size();
	stream.writeInt(this.id);
	stream.writeInt(size);
	for (int i=0; i<size; i++) {
		BuildResults buildResults = (BuildResults) this.children.get(i);
		buildResults.write(stream);
	}
}

}