new/js/app.js

'use strict';

/**
 * TODO (stephana@): This is still work in progress.
 * It does not offer the same functionality as the current version, but
 * will serve as the starting point for a new backend.
 * It works with the current backend, but does not support rebaselining.
 */

/*
 * Wrap everything into an IIFE to not polute the global namespace.
 */
(function () {

  // Declare app level module which contains everything of the current app.
  // ui.bootstrap refers to directives defined in the AngularJS Bootstrap
  // UI package (http://angular-ui.github.io/bootstrap/).
  var app = angular.module('rbtApp', ['ngRoute', 'ui.bootstrap']);

  // Configure the different within app views.
  app.config(['$routeProvider', function($routeProvider) {
    $routeProvider.when('/', {templateUrl: 'partials/index-view.html',
                              controller: 'IndexCtrl'});
    $routeProvider.when('/view', {templateUrl: 'partials/rebaseline-view.html',
                                  controller: 'RebaselineCrtrl'});
    $routeProvider.otherwise({redirectTo: '/'});
  }]);


  // TODO (stephana): Some of these constants are 'gm' specific. In the
  // next iteration we need to remove those as we move the more generic
  // 'dm' testing tool.
  //
  // Shared constants used here and in the markup. These are exported when
  // when used by a controller.
  var c = {
    // Define different view states as we load the data.
    ST_LOADING: 1,
    ST_STILL_LOADING: 2,
    ST_READY: 3,

    // These column types are used by the Column class.
    COL_T_FILTER: 'filter',
    COL_T_IMAGE: 'image',
    COL_T_REGULAR: 'regular',

    // Request parameters used to select between subsets of results.
    RESULTS_ALL: 'all',
    RESULTS_FAILURES: 'failures',

    // Filter types are used by the Column class.
    FILTER_FREE_FORM: 'free_form',
    FILTER_CHECK_BOX: 'checkbox',

    // Columns either provided by the backend response or added in code.
    // TODO (stephana): This should go away once we switch to 'dm'.
    COL_BUGS: 'bugs',
    COL_IGNORE_FAILURE: 'ignore-failure',
    COL_REVIEWED_BY_HUMANS: 'reviewed-by-human',

    // Defines the order in which image columns appear.
    // TODO (stephana@): needs to be driven by backend data.
    IMG_COL_ORDER: [
       {
        key: 'imageA',
        urlField: ['imageAUrl']
      },
      {
        key: 'imageB',
        urlField: ['imageBUrl']
      },
      {
        key: 'whiteDiffs',
        urlField: ['differenceData', 'whiteDiffUrl'],
        percentField: ['differenceData', 'percentDifferingPixels'],
        valueField: ['differenceData', 'numDifferingPixels']
      },
      {
        key: 'diffs',
        urlField: ['differenceData', 'diffUrl'],
        percentField: ['differenceData', 'perceptualDifference'],
        valueField: ['differenceData', 'maxDiffPerChannel']
      }
    ],

    // Choice of availabe image size selection.
    IMAGE_SIZES: [
      100,
      200,
      400
    ],

    // Choice of available number of records selection.
    MAX_RECORDS: [
      '100',
      '200',
      '300'
    ]
  };  // end constants

  /*
   * Index Controller
   */
  // TODO (stephana): Remove $timeout since it only simulates loading delay.
  app.controller('IndexCtrl', ['$scope', '$timeout', 'dataService',
  function($scope, $timeout, dataService) {
    // init the scope
    $scope.c = c;
    $scope.state = c.ST_LOADING;
    $scope.qStr = dataService.getQueryString;

    // TODO (stephana): Remove and replace with index data generated by the
    // backend to reflect the current "known" image sets to compare.
    $scope.allSKPs = [
    {
      params: {
        setBSection: 'actual-results',
        setASection: 'expected-results',
        setBDir: 'gs://chromium-skia-skp-summaries/' +
                 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',
        setADir: 'repo:expectations/skp/' +
                 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug'
      },
      title: 'expected vs actuals on ' +
             'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug'
    },
    {
      params: {
        setBSection: 'actual-results',
        setASection: 'expected-results',
        setBDir: 'gs://chromium-skia-skp-summaries/' +
                 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release',
        setADir: 'repo:expectations/skp/'+
                 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'
      },
      title: 'expected vs actuals on Test-Ubuntu12-ShuttleA-GTX660-x86-Release'
    },
    {
      params: {
        setBSection: 'actual-results',
        setASection: 'actual-results',
        setBDir: 'gs://chromium-skia-skp-summaries/' +
                 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release',
        setADir: 'gs://chromium-skia-skp-summaries/' +
                 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug'
      },
      title: 'Actuals on Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug ' +
             'vs Test-Ubuntu12-ShuttleA-GTX660-x86-Release'
    }
    ];

    // TODO (stephana): Remove this once we load index data from the server.
    $timeout(function () {
      $scope.state = c.ST_READY;
    });
  }]);

  /*
   *  RebaselineCtrl
   *  Controls the main comparison view.
   *
   *  @param {service} dataService Service that encapsulates functions to
   *                               retrieve data from the backend.
   *
   */
  app.controller('RebaselineCrtrl', ['$scope', '$timeout', 'dataService',
  function($scope, $timeout, dataService) {
    // determine which to request
    // TODO (stephana): This should be extracted from the query parameters.
    var target = c.TARGET_GM;

    // process the rquest arguments
    // TODO (stephana): This should be determined from the query parameters.
    var loadFn = dataService.loadAll;

    // controller state variables
    var allData = null;
    var filterFuncs = null;
    var currentData = null;
    var selectedData = null;

    // Index of the column that should provide the sort key
    var sortByIdx = 0;

    // Sort in asending (true) or descending (false) order
    var sortOrderAsc = true;

    // Array of functions for each column used for comparison during sort.
    var compareFunctions = null;

    // Variables to track load and render times
    var startTime;
    var loadStartTime;


    /** Load the data from the backend **/
    loadStartTime = Date.now();
    function loadData() {
      loadFn().then(
        function (serverData) {
          $scope.header = serverData.header;
          $scope.loadTime = (Date.now() - loadStartTime)/1000;

          // keep polling if the data are not ready yet
          if ($scope.header.resultsStillLoading) {
            $scope.state = c.ST_STILL_LOADING;
            $timeout(loadData, 5000);
            return;
          }

          // get the filter colunms and an array to hold filter data by user
          var fcol = getFilterColumns(serverData);
          $scope.filterCols = fcol[0];
          $scope.filterVals = fcol[1];

          // Add extra columns and retrieve the image columns
          var otherCols = [ Column.regular(c.COL_BUGS) ];
          var imageCols = getImageColumns(serverData);

          // Concat to get all columns
          // NOTE: The order is important since filters are rendered first,
          // followed by regular columns and images
          $scope.allCols = $scope.filterCols.concat(otherCols, imageCols);

          // Pre-process the data and get the filter functions.
          var dataFilters = getDataAndFilters(serverData, $scope.filterCols,
                                              otherCols, imageCols);
          allData = dataFilters[0];
          filterFuncs = dataFilters[1];

          // Get regular columns (== not image columns)
          var regularCols = $scope.filterCols.concat(otherCols);

          // Get the compare functions for regular and image columns. These
          // are then used to sort by the respective columns.
          compareFunctions = DataRow.getCompareFunctions(regularCols,
                                                         imageCols);

          // Filter and sort the results to get them ready for rendering
          updateResults();

          // Data are ready for display
          $scope.state = c.ST_READY;
        },
        function (httpErrResponse) {
          console.log(httpErrResponse);
        });
    };

    /*
     * updateResults
     * Central render function. Everytime settings/filters/etc. changed
     * this function is called to filter, sort and splice the data.
     *
     * NOTE (stephana): There is room for improvement here: before filtering
     * and sorting we could check if this is necessary. But this has not been
     * a bottleneck so far.
     */
    function updateResults () {
      // run digest before we update the results. This allows
      // updateResults to be called from functions trigger by ngChange
      $scope.updating = true;
      startTime = Date.now();

      // delay by one render cycle so it can be called via ng-change
      $timeout(function() {
        // filter data
        selectedData = filterData(allData, filterFuncs, $scope.filterVals);

        // sort the selected data.
        sortData(selectedData, compareFunctions, sortByIdx, sortOrderAsc);

        // only conside the elements that we really need
        var nRecords = $scope.settings.nRecords;
        currentData = selectedData.slice(0, parseInt(nRecords));

        DataRow.setRowspanValues(currentData, $scope.mergeIdenticalRows);

        // update the scope with relevant data for rendering.
        $scope.data = currentData;
        $scope.totalRecords = allData.length;
        $scope.showingRecords = currentData.length;
        $scope.selectedRecords = selectedData.length;
        $scope.updating = false;

        // measure the filter time and total render time (via timeout).
        $scope.filterTime = Date.now() - startTime;
        $timeout(function() {
          $scope.renderTime = Date.now() - startTime;
        });
      });
    };

    /**
     * Generate the style value to set the width of images.
     *
     * @param {Column} col Column that we are trying to render.
     * @param {int} paddingPx Number of padding pixels.
     * @param {string} defaultVal Default value if not an image column.
     *
     * @return {string} Value to be used in ng-style element to set the width
     *                  of a image column.
     **/
    $scope.getImageWidthStyle = function (col, paddingPx, defaultVal) {
      var result = (col.ctype === c.COL_T_IMAGE) ?
                   ($scope.imageSize + paddingPx + 'px') : defaultVal;
      return result;
    };

    /**
     * Sets the column by which to sort the data. If called for the
     * currently sorted column it will cause the sort to toggle between
     * ascending and descending.
     *
     * @param {int} colIdx Index of the column to use for sorting.
     **/
    $scope.sortBy = function (colIdx) {
      if (sortByIdx === colIdx) {
        sortOrderAsc = !sortOrderAsc;
      } else {
        sortByIdx = colIdx;
        sortOrderAsc = true;
      }
      updateResults();
    };

    /**
     * Helper function to generate a CSS class indicating whether this column
     * is the sort key. If it is a class name with the sort direction (Asc/Desc) is
     * return otherwise the default value is returned. In markup we use this
     * to display (or not display) an arrow next to the column name.
     *
     * @param {string} prefix Prefix of the classname to be generated.
     * @param {int} idx Index of the target column.
     * @param {string} defaultVal Value to return if current column is not used
     *                            for sorting.
     *
     * @return {string} CSS class name that a combination of the prefix and
     *                  direction indicator ('Asc' or 'Desc') if the column is
     *                  used for sorting. Otherwise the defaultVal is returned.
     **/
    $scope.getSortedClass = function (prefix, idx, defaultVal) {
      if (idx === sortByIdx) {
        return prefix + ((sortOrderAsc) ? 'Asc' : 'Desc');
      }

      return defaultVal;
    };

    /**
     * Checkbox to merge identical records has change. Force an update.
     **/
    $scope.mergeRowsChanged = function () {
      updateResults();
    }

    /**
     * Max number of records to display has changed. Force an update.
     **/
    $scope.maxRecordsChanged = function () {
      updateResults();
    };

    /**
     * Filter settings changed. Force an update.
     **/
    $scope.filtersChanged = function () {
      updateResults();
    };

    /**
     * Sets all possible values of the specified values to the given value.
     * That means all checkboxes are eiter selected or unselected.
     * Then force an update.
     *
     * @param {int} idx Index of the target filter column.
     * @param {boolean} val Value to set the filter values to.
     *
     **/
    $scope.setFilterAll = function (idx, val) {
      for(var i=0, len=$scope.filterVals[idx].length; i<len; i++) {
        $scope.filterVals[idx][i] = val;
      }
      updateResults();
    };

    /**
     * Toggle the values of a filter. This toggles all values in a
     * filter.
     *
     * @param {int} idx Index of the target filter column.
     **/
    $scope.setFilterToggle = function (idx) {
      for(var i=0, len=$scope.filterVals[idx].length; i<len; i++) {
        $scope.filterVals[idx][i] = !$scope.filterVals[idx][i];
      }
      updateResults();
    };

    // ****************************************
    // Initialize the scope.
    // ****************************************

    // Inject the constants into the scope and set the initial state.
    $scope.c = c;
    $scope.state = c.ST_LOADING;

    // Initial settings
    $scope.settings = {
      showThumbnails: true,
      imageSize: c.IMAGE_SIZES[0],
      nRecords: c.MAX_RECORDS[0],
      mergeIdenticalRows: true
    };

    // Initial values for filters set in loadData()
    $scope.filterVals = [];

    // Information about records - set in loadData()
    $scope.totalRecords = 0;
    $scope.showingRecords = 0;
    $scope.updating = false;

    // Trigger the data loading.
    loadData();

  }]);

  // data structs to interface with markup and backend
  /**
   * Models a column. It aggregates attributes of all
   * columns types. Some might be empty. See convenience
   * factory methods below for different column types.
   *
   * @param {string} key Uniquely identifies this columns
   * @param {string} ctype Type of columns. Use COL_* constants.
   * @param {string} ctitle Human readable title of the column.
   * @param {string} ftype Filter type. Use FILTER_* constants.
   * @param {FilterOpt[]} foptions Filter options. For 'checkbox' filters this
                                   is used to render all the checkboxes.
                                   For freeform filters this is a list of all
                                   available values.
   * @param {string} baseUrl Baseurl for image columns. All URLs are relative
                             to this.
   *
   * @return {Column} Instance of the Column class.
   **/
  function Column(key, ctype, ctitle, ftype, foptions, baseUrl) {
    this.key = key;
    this.ctype = ctype;
    this.ctitle = ctitle;
    this.ftype = ftype;
    this.foptions = foptions;
    this.baseUrl = baseUrl;
    this.foptionsArr = [];

    // get the array of filter options for lookup in indexOfOptVal
    if (this.foptions) {
      for(var i=0, len=foptions.length; i<len; i++) {
        this.foptionsArr.push(this.foptions[i].value);
      }
    }
  }

  /**
   * Find the index of an value in a column with a fixed set
   * of options.
   *
   * @param {string} optVal Value of the column.
   *
   * @return {int} Index of optVal in this column.
   **/
  Column.prototype.indexOfOptVal = function (optVal) {
    return this.foptionsArr.indexOf(optVal);
  };

  /**
   * Set filter options for this column.
   *
   * @param {FilterOpt[]} foptions Possible values for this column.
   **/
  Column.prototype.setFilterOptions = function (foptions) {
    this.foptions = foptions;
  };

  /**
   * Factory function to create a filter column. Same args as Column()
   **/
  Column.filter = function(key, ctitle, ftype, foptions) {
    return new Column(key, c.COL_T_FILTER, ctitle || key, ftype, foptions);
  }

  /**
   * Factory function to create an image column. Same args as Column()
   **/
  Column.image = function (key, ctitle, baseUrl) {
    return new Column(key, c.COL_T_IMAGE, ctitle || key, null, null, baseUrl);
  };

  /**
   * Factory function to create a regular column. Same args as Column()
   **/
  Column.regular = function (key, ctitle) {
    return new Column(key, c.COL_T_REGULAR, ctitle || key);
  };

  /**
   * Helper class to wrap a single option in a filter.
   *
   * @param {string} value Option value.
   * @param {int} count Number of instances of this option in the dataset.
   *
   * @return {} Instance of FiltertOpt
   **/
  function FilterOpt(value, count) {
    this.value = value;
    this.count = count;
  }

  /**
   * Container for a single row in the dataset.
   *
   * @param {int} rowspan Number of rows (including this and following rows)
                          that have identical values.
   * @param {string[]} dataCols Values of the respective columns (combination
                                of filter and regular columns)
   * @param {ImgVal[]} imageCols Image meta data for the image columns.
   *
   * @return {DataRow} Instance of DataRow.
   **/
  function DataRow(rowspan, dataCols, imageCols) {
    this.rowspan = rowspan;
    this.dataCols = dataCols;
    this.imageCols = imageCols;
  }

  /**
   * Gets the comparator functions for the columns in this dataset.
   * The comparators are then used to sort the dataset by the respective
   * column.
   *
   * @param {Column[]} dataCols Data columns (= non-image columns)
   * @param {Column[]} imgCols Image columns.
   *
   * @return {Function[]} Array of functions that can be used to sort by the
   *                      respective column.
   **/
  DataRow.getCompareFunctions = function (dataCols, imgCols) {
    var result = [];
    for(var i=0, len=dataCols.length; i<len; i++) {
      result.push(( function (col, idx) {
        return function (a, b) {
          return (a.dataCols[idx] < b.dataCols[idx]) ? -1 :
                 ((a.dataCols[idx] === b.dataCols[idx]) ? 0 : 1);
        };
      }(dataCols[i], i) ));
    }

    for(var i=0, len=imgCols.length; i<len; i++) {
      result.push((function (col, idx) {
        return function (a,b) {
          var aVal = a.imageCols[idx].percent;
          var bVal = b.imageCols[idx].percent;

          return (aVal < bVal) ? -1 : ((aVal === bVal) ? 0 : 1);
        };
      }(imgCols[i], i) ));
    }

    return result;
  };

  /**
  * Set the rowspan values of a given array of DataRow instances.
  *
  * @param {DataRow[]} data Dataset in desired order (after sorting).
  * @param {mergeRows} mergeRows Indicate whether to sort
   **/
  DataRow.setRowspanValues = function (data, mergeRows) {
    var curIdx, rowspan, cur;
    if (mergeRows) {
      for(var i=0, len=data.length; i<len;) {
        curIdx = i;
        cur = data[i];
        rowspan = 1;
        for(i++; ((i<len) && (data[i].dataCols === cur.dataCols)); i++) {
          rowspan++;
          data[i].rowspan=0;
        }
        data[curIdx].rowspan = rowspan;
      }
    } else {
      for(var i=0, len=data.length; i<len; i++) {
        data[i].rowspan = 1;
      }
    }
  };

  /**
   * Wrapper class for image related data.
   *
   * @param {string} url Relative Url of the image or null if not available.
   * @param {float} percent Percent of pixels that are differing.
   * @param {int} value Absolute number of pixes differing.
   *
   * @return {ImgVal} Instance of ImgVal.
   **/
  function ImgVal(url, percent, value) {
    this.url = url;
    this.percent = percent;
    this.value = value;
  }

  /**
   * Extracts the filter columns from the JSON response of the server.
   *
   * @param {object} data Server response.
   *
   * @return {Column[]} List of filter columns as described in 'header' field.
   **/
  function getFilterColumns(data) {
    var result = [];
    var vals = [];
    var colOrder = data.extraColumnOrder;
    var colHeaders = data.extraColumnHeaders;
    var fopts, optVals, val;

    for(var i=0, len=colOrder.length; i<len; i++) {
      if (colHeaders[colOrder[i]].isFilterable) {
        if (colHeaders[colOrder[i]].useFreeformFilter) {
          result.push(Column.filter(colOrder[i],
                                    colHeaders[colOrder[i]].headerText,
                                    c.FILTER_FREE_FORM));
          vals.push('');
        }
        else {
          fopts = [];
          optVals = [];

          // extract the different options for this column
          for(var j=0, jlen=colHeaders[colOrder[i]].valuesAndCounts.length;
              j<jlen; j++) {
                val = colHeaders[colOrder[i]].valuesAndCounts[j];
                fopts.push(new FilterOpt(val[0], val[1]));
                optVals.push(false);
          }

          // ad the column and values
          result.push(Column.filter(colOrder[i],
                                    colHeaders[colOrder[i]].headerText,
                                    c.FILTER_CHECK_BOX,
                                    fopts));
          vals.push(optVals);
        }
      }
    }

    return [result, vals];
  }

  /**
   * Extracts the image columns from the JSON response of the server.
   *
   * @param {object} data Server response.
   *
   * @return {Column[]} List of images columns as described in 'header' field.
   **/
  function getImageColumns(data) {
    var CO = c.IMG_COL_ORDER;
    var imgSet;
    var result = [];
    for(var i=0, len=CO.length; i<len; i++) {
      imgSet = data.imageSets[CO[i].key];
      result.push(Column.image(CO[i].key,
                               imgSet.description,
                               ensureTrailingSlash(imgSet.baseUrl)));
    }
    return result;
  }

  /**
   * Make sure Url has a trailing '/'.
   *
   * @param {string} url Base url.
   * @return {string} Same url with a trailing '/' or same as input if it
                      already contained '/'.
   **/
  function ensureTrailingSlash(url) {
    var result = url.trim();

    // TODO: remove !!!
    result = fixUrl(url);
    if (result[result.length-1] !== '/') {
      result += '/';
    }
    return result;
  }

  // TODO: remove. The backend should provide absoute URLs
  function fixUrl(url) {
    url = url.trim();
    if ('http' === url.substr(0, 4)) {
      return url;
    }

    var idx = url.indexOf('static');
    if (idx != -1) {
      return '/' + url.substr(idx);
    }

    return url;
  };

  /**
   * Processes that data and returns filter functions.
   *
   * @param {object} Server response.
   * @param {Column[]} filterCols Filter columns.
   * @param {Column[]} otherCols Columns that are neither filters nor images.
   * @param {Column[]} imageCols Image columns.
   *
   * @return {[]} Returns a pair [dataRows, filterFunctions] where:
   *       - dataRows is an array of DataRow instances.
   *       - filterFunctions is an array of functions that can be used to
   *         filter the column at the corresponding index.
   *
   **/
  function getDataAndFilters(data, filterCols, otherCols, imageCols) {
    var el;
    var result = [];
    var lookupIndices = [];
    var indexerFuncs = [];
    var temp;

    // initialize the lookupIndices
    var filterFuncs = initIndices(filterCols, lookupIndices, indexerFuncs);

    // iterate over the data and get the rows
    for(var i=0, len=data.imagePairs.length; i<len; i++) {
      el = data.imagePairs[i];
      temp = new DataRow(1, getColValues(el, filterCols, otherCols),
                                 getImageValues(el, imageCols));
      result.push(temp);

      // index the row
      for(var j=0, jlen=filterCols.length; j < jlen; j++) {
        indexerFuncs[j](lookupIndices[j], filterCols[j], temp.dataCols[j], i);
      }
    }

    setFreeFormFilterOptions(filterCols, lookupIndices);
    return [result, filterFuncs];
  }

  /**
   * Initiazile the lookup indices and indexer functions for the filter
   * columns.
   *
   * @param {Column} filterCols Filter columns
   * @param {[]} lookupIndices Will be filled with datastructures for
                               fast lookup (output parameter)
   * @param {[]} lookupIndices Will be filled with functions to index data
                               of the column with the corresponding column.
   *
   * @return {[]} Returns an array of filter functions that can be used to
                  filter the respective column.
   **/
  function initIndices(filterCols, lookupIndices, indexerFuncs) {
    var filterFuncs = [];
    var temp;

    for(var i=0, len=filterCols.length; i<len; i++) {
      if (filterCols[i].ftype === c.FILTER_FREE_FORM) {
        lookupIndices.push({});
        indexerFuncs.push(indexFreeFormValue);
        filterFuncs.push(
          getFreeFormFilterFunc(lookupIndices[lookupIndices.length-1]));
      }
      else if (filterCols[i].ftype === c.FILTER_CHECK_BOX) {
        temp = [];
        for(var j=0, jlen=filterCols[i].foptions.length; j<jlen; j++) {
          temp.push([]);
        }
        lookupIndices.push(temp);
        indexerFuncs.push(indexDiscreteValue);
        filterFuncs.push(
          getDiscreteFilterFunc(lookupIndices[lookupIndices.length-1]));
      }
    }

    return filterFuncs;
  }

  /**
   * Helper function that extracts the values of free form columns from
   * the lookupIndex and injects them into the Column object as FilterOpt
   * objects.
   **/
  function setFreeFormFilterOptions(filterCols, lookupIndices) {
    var temp, k;
    for(var i=0, len=filterCols.length; i<len; i++) {
      if (filterCols[i].ftype === c.FILTER_FREE_FORM) {
        temp = []
        for(k in lookupIndices[i]) {
          if (lookupIndices[i].hasOwnProperty(k)) {
            temp.push(new FilterOpt(k, lookupIndices[i][k].length));
          }
        }
        filterCols[i].setFilterOptions(temp);
      }
    }
  }

  /**
   * Index a discrete column (column with fixed number of values).
   *
   **/
  function indexDiscreteValue(lookupIndex, col, dataVal, dataRowIndex) {
    var i = col.indexOfOptVal(dataVal);
    lookupIndex[i].push(dataRowIndex);
  }

  /**
   * Index a column with free form text (= not fixed upfront)
   *
   **/
  function indexFreeFormValue(lookupIndex, col, dataVal, dataRowIndex) {
    if (!lookupIndex[dataVal]) {
      lookupIndex[dataVal] = [];
    }
    lookupIndex[dataVal].push(dataRowIndex);
  }


  /**
   * Get the function to filter a column with the given lookup index
   * for discrete (fixed upfront) values.
   *
   **/
  function getDiscreteFilterFunc(lookupIndex) {
    return function(filterVal) {
      var result = [];
      for(var i=0, len=lookupIndex.length; i < len; i++) {
        if (filterVal[i]) {
          // append the indices to the current array
          result.push.apply(result, lookupIndex[i]);
        }
      }
      return { nofilter: false, records: result };
    };
  }

  /**
   * Get the function to filter a column with the given lookup index
   * for free form values.
   *
   **/
  function getFreeFormFilterFunc(lookupIndex) {
    return function(filterVal) {
      filterVal = filterVal.trim();
      if (filterVal === '') {
        return { nofilter: true };
      }
      return {
        nofilter: false,
        records: lookupIndex[filterVal] || []
      };
    };
  }

  /**
   * Filters the data based on the given filterColumns and
   * corresponding filter values.
   *
   * @return {[]} Subset of the input dataset based on the
   *              filter values.
   **/
  function filterData(data, filterFuncs, filterVals) {
    var recordSets = [];
    var filterResult;

    // run through all the filters
    for(var i=0, len=filterFuncs.length; i<len; i++) {
      filterResult = filterFuncs[i](filterVals[i]);
      if (!filterResult.nofilter) {
        recordSets.push(filterResult.records);
      }
    }

    // If there are no restrictions then return the whole dataset.
    if (recordSets.length === 0) {
      return data;
    }

    // intersect the records returned by filters.
    var targets = intersectArrs(recordSets);
    var result = [];
    for(var i=0, len=targets.length; i<len; i++) {
      result.push(data[targets[i]]);
    }

    return result;
  }

  /**
   * Creates an object where the keys are the elements of the input array
   * and the values are true. To be used for set operations with integer.
   **/
  function arrToObj(arr) {
    var o = {};
    var i,len;
    for(i=0, len=arr.length; i<len; i++) {
      o[arr[i]] = true;
    }
    return o;
  }

  /**
   * Converts the keys of an object to an array after converting
   * each key to integer. To be used for set operations with integers.
   **/
  function objToArr(obj) {
    var result = [];
    for(var k in obj) {
      if (obj.hasOwnProperty(k)) {
        result.push(parseInt(k));
      }
    }
    return result;
  }

  /**
   * Find the intersection of a set of arrays.
   **/
  function intersectArrs(sets) {
    var temp, obj;

    if (sets.length === 1) {
      return sets[0];
    }

    // sort by size and load the smallest into the object
    sets.sort(function(a,b) { return a.length - b.length; });
    obj = arrToObj(sets[0]);

    // shrink the hash as we fail to find elements in the other sets
    for(var i=1, len=sets.length; i<len; i++) {
      temp = arrToObj(sets[i]);
      for(var k in obj) {
        if (obj.hasOwnProperty(k) && !temp[k]) {
          delete obj[k];
        }
      }
    }

    return objToArr(obj);
  }

  /**
   * Extract the column values from an ImagePair (contained in the server
   * response) into filter and data columns.
   *
   * @return {[]} Array of data contained in one data row.
   **/
  function getColValues(imagePair, filterCols, otherCols) {
    var result = [];
    for(var i=0, len=filterCols.length; i<len; i++) {
      result.push(imagePair.extraColumns[filterCols[i].key]);
    }

    for(var i=0, len=otherCols.length; i<len; i++) {
      result.push(get_robust(imagePair, ['expectations', otherCols[i].key]));
    }

    return result;
  }

  /**
   * Extract the image meta data from an Image pair returned by the server.
   **/
  function getImageValues(imagePair, imageCols) {
    var result=[];
    var url, value, percent, diff;
    var CO = c.IMG_COL_ORDER;

    for(var i=0, len=imageCols.length; i<len; i++) {
      percent = get_robust(imagePair, CO[i].percentField);
      value = get_robust(imagePair, CO[i].valueField);
      url = get_robust(imagePair, CO[i].urlField);
      if (url) {
        url = imageCols[i].baseUrl + url;
      }
      result.push(new ImgVal(url, percent, value));
    }

    return result;
  }

  /**
   * Given an object find sub objects for the given index without
   * throwing an error if any of the sub objects do not exist.
   **/
  function get_robust(obj, idx) {
    if (!idx) {
      return;
    }

    for(var i=0, len=idx.length; i<len; i++) {
      if ((typeof obj === 'undefined') || (!idx[i])) {
        return;  // returns 'undefined'
      }

      obj = obj[idx[i]];
    }

    return obj;
  }

  /**
   * Set all elements in the array to the given value.
   **/
  function setArrVals(arr, newVal) {
    for(var i=0, len=arr.length; i<len; i++) {
      arr[i] = newVal;
    }
  }

  /**
   * Toggle the elements of a boolean array.
   *
   **/
  function toggleArrVals(arr) {
    for(var i=0, len=arr.length; i<len; i++) {
      arr[i] = !arr[i];
    }
  }

  /**
   * Sort the array of DataRow instances with the given compare functions
   * and the column at the given index either in ascending or descending order.
   **/
  function sortData (allData, compareFunctions, sortByIdx, sortOrderAsc) {
    var cmpFn = compareFunctions[sortByIdx];
    var useCmp = cmpFn;
    if (!sortOrderAsc) {
      useCmp = function ( _ ) {
        return -cmpFn.apply(this, arguments);
      };
    }
    allData.sort(useCmp);
  }


  // *****************************  Services *********************************

  /**
   *  Encapsulates all interactions with the backend by handling
   *  Urls and HTTP requests. Also exposes some utility functions
   *  related to processing Urls.
   */
  app.factory('dataService', [ '$http', function ($http) {
    /** Backend related constants  **/
    var c = {
      /** Url to retrieve failures */
      FAILURES: '/results/failures',

      /** Url to retrieve all GM results */
      ALL:      '/results/all'
    };

    /**
     * Convenience function to retrieve all results.
     *
     * @return {Promise} Will resolve to either the data (success) or to
     *                   the HTTP response (error).
     **/
    function loadAll() {
      return httpGetData(c.ALL);
    }

    /**
     * Make a HTTP get request with the given query parameters.
     *
     * @param {}
     * @param {}
     *
     * @return {}
     **/
    function httpGetData(url, queryParams) {
      var reqConfig = {
        method: 'GET',
        url: url,
        params: queryParams
      };

      return $http(reqConfig).then(
        function(successResp) {
          return successResp.data;
        });
    }

    /**
     * Takes an arbitrary number of objects and generates a Url encoded
     * query string.
     *
     **/
    function getQueryString( _params_ ) {
      var result = [];
      for(var i=0, len=arguments.length; i < len; i++) {
        if (arguments[i]) {
          for(var k in arguments[i]) {
            if (arguments[i].hasOwnProperty(k)) {
              result.push(encodeURIComponent(k) + '=' +
                          encodeURIComponent(arguments[i][k]));
            }
          }
        }
      }
      return result.join("&");
    }

    // Interface of the service:
    return {
      getQueryString: getQueryString,
      loadAll: loadAll
    };

  }]);

})();