1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """An example main file running the algorithms. 5 6 Part of the Chrome build flags optimization. 7 8 An example use of the framework. It parses the input json configuration file. 9 Then it initiates the variables of the generation. Finally, it sets up the 10 processes for different modules and runs the experiment. 11 """ 12 13 __author__ = 'yuhenglong (at] google.com (Yuheng Long)' 14 15 import json 16 import multiprocessing 17 from optparse import OptionParser 18 import sys 19 20 import flags 21 from genetic_algorithm import GAGeneration 22 from pipeline_process import PipelineProcess 23 import pipeline_worker 24 from steering import Steering 25 from task import BUILD_STAGE 26 from task import Task 27 from task import TEST_STAGE 28 import testing_batch 29 30 parser = OptionParser() 31 32 parser.add_option('-f', 33 '--file', 34 dest='filename', 35 help='configuration file FILE input', 36 metavar='FILE') 37 38 # The meta data for the genetic algorithm. 39 BUILD_CMD = 'BUILD_CMD' 40 TEST_CMD = 'TEST_CMD' 41 OUTPUT = 'OUTPUT' 42 DEFAULT_OUTPUT = 'output' 43 CONF = 'CONF' 44 DEFAULT_CONF = 'conf' 45 NUM_BUILDER = 'NUM_BUILDER' 46 DEFAULT_NUM_BUILDER = 1 47 NUM_TESTER = 'NUM_TESTER' 48 DEFAULT_NUM_TESTER = 1 49 STOP_THRESHOLD = 'STOP_THRESHOLD' 50 DEFAULT_STOP_THRESHOLD = 1 51 NUM_CHROMOSOMES = 'NUM_CHROMOSOMES' 52 DEFAULT_NUM_CHROMOSOMES = 20 53 NUM_TRIALS = 'NUM_TRIALS' 54 DEFAULT_NUM_TRIALS = 20 55 MUTATION_RATE = 'MUTATION_RATE' 56 DEFAULT_MUTATION_RATE = 0.01 57 58 59 def _ProcessGA(meta_data): 60 """Set up the meta data for the genetic algorithm. 61 62 Args: 63 meta_data: the meta data for the genetic algorithm. 64 """ 65 assert BUILD_CMD in meta_data 66 build_cmd = meta_data[BUILD_CMD] 67 68 assert TEST_CMD in meta_data 69 test_cmd = meta_data[TEST_CMD] 70 71 if OUTPUT not in meta_data: 72 output_file = DEFAULT_OUTPUT 73 else: 74 output_file = meta_data[OUTPUT] 75 76 if CONF not in meta_data: 77 conf_file = DEFAULT_CONF 78 else: 79 conf_file = meta_data[CONF] 80 81 if NUM_BUILDER not in meta_data: 82 num_builders = DEFAULT_NUM_BUILDER 83 else: 84 num_builders = meta_data[NUM_BUILDER] 85 86 if NUM_TESTER not in meta_data: 87 num_testers = DEFAULT_NUM_TESTER 88 else: 89 num_testers = meta_data[NUM_TESTER] 90 91 if STOP_THRESHOLD not in meta_data: 92 stop_threshold = DEFAULT_STOP_THRESHOLD 93 else: 94 stop_threshold = meta_data[STOP_THRESHOLD] 95 96 if NUM_CHROMOSOMES not in meta_data: 97 num_chromosomes = DEFAULT_NUM_CHROMOSOMES 98 else: 99 num_chromosomes = meta_data[NUM_CHROMOSOMES] 100 101 if NUM_TRIALS not in meta_data: 102 num_trials = DEFAULT_NUM_TRIALS 103 else: 104 num_trials = meta_data[NUM_TRIALS] 105 106 if MUTATION_RATE not in meta_data: 107 mutation_rate = DEFAULT_MUTATION_RATE 108 else: 109 mutation_rate = meta_data[MUTATION_RATE] 110 111 specs = flags.ReadConf(conf_file) 112 113 # Initiate the build/test command and the log directory. 114 Task.InitLogCommand(build_cmd, test_cmd, output_file) 115 116 # Initiate the build/test command and the log directory. 117 GAGeneration.InitMetaData(stop_threshold, num_chromosomes, num_trials, specs, 118 mutation_rate) 119 120 # Generate the initial generations. 121 generation_tasks = testing_batch.GenerateRandomGATasks(specs, num_chromosomes, 122 num_trials) 123 generations = [GAGeneration(generation_tasks, set([]), 0)] 124 125 # Execute the experiment. 126 _StartExperiment(num_builders, num_testers, generations) 127 128 129 def _ParseJson(file_name): 130 """Parse the input json file. 131 132 Parse the input json file and call the proper function to perform the 133 algorithms. 134 135 Args: 136 file_name: the input json file name. 137 """ 138 139 experiments = json.load(open(file_name)) 140 141 for experiment in experiments: 142 if experiment == 'GA': 143 # An GA experiment 144 _ProcessGA(experiments[experiment]) 145 146 147 def _StartExperiment(num_builders, num_testers, generations): 148 """Set up the experiment environment and execute the framework. 149 150 Args: 151 num_builders: number of concurrent builders. 152 num_testers: number of concurrent testers. 153 generations: the initial generation for the framework. 154 """ 155 156 manager = multiprocessing.Manager() 157 158 # The queue between the steering algorithm and the builder. 159 steering_build = manager.Queue() 160 # The queue between the builder and the tester. 161 build_test = manager.Queue() 162 # The queue between the tester and the steering algorithm. 163 test_steering = manager.Queue() 164 165 # Set up the processes for the builder, tester and steering algorithm module. 166 build_process = PipelineProcess(num_builders, 'builder', {}, BUILD_STAGE, 167 steering_build, pipeline_worker.Helper, 168 pipeline_worker.Worker, build_test) 169 170 test_process = PipelineProcess(num_testers, 'tester', {}, TEST_STAGE, 171 build_test, pipeline_worker.Helper, 172 pipeline_worker.Worker, test_steering) 173 174 steer_process = multiprocessing.Process( 175 target=Steering, 176 args=(set([]), generations, test_steering, steering_build)) 177 178 # Start the processes. 179 build_process.start() 180 test_process.start() 181 steer_process.start() 182 183 # Wait for the processes to finish. 184 build_process.join() 185 test_process.join() 186 steer_process.join() 187 188 189 def main(argv): 190 (options, _) = parser.parse_args(argv) 191 assert options.filename 192 _ParseJson(options.filename) 193 194 195 if __name__ == '__main__': 196 main(sys.argv) 197
