1 from __future__ import print_function 2 import collections 3 import itertools 4 import json 5 import os 6 import os.path 7 import re 8 import shutil 9 import string 10 import subprocess 11 import sys 12 import cgi 13 14 class BuildDesc: 15 def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): 16 self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] 17 self.variables = variables or [] 18 self.build_type = build_type 19 self.generator = generator 20 21 def merged_with( self, build_desc ): 22 """Returns a new BuildDesc by merging field content. 23 Prefer build_desc fields to self fields for single valued field. 24 """ 25 return BuildDesc( self.prepend_envs + build_desc.prepend_envs, 26 self.variables + build_desc.variables, 27 build_desc.build_type or self.build_type, 28 build_desc.generator or self.generator ) 29 30 def env( self ): 31 environ = os.environ.copy() 32 for values_by_name in self.prepend_envs: 33 for var, value in list(values_by_name.items()): 34 var = var.upper() 35 if type(value) is unicode: 36 value = value.encode( sys.getdefaultencoding() ) 37 if var in environ: 38 environ[var] = value + os.pathsep + environ[var] 39 else: 40 environ[var] = value 41 return environ 42 43 def cmake_args( self ): 44 args = ["-D%s" % var for var in self.variables] 45 # skip build type for Visual Studio solution as it cause warning 46 if self.build_type and 'Visual' not in self.generator: 47 args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) 48 if self.generator: 49 args.extend( ['-G', self.generator] ) 50 return args 51 52 def __repr__( self ): 53 return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) 54 55 class BuildData: 56 def __init__( self, desc, work_dir, source_dir ): 57 self.desc = desc 58 self.work_dir = work_dir 59 self.source_dir = source_dir 60 self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) 61 self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) 62 self.cmake_succeeded = False 63 self.build_succeeded = False 64 65 def execute_build(self): 66 print('Build %s' % self.desc) 67 self._make_new_work_dir( ) 68 self.cmake_succeeded = self._generate_makefiles( ) 69 if self.cmake_succeeded: 70 self.build_succeeded = self._build_using_makefiles( ) 71 return self.build_succeeded 72 73 def _generate_makefiles(self): 74 print(' Generating makefiles: ', end=' ') 75 cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] 76 succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) 77 print('done' if succeeded else 'FAILED') 78 return succeeded 79 80 def _build_using_makefiles(self): 81 print(' Building:', end=' ') 82 cmd = ['cmake', '--build', self.work_dir] 83 if self.desc.build_type: 84 cmd += ['--config', self.desc.build_type] 85 succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) 86 print('done' if succeeded else 'FAILED') 87 return succeeded 88 89 def _execute_build_subprocess(self, cmd, env, log_path): 90 process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, 91 env=env ) 92 stdout, _ = process.communicate( ) 93 succeeded = (process.returncode == 0) 94 with open( log_path, 'wb' ) as flog: 95 log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode 96 flog.write( fix_eol( log ) ) 97 return succeeded 98 99 def _make_new_work_dir(self): 100 if os.path.isdir( self.work_dir ): 101 print(' Removing work directory', self.work_dir) 102 shutil.rmtree( self.work_dir, ignore_errors=True ) 103 if not os.path.isdir( self.work_dir ): 104 os.makedirs( self.work_dir ) 105 106 def fix_eol( stdout ): 107 """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). 108 """ 109 return re.sub( '\r*\n', os.linesep, stdout ) 110 111 def load_build_variants_from_config( config_path ): 112 with open( config_path, 'rb' ) as fconfig: 113 data = json.load( fconfig ) 114 variants = data[ 'cmake_variants' ] 115 build_descs_by_axis = collections.defaultdict( list ) 116 for axis in variants: 117 axis_name = axis["name"] 118 build_descs = [] 119 if "generators" in axis: 120 for generator_data in axis["generators"]: 121 for generator in generator_data["generator"]: 122 build_desc = BuildDesc( generator=generator, 123 prepend_envs=generator_data.get("env_prepend") ) 124 build_descs.append( build_desc ) 125 elif "variables" in axis: 126 for variables in axis["variables"]: 127 build_desc = BuildDesc( variables=variables ) 128 build_descs.append( build_desc ) 129 elif "build_types" in axis: 130 for build_type in axis["build_types"]: 131 build_desc = BuildDesc( build_type=build_type ) 132 build_descs.append( build_desc ) 133 build_descs_by_axis[axis_name].extend( build_descs ) 134 return build_descs_by_axis 135 136 def generate_build_variants( build_descs_by_axis ): 137 """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" 138 axis_names = list(build_descs_by_axis.keys()) 139 build_descs = [] 140 for axis_name, axis_build_descs in list(build_descs_by_axis.items()): 141 if len(build_descs): 142 # for each existing build_desc and each axis build desc, create a new build_desc 143 new_build_descs = [] 144 for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): 145 new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) 146 build_descs = new_build_descs 147 else: 148 build_descs = axis_build_descs 149 return build_descs 150 151 HTML_TEMPLATE = string.Template('''<html> 152 <head> 153 <title>$title</title> 154 <style type="text/css"> 155 td.failed {background-color:#f08080;} 156 td.ok {background-color:#c0eec0;} 157 </style> 158 </head> 159 <body> 160 <table border="1"> 161 <thead> 162 <tr> 163 <th>Variables</th> 164 $th_vars 165 </tr> 166 <tr> 167 <th>Build type</th> 168 $th_build_types 169 </tr> 170 </thead> 171 <tbody> 172 $tr_builds 173 </tbody> 174 </table> 175 </body></html>''') 176 177 def generate_html_report( html_report_path, builds ): 178 report_dir = os.path.dirname( html_report_path ) 179 # Vertical axis: generator 180 # Horizontal: variables, then build_type 181 builds_by_generator = collections.defaultdict( list ) 182 variables = set() 183 build_types_by_variable = collections.defaultdict( set ) 184 build_by_pos_key = {} # { (generator, var_key, build_type): build } 185 for build in builds: 186 builds_by_generator[build.desc.generator].append( build ) 187 var_key = tuple(sorted(build.desc.variables)) 188 variables.add( var_key ) 189 build_types_by_variable[var_key].add( build.desc.build_type ) 190 pos_key = (build.desc.generator, var_key, build.desc.build_type) 191 build_by_pos_key[pos_key] = build 192 variables = sorted( variables ) 193 th_vars = [] 194 th_build_types = [] 195 for variable in variables: 196 build_types = sorted( build_types_by_variable[variable] ) 197 nb_build_type = len(build_types_by_variable[variable]) 198 th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) 199 for build_type in build_types: 200 th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) ) 201 tr_builds = [] 202 for generator in sorted( builds_by_generator ): 203 tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ] 204 for variable in variables: 205 build_types = sorted( build_types_by_variable[variable] ) 206 for build_type in build_types: 207 pos_key = (generator, variable, build_type) 208 build = build_by_pos_key.get(pos_key) 209 if build: 210 cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' 211 build_status = 'ok' if build.build_succeeded else 'FAILED' 212 cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) 213 build_log_url = os.path.relpath( build.build_log_path, report_dir ) 214 td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % ( 215 build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) 216 if build.cmake_succeeded: 217 td += '<br><a href="%s" class="%s">Build: %s</a>' % ( 218 build_log_url, build_status.lower(), build_status) 219 td += '</td>' 220 else: 221 td = '<td></td>' 222 tds.append( td ) 223 tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) ) 224 html = HTML_TEMPLATE.substitute( 225 title='Batch build report', 226 th_vars=' '.join(th_vars), 227 th_build_types=' '.join( th_build_types), 228 tr_builds='\n'.join( tr_builds ) ) 229 with open( html_report_path, 'wt' ) as fhtml: 230 fhtml.write( html ) 231 print('HTML report generated in:', html_report_path) 232 233 def main(): 234 usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] 235 Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run 236 as described in CONFIG_JSON_PATH building in WORK_DIR. 237 238 Example of call: 239 python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json 240 """ 241 from optparse import OptionParser 242 parser = OptionParser(usage=usage) 243 parser.allow_interspersed_args = True 244 # parser.add_option('-v', '--verbose', dest="verbose", action='store_true', 245 # help="""Be verbose.""") 246 parser.enable_interspersed_args() 247 options, args = parser.parse_args() 248 if len(args) < 3: 249 parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) 250 work_dir = args[0] 251 source_dir = args[1].rstrip('/\\') 252 config_paths = args[2:] 253 for config_path in config_paths: 254 if not os.path.isfile( config_path ): 255 parser.error( "Can not read: %r" % config_path ) 256 257 # generate build variants 258 build_descs = [] 259 for config_path in config_paths: 260 build_descs_by_axis = load_build_variants_from_config( config_path ) 261 build_descs.extend( generate_build_variants( build_descs_by_axis ) ) 262 print('Build variants (%d):' % len(build_descs)) 263 # assign build directory for each variant 264 if not os.path.isdir( work_dir ): 265 os.makedirs( work_dir ) 266 builds = [] 267 with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: 268 for index, build_desc in enumerate( build_descs ): 269 build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) 270 builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) 271 fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) 272 for build in builds: 273 build.execute_build() 274 html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) 275 generate_html_report( html_report_path, builds ) 276 print('Done') 277 278 279 if __name__ == '__main__': 280 main() 281 282
