Home | History | Annotate | Download | only in devtools
      1 from __future__ import print_function
      2 import collections
      3 import itertools
      4 import json
      5 import os
      6 import os.path
      7 import re
      8 import shutil
      9 import string
     10 import subprocess
     11 import sys
     12 import cgi
     13 
     14 class BuildDesc:
     15     def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None):
     16         self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ]
     17         self.variables = variables or []
     18         self.build_type = build_type
     19         self.generator = generator
     20 
     21     def merged_with( self, build_desc ):
     22         """Returns a new BuildDesc by merging field content.
     23            Prefer build_desc fields to self fields for single valued field.
     24         """
     25         return BuildDesc( self.prepend_envs + build_desc.prepend_envs,
     26                           self.variables + build_desc.variables,
     27                           build_desc.build_type or self.build_type,
     28                           build_desc.generator or self.generator )
     29 
     30     def env( self ):
     31         environ = os.environ.copy()
     32         for values_by_name in self.prepend_envs:
     33             for var, value in list(values_by_name.items()):
     34                 var = var.upper()
     35                 if type(value) is unicode:
     36                     value = value.encode( sys.getdefaultencoding() )
     37                 if var in environ:
     38                     environ[var] = value + os.pathsep + environ[var]
     39                 else:
     40                     environ[var] = value
     41         return environ
     42 
     43     def cmake_args( self ):
     44         args = ["-D%s" % var for var in self.variables]
     45         # skip build type for Visual Studio solution as it cause warning
     46         if self.build_type and 'Visual' not in self.generator:
     47             args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type )
     48         if self.generator:
     49             args.extend( ['-G', self.generator] )
     50         return args
     51 
     52     def __repr__( self ):
     53         return "BuildDesc( %s, build_type=%s )" %  (" ".join( self.cmake_args()), self.build_type)
     54 
     55 class BuildData:
     56     def __init__( self, desc, work_dir, source_dir ):
     57         self.desc = desc
     58         self.work_dir = work_dir
     59         self.source_dir = source_dir
     60         self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' )
     61         self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' )
     62         self.cmake_succeeded = False
     63         self.build_succeeded = False
     64 
     65     def execute_build(self):
     66         print('Build %s' % self.desc)
     67         self._make_new_work_dir( )
     68         self.cmake_succeeded = self._generate_makefiles( )
     69         if self.cmake_succeeded:
     70             self.build_succeeded = self._build_using_makefiles( )
     71         return self.build_succeeded
     72 
     73     def _generate_makefiles(self):
     74         print('  Generating makefiles: ', end=' ')
     75         cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )]
     76         succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path )
     77         print('done' if succeeded else 'FAILED')
     78         return succeeded
     79 
     80     def _build_using_makefiles(self):
     81         print('  Building:', end=' ')
     82         cmd = ['cmake', '--build', self.work_dir]
     83         if self.desc.build_type:
     84             cmd += ['--config', self.desc.build_type]
     85         succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path )
     86         print('done' if succeeded else 'FAILED')
     87         return succeeded
     88 
     89     def _execute_build_subprocess(self, cmd, env, log_path):
     90         process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir,
     91                                     env=env )
     92         stdout, _ = process.communicate( )
     93         succeeded = (process.returncode == 0)
     94         with open( log_path, 'wb' ) as flog:
     95             log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode
     96             flog.write( fix_eol( log ) )
     97         return succeeded
     98 
     99     def _make_new_work_dir(self):
    100         if os.path.isdir( self.work_dir ):
    101             print('  Removing work directory', self.work_dir)
    102             shutil.rmtree( self.work_dir, ignore_errors=True )
    103         if not os.path.isdir( self.work_dir ):
    104             os.makedirs( self.work_dir )
    105 
    106 def fix_eol( stdout ):
    107     """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n).
    108     """
    109     return re.sub( '\r*\n', os.linesep, stdout )
    110 
    111 def load_build_variants_from_config( config_path ):
    112     with open( config_path, 'rb' ) as fconfig:
    113         data = json.load( fconfig )
    114     variants = data[ 'cmake_variants' ]
    115     build_descs_by_axis = collections.defaultdict( list )
    116     for axis in variants:
    117         axis_name = axis["name"]
    118         build_descs = []
    119         if "generators" in axis:
    120             for generator_data in axis["generators"]:
    121                 for generator in generator_data["generator"]:
    122                     build_desc = BuildDesc( generator=generator,
    123                                             prepend_envs=generator_data.get("env_prepend") )
    124                     build_descs.append( build_desc )
    125         elif "variables" in axis:
    126             for variables in axis["variables"]:
    127                 build_desc = BuildDesc( variables=variables )
    128                 build_descs.append( build_desc )
    129         elif "build_types" in axis:
    130             for build_type in axis["build_types"]:
    131                 build_desc = BuildDesc( build_type=build_type )
    132                 build_descs.append( build_desc )
    133         build_descs_by_axis[axis_name].extend( build_descs )
    134     return build_descs_by_axis
    135 
    136 def generate_build_variants( build_descs_by_axis ):
    137     """Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""
    138     axis_names = list(build_descs_by_axis.keys())
    139     build_descs = []
    140     for axis_name, axis_build_descs in list(build_descs_by_axis.items()):
    141         if len(build_descs):
    142             # for each existing build_desc and each axis build desc, create a new build_desc
    143             new_build_descs = []
    144             for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs):
    145                 new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) )
    146             build_descs = new_build_descs
    147         else:
    148             build_descs = axis_build_descs
    149     return build_descs
    150 
    151 HTML_TEMPLATE = string.Template('''<html>
    152 <head>
    153     <title>$title</title>
    154     <style type="text/css">
    155     td.failed {background-color:#f08080;}
    156     td.ok {background-color:#c0eec0;}
    157     </style>
    158 </head>
    159 <body>
    160 <table border="1">
    161 <thead>
    162     <tr>
    163         <th>Variables</th>
    164         $th_vars
    165     </tr>
    166     <tr>
    167         <th>Build type</th>
    168         $th_build_types
    169     </tr>
    170 </thead>
    171 <tbody>
    172 $tr_builds
    173 </tbody>
    174 </table>
    175 </body></html>''')
    176 
    177 def generate_html_report( html_report_path, builds ):
    178     report_dir = os.path.dirname( html_report_path )
    179     # Vertical axis: generator
    180     # Horizontal: variables, then build_type
    181     builds_by_generator = collections.defaultdict( list )
    182     variables = set()
    183     build_types_by_variable = collections.defaultdict( set )
    184     build_by_pos_key = {} # { (generator, var_key, build_type): build }
    185     for build in builds:
    186         builds_by_generator[build.desc.generator].append( build )
    187         var_key = tuple(sorted(build.desc.variables))
    188         variables.add( var_key )
    189         build_types_by_variable[var_key].add( build.desc.build_type )
    190         pos_key = (build.desc.generator, var_key, build.desc.build_type)
    191         build_by_pos_key[pos_key] = build
    192     variables = sorted( variables )
    193     th_vars = []
    194     th_build_types = []
    195     for variable in variables:
    196         build_types = sorted( build_types_by_variable[variable] )
    197         nb_build_type = len(build_types_by_variable[variable])
    198         th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) )
    199         for build_type in build_types:
    200             th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) )
    201     tr_builds = []
    202     for generator in sorted( builds_by_generator ):
    203         tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ]
    204         for variable in variables:
    205             build_types = sorted( build_types_by_variable[variable] )
    206             for build_type in build_types:
    207                 pos_key = (generator, variable, build_type)
    208                 build = build_by_pos_key.get(pos_key)
    209                 if build:
    210                     cmake_status = 'ok' if build.cmake_succeeded else 'FAILED'
    211                     build_status = 'ok' if build.build_succeeded else 'FAILED'
    212                     cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir )
    213                     build_log_url = os.path.relpath( build.build_log_path, report_dir )
    214                     td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (
    215                         build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
    216                     if build.cmake_succeeded:
    217                         td += '<br><a href="%s" class="%s">Build: %s</a>' % (
    218                             build_log_url, build_status.lower(), build_status)
    219                     td += '</td>'
    220                 else:
    221                     td = '<td></td>'
    222                 tds.append( td )
    223         tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) )
    224     html = HTML_TEMPLATE.substitute(
    225         title='Batch build report',
    226         th_vars=' '.join(th_vars),
    227         th_build_types=' '.join( th_build_types),
    228         tr_builds='\n'.join( tr_builds ) )
    229     with open( html_report_path, 'wt' ) as fhtml:
    230         fhtml.write( html )
    231     print('HTML report generated in:', html_report_path)
    232 
    233 def main():
    234     usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]
    235 Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run
    236 as described in CONFIG_JSON_PATH building in WORK_DIR.
    237 
    238 Example of call:
    239 python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json
    240 """
    241     from optparse import OptionParser
    242     parser = OptionParser(usage=usage)
    243     parser.allow_interspersed_args = True
    244 #    parser.add_option('-v', '--verbose', dest="verbose", action='store_true',
    245 #        help="""Be verbose.""")
    246     parser.enable_interspersed_args()
    247     options, args = parser.parse_args()
    248     if len(args) < 3:
    249         parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." )
    250     work_dir = args[0]
    251     source_dir = args[1].rstrip('/\\')
    252     config_paths = args[2:]
    253     for config_path in config_paths:
    254         if not os.path.isfile( config_path ):
    255             parser.error( "Can not read: %r" % config_path )
    256 
    257     # generate build variants
    258     build_descs = []
    259     for config_path in config_paths:
    260         build_descs_by_axis = load_build_variants_from_config( config_path )
    261         build_descs.extend( generate_build_variants( build_descs_by_axis ) )
    262     print('Build variants (%d):' % len(build_descs))
    263     # assign build directory for each variant
    264     if not os.path.isdir( work_dir ):
    265         os.makedirs( work_dir )
    266     builds = []
    267     with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap:
    268         for index, build_desc in enumerate( build_descs ):
    269             build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) )
    270             builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) )
    271             fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) )
    272     for build in builds:
    273         build.execute_build()
    274     html_report_path = os.path.join( work_dir, 'batchbuild-report.html' )
    275     generate_html_report( html_report_path, builds )
    276     print('Done')
    277 
    278 
    279 if __name__ == '__main__':
    280     main()
    281 
    282