1 """Tag the sandbox for release, make source and doc tarballs. 2 3 Requires Python 2.6 4 5 Example of invocation (use to test the script): 6 python makerelease.py --platform=msvc6,msvc71,msvc80,msvc90,mingw -ublep 0.6.0 0.7.0-dev 7 8 When testing this script: 9 python makerelease.py --force --retag --platform=msvc6,msvc71,msvc80,mingw -ublep test-0.6.0 test-0.6.1-dev 10 11 Example of invocation when doing a release: 12 python makerelease.py 0.5.0 0.6.0-dev 13 14 Note: This was for Subversion. Now that we are in GitHub, we do not 15 need to build versioned tarballs anymore, so makerelease.py is defunct. 16 """ 17 from __future__ import print_function 18 import os.path 19 import subprocess 20 import sys 21 import doxybuild 22 import subprocess 23 import xml.etree.ElementTree as ElementTree 24 import shutil 25 import urllib2 26 import tempfile 27 import os 28 import time 29 from devtools import antglob, fixeol, tarball 30 import amalgamate 31 32 SVN_ROOT = 'https://jsoncpp.svn.sourceforge.net/svnroot/jsoncpp/' 33 SVN_TAG_ROOT = SVN_ROOT + 'tags/jsoncpp' 34 SCONS_LOCAL_URL = 'http://sourceforge.net/projects/scons/files/scons-local/1.2.0/scons-local-1.2.0.tar.gz/download' 35 SOURCEFORGE_PROJECT = 'jsoncpp' 36 37 def set_version( version ): 38 with open('version','wb') as f: 39 f.write( version.strip() ) 40 41 def rmdir_if_exist( dir_path ): 42 if os.path.isdir( dir_path ): 43 shutil.rmtree( dir_path ) 44 45 class SVNError(Exception): 46 pass 47 48 def svn_command( command, *args ): 49 cmd = ['svn', '--non-interactive', command] + list(args) 50 print('Running:', ' '.join( cmd )) 51 process = subprocess.Popen( cmd, 52 stdout=subprocess.PIPE, 53 stderr=subprocess.STDOUT ) 54 stdout = process.communicate()[0] 55 if process.returncode: 56 error = SVNError( 'SVN command failed:\n' + stdout ) 57 error.returncode = process.returncode 58 raise error 59 return stdout 60 61 def check_no_pending_commit(): 62 """Checks that there is no pending commit in the sandbox.""" 63 stdout = svn_command( 'status', '--xml' ) 64 etree = ElementTree.fromstring( stdout ) 65 msg = [] 66 for entry in etree.getiterator( 'entry' ): 67 path = entry.get('path') 68 status = entry.find('wc-status').get('item') 69 if status != 'unversioned' and path != 'version': 70 msg.append( 'File "%s" has pending change (status="%s")' % (path, status) ) 71 if msg: 72 msg.insert(0, 'Pending change to commit found in sandbox. Commit them first!' ) 73 return '\n'.join( msg ) 74 75 def svn_join_url( base_url, suffix ): 76 if not base_url.endswith('/'): 77 base_url += '/' 78 if suffix.startswith('/'): 79 suffix = suffix[1:] 80 return base_url + suffix 81 82 def svn_check_if_tag_exist( tag_url ): 83 """Checks if a tag exist. 84 Returns: True if the tag exist, False otherwise. 85 """ 86 try: 87 list_stdout = svn_command( 'list', tag_url ) 88 except SVNError as e: 89 if e.returncode != 1 or not str(e).find('tag_url'): 90 raise e 91 # otherwise ignore error, meaning tag does not exist 92 return False 93 return True 94 95 def svn_commit( message ): 96 """Commit the sandbox, providing the specified comment. 97 """ 98 svn_command( 'ci', '-m', message ) 99 100 def svn_tag_sandbox( tag_url, message ): 101 """Makes a tag based on the sandbox revisions. 102 """ 103 svn_command( 'copy', '-m', message, '.', tag_url ) 104 105 def svn_remove_tag( tag_url, message ): 106 """Removes an existing tag. 107 """ 108 svn_command( 'delete', '-m', message, tag_url ) 109 110 def svn_export( tag_url, export_dir ): 111 """Exports the tag_url revision to export_dir. 112 Target directory, including its parent is created if it does not exist. 113 If the directory export_dir exist, it is deleted before export proceed. 114 """ 115 rmdir_if_exist( export_dir ) 116 svn_command( 'export', tag_url, export_dir ) 117 118 def fix_sources_eol( dist_dir ): 119 """Set file EOL for tarball distribution. 120 """ 121 print('Preparing exported source file EOL for distribution...') 122 prune_dirs = antglob.prune_dirs + 'scons-local* ./build* ./libs ./dist' 123 win_sources = antglob.glob( dist_dir, 124 includes = '**/*.sln **/*.vcproj', 125 prune_dirs = prune_dirs ) 126 unix_sources = antglob.glob( dist_dir, 127 includes = '''**/*.h **/*.cpp **/*.inl **/*.txt **/*.dox **/*.py **/*.html **/*.in 128 sconscript *.json *.expected AUTHORS LICENSE''', 129 excludes = antglob.default_excludes + 'scons.py sconsign.py scons-*', 130 prune_dirs = prune_dirs ) 131 for path in win_sources: 132 fixeol.fix_source_eol( path, is_dry_run = False, verbose = True, eol = '\r\n' ) 133 for path in unix_sources: 134 fixeol.fix_source_eol( path, is_dry_run = False, verbose = True, eol = '\n' ) 135 136 def download( url, target_path ): 137 """Download file represented by url to target_path. 138 """ 139 f = urllib2.urlopen( url ) 140 try: 141 data = f.read() 142 finally: 143 f.close() 144 fout = open( target_path, 'wb' ) 145 try: 146 fout.write( data ) 147 finally: 148 fout.close() 149 150 def check_compile( distcheck_top_dir, platform ): 151 cmd = [sys.executable, 'scons.py', 'platform=%s' % platform, 'check'] 152 print('Running:', ' '.join( cmd )) 153 log_path = os.path.join( distcheck_top_dir, 'build-%s.log' % platform ) 154 flog = open( log_path, 'wb' ) 155 try: 156 process = subprocess.Popen( cmd, 157 stdout=flog, 158 stderr=subprocess.STDOUT, 159 cwd=distcheck_top_dir ) 160 stdout = process.communicate()[0] 161 status = (process.returncode == 0) 162 finally: 163 flog.close() 164 return (status, log_path) 165 166 def write_tempfile( content, **kwargs ): 167 fd, path = tempfile.mkstemp( **kwargs ) 168 f = os.fdopen( fd, 'wt' ) 169 try: 170 f.write( content ) 171 finally: 172 f.close() 173 return path 174 175 class SFTPError(Exception): 176 pass 177 178 def run_sftp_batch( userhost, sftp, batch, retry=0 ): 179 path = write_tempfile( batch, suffix='.sftp', text=True ) 180 # psftp -agent -C blep,jsoncpp (at] web.sourceforge.net -batch -b batch.sftp -bc 181 cmd = [sftp, '-agent', '-C', '-batch', '-b', path, '-bc', userhost] 182 error = None 183 for retry_index in range(0, max(1,retry)): 184 heading = retry_index == 0 and 'Running:' or 'Retrying:' 185 print(heading, ' '.join( cmd )) 186 process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) 187 stdout = process.communicate()[0] 188 if process.returncode != 0: 189 error = SFTPError( 'SFTP batch failed:\n' + stdout ) 190 else: 191 break 192 if error: 193 raise error 194 return stdout 195 196 def sourceforge_web_synchro( sourceforge_project, doc_dir, 197 user=None, sftp='sftp' ): 198 """Notes: does not synchronize sub-directory of doc-dir. 199 """ 200 userhost = '%s,%s (at] web.sourceforge.net' % (user, sourceforge_project) 201 stdout = run_sftp_batch( userhost, sftp, """ 202 cd htdocs 203 dir 204 exit 205 """ ) 206 existing_paths = set() 207 collect = 0 208 for line in stdout.split('\n'): 209 line = line.strip() 210 if not collect and line.endswith('> dir'): 211 collect = True 212 elif collect and line.endswith('> exit'): 213 break 214 elif collect == 1: 215 collect = 2 216 elif collect == 2: 217 path = line.strip().split()[-1:] 218 if path and path[0] not in ('.', '..'): 219 existing_paths.add( path[0] ) 220 upload_paths = set( [os.path.basename(p) for p in antglob.glob( doc_dir )] ) 221 paths_to_remove = existing_paths - upload_paths 222 if paths_to_remove: 223 print('Removing the following file from web:') 224 print('\n'.join( paths_to_remove )) 225 stdout = run_sftp_batch( userhost, sftp, """cd htdocs 226 rm %s 227 exit""" % ' '.join(paths_to_remove) ) 228 print('Uploading %d files:' % len(upload_paths)) 229 batch_size = 10 230 upload_paths = list(upload_paths) 231 start_time = time.time() 232 for index in range(0,len(upload_paths),batch_size): 233 paths = upload_paths[index:index+batch_size] 234 file_per_sec = (time.time() - start_time) / (index+1) 235 remaining_files = len(upload_paths) - index 236 remaining_sec = file_per_sec * remaining_files 237 print('%d/%d, ETA=%.1fs' % (index+1, len(upload_paths), remaining_sec)) 238 run_sftp_batch( userhost, sftp, """cd htdocs 239 lcd %s 240 mput %s 241 exit""" % (doc_dir, ' '.join(paths) ), retry=3 ) 242 243 def sourceforge_release_tarball( sourceforge_project, paths, user=None, sftp='sftp' ): 244 userhost = '%s,%s (at] frs.sourceforge.net' % (user, sourceforge_project) 245 run_sftp_batch( userhost, sftp, """ 246 mput %s 247 exit 248 """ % (' '.join(paths),) ) 249 250 251 def main(): 252 usage = """%prog release_version next_dev_version 253 Update 'version' file to release_version and commit. 254 Generates the document tarball. 255 Tags the sandbox revision with release_version. 256 Update 'version' file to next_dev_version and commit. 257 258 Performs an svn export of tag release version, and build a source tarball. 259 260 Must be started in the project top directory. 261 262 Warning: --force should only be used when developping/testing the release script. 263 """ 264 from optparse import OptionParser 265 parser = OptionParser(usage=usage) 266 parser.allow_interspersed_args = False 267 parser.add_option('--dot', dest="dot_path", action='store', default=doxybuild.find_program('dot'), 268 help="""Path to GraphViz dot tool. Must be full qualified path. [Default: %default]""") 269 parser.add_option('--doxygen', dest="doxygen_path", action='store', default=doxybuild.find_program('doxygen'), 270 help="""Path to Doxygen tool. [Default: %default]""") 271 parser.add_option('--force', dest="ignore_pending_commit", action='store_true', default=False, 272 help="""Ignore pending commit. [Default: %default]""") 273 parser.add_option('--retag', dest="retag_release", action='store_true', default=False, 274 help="""Overwrite release existing tag if it exist. [Default: %default]""") 275 parser.add_option('-p', '--platforms', dest="platforms", action='store', default='', 276 help="""Comma separated list of platform passed to scons for build check.""") 277 parser.add_option('--no-test', dest="no_test", action='store_true', default=False, 278 help="""Skips build check.""") 279 parser.add_option('--no-web', dest="no_web", action='store_true', default=False, 280 help="""Do not update web site.""") 281 parser.add_option('-u', '--upload-user', dest="user", action='store', 282 help="""Sourceforge user for SFTP documentation upload.""") 283 parser.add_option('--sftp', dest='sftp', action='store', default=doxybuild.find_program('psftp', 'sftp'), 284 help="""Path of the SFTP compatible binary used to upload the documentation.""") 285 parser.enable_interspersed_args() 286 options, args = parser.parse_args() 287 288 if len(args) != 2: 289 parser.error( 'release_version missing on command-line.' ) 290 release_version = args[0] 291 next_version = args[1] 292 293 if not options.platforms and not options.no_test: 294 parser.error( 'You must specify either --platform or --no-test option.' ) 295 296 if options.ignore_pending_commit: 297 msg = '' 298 else: 299 msg = check_no_pending_commit() 300 if not msg: 301 print('Setting version to', release_version) 302 set_version( release_version ) 303 svn_commit( 'Release ' + release_version ) 304 tag_url = svn_join_url( SVN_TAG_ROOT, release_version ) 305 if svn_check_if_tag_exist( tag_url ): 306 if options.retag_release: 307 svn_remove_tag( tag_url, 'Overwriting previous tag' ) 308 else: 309 print('Aborting, tag %s already exist. Use --retag to overwrite it!' % tag_url) 310 sys.exit( 1 ) 311 svn_tag_sandbox( tag_url, 'Release ' + release_version ) 312 313 print('Generated doxygen document...') 314 ## doc_dirname = r'jsoncpp-api-html-0.5.0' 315 ## doc_tarball_path = r'e:\prg\vc\Lib\jsoncpp-trunk\dist\jsoncpp-api-html-0.5.0.tar.gz' 316 doc_tarball_path, doc_dirname = doxybuild.build_doc( options, make_release=True ) 317 doc_distcheck_dir = 'dist/doccheck' 318 tarball.decompress( doc_tarball_path, doc_distcheck_dir ) 319 doc_distcheck_top_dir = os.path.join( doc_distcheck_dir, doc_dirname ) 320 321 export_dir = 'dist/export' 322 svn_export( tag_url, export_dir ) 323 fix_sources_eol( export_dir ) 324 325 source_dir = 'jsoncpp-src-' + release_version 326 source_tarball_path = 'dist/%s.tar.gz' % source_dir 327 print('Generating source tarball to', source_tarball_path) 328 tarball.make_tarball( source_tarball_path, [export_dir], export_dir, prefix_dir=source_dir ) 329 330 amalgamation_tarball_path = 'dist/%s-amalgamation.tar.gz' % source_dir 331 print('Generating amalgamation source tarball to', amalgamation_tarball_path) 332 amalgamation_dir = 'dist/amalgamation' 333 amalgamate.amalgamate_source( export_dir, '%s/jsoncpp.cpp' % amalgamation_dir, 'json/json.h' ) 334 amalgamation_source_dir = 'jsoncpp-src-amalgamation' + release_version 335 tarball.make_tarball( amalgamation_tarball_path, [amalgamation_dir], 336 amalgamation_dir, prefix_dir=amalgamation_source_dir ) 337 338 # Decompress source tarball, download and install scons-local 339 distcheck_dir = 'dist/distcheck' 340 distcheck_top_dir = distcheck_dir + '/' + source_dir 341 print('Decompressing source tarball to', distcheck_dir) 342 rmdir_if_exist( distcheck_dir ) 343 tarball.decompress( source_tarball_path, distcheck_dir ) 344 scons_local_path = 'dist/scons-local.tar.gz' 345 print('Downloading scons-local to', scons_local_path) 346 download( SCONS_LOCAL_URL, scons_local_path ) 347 print('Decompressing scons-local to', distcheck_top_dir) 348 tarball.decompress( scons_local_path, distcheck_top_dir ) 349 350 # Run compilation 351 print('Compiling decompressed tarball') 352 all_build_status = True 353 for platform in options.platforms.split(','): 354 print('Testing platform:', platform) 355 build_status, log_path = check_compile( distcheck_top_dir, platform ) 356 print('see build log:', log_path) 357 print(build_status and '=> ok' or '=> FAILED') 358 all_build_status = all_build_status and build_status 359 if not build_status: 360 print('Testing failed on at least one platform, aborting...') 361 svn_remove_tag( tag_url, 'Removing tag due to failed testing' ) 362 sys.exit(1) 363 if options.user: 364 if not options.no_web: 365 print('Uploading documentation using user', options.user) 366 sourceforge_web_synchro( SOURCEFORGE_PROJECT, doc_distcheck_top_dir, user=options.user, sftp=options.sftp ) 367 print('Completed documentation upload') 368 print('Uploading source and documentation tarballs for release using user', options.user) 369 sourceforge_release_tarball( SOURCEFORGE_PROJECT, 370 [source_tarball_path, doc_tarball_path], 371 user=options.user, sftp=options.sftp ) 372 print('Source and doc release tarballs uploaded') 373 else: 374 print('No upload user specified. Web site and download tarbal were not uploaded.') 375 print('Tarball can be found at:', doc_tarball_path) 376 377 # Set next version number and commit 378 set_version( next_version ) 379 svn_commit( 'Released ' + release_version ) 380 else: 381 sys.stderr.write( msg + '\n' ) 382 383 if __name__ == '__main__': 384 main() 385
