1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import hashlib 7 import json 8 import os 9 import sys 10 11 import buildbot_common 12 import build_version 13 from build_paths import SCRIPT_DIR 14 15 GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/' 16 SDK_MANIFEST = 'naclsdk_manifest2.json' 17 MONO_MANIFEST = 'naclmono_manifest.json' 18 19 def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url, 20 upload_path, args): 21 install_dir = 'naclmono' 22 buildbot_common.RemoveDir(install_dir) 23 24 revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else [] 25 url_opt = ['--sdk-url', sdk_url] if sdk_url else [] 26 27 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py', 28 '--arch', 'x86-32', '--install-dir', install_dir] + 29 revision_opt + url_opt + args) 30 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py', 31 '--arch', 'x86-64', '--install-dir', install_dir] + 32 revision_opt + url_opt + args) 33 buildbot_common.Run([sys.executable, 'nacl-mono-builder.py', 34 '--arch', 'arm', '--install-dir', install_dir] + 35 revision_opt + url_opt + args) 36 buildbot_common.Run([sys.executable, 'nacl-mono-archive.py', 37 '--upload-path', upload_path, 38 '--pepper-revision', pepper_revision, 39 '--install-dir', install_dir] + args) 40 41 def get_sdk_build_info(): 42 '''Returns a list of dictionaries for versions of NaCl Mono to build which are 43 out of date compared to the SDKs available to naclsdk''' 44 45 # Get a copy of the naclsdk manifest file 46 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', 47 GS_MANIFEST_PATH + SDK_MANIFEST, '.']) 48 manifest_file = open(SDK_MANIFEST, 'r') 49 sdk_manifest = json.loads(manifest_file.read()) 50 manifest_file.close() 51 52 pepper_infos = [] 53 for key, value in sdk_manifest.items(): 54 if key == 'bundles': 55 stabilities = ['stable', 'beta', 'dev', 'post_stable'] 56 # Pick pepper_* bundles, need pepper_19 or greater to build Mono 57 bundles = filter(lambda b: (b['stability'] in stabilities 58 and 'pepper_' in b['name']) 59 and b['version'] >= 19, value) 60 for b in bundles: 61 newdict = {} 62 newdict['pepper_revision'] = str(b['version']) 63 linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives']) 64 newdict['sdk_url'] = linux_arch[0]['url'] 65 newdict['sdk_revision'] = b['revision'] 66 newdict['stability'] = b['stability'] 67 newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision'] 68 pepper_infos.append(newdict) 69 70 # Get a copy of the naclmono manifest file 71 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', 72 GS_MANIFEST_PATH + MONO_MANIFEST, '.']) 73 manifest_file = open(MONO_MANIFEST, 'r') 74 mono_manifest = json.loads(manifest_file.read()) 75 manifest_file.close() 76 77 ret = [] 78 mono_manifest_dirty = False 79 # Check to see if we need to rebuild mono based on sdk revision 80 for key, value in mono_manifest.items(): 81 if key == 'bundles': 82 for info in pepper_infos: 83 bundle = filter(lambda b: b['name'] == info['naclmono_name'], value) 84 if len(bundle) == 0: 85 info['naclmono_rev'] = '1' 86 ret.append(info) 87 else: 88 if info['sdk_revision'] != bundle[0]['sdk_revision']: 89 # This bundle exists in the mono manifest, bump the revision 90 # for the new build we're about to make. 91 info['naclmono_rev'] = str(bundle[0]['revision'] + 1) 92 ret.append(info) 93 elif info['stability'] != bundle[0]['stability']: 94 # If all that happened was the SDK bundle was promoted in stability, 95 # change only that and re-write the manifest 96 mono_manifest_dirty = True 97 bundle[0]['stability'] = info['stability'] 98 99 # re-write the manifest here because there are no bundles to build but 100 # the manifest has changed 101 if mono_manifest_dirty and len(ret) == 0: 102 manifest_file = open(MONO_MANIFEST, 'w') 103 manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2)) 104 manifest_file.close() 105 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read', 106 MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST]) 107 108 return ret 109 110 def update_mono_sdk_json(infos): 111 '''Update the naclmono manifest with the newly built packages''' 112 if len(infos) == 0: 113 return 114 115 manifest_file = open(MONO_MANIFEST, 'r') 116 mono_manifest = json.loads(manifest_file.read()) 117 manifest_file.close() 118 119 for info in infos: 120 bundle = {} 121 bundle['name'] = info['naclmono_name'] 122 bundle['description'] = 'Mono for Native Client' 123 bundle['stability'] = info['stability'] 124 bundle['recommended'] = 'no' 125 bundle['version'] = 'experimental' 126 archive = {} 127 sha1_hash = hashlib.sha1() 128 f = open(info['naclmono_name'] + '.bz2', 'rb') 129 sha1_hash.update(f.read()) 130 archive['size'] = f.tell() 131 f.close() 132 archive['checksum'] = { 'sha1': sha1_hash.hexdigest() } 133 archive['host_os'] = 'all' 134 archive['url'] = ('https://commondatastorage.googleapis.com/' 135 'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2' 136 % (info['naclmono_name'], info['naclmono_rev'], 137 info['naclmono_name'])) 138 bundle['archives'] = [archive] 139 bundle['revision'] = int(info['naclmono_rev']) 140 bundle['sdk_revision'] = int(info['sdk_revision']) 141 142 # Insert this new bundle into the manifest, 143 # probably overwriting an existing bundle. 144 for key, value in mono_manifest.items(): 145 if key == 'bundles': 146 existing = filter(lambda b: b['name'] == info['naclmono_name'], value) 147 if len(existing) > 0: 148 loc = value.index(existing[0]) 149 value[loc] = bundle 150 else: 151 value.append(bundle) 152 153 # Write out the file locally, then upload to its known location. 154 manifest_file = open(MONO_MANIFEST, 'w') 155 manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2)) 156 manifest_file.close() 157 buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read', 158 MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST]) 159 160 161 def main(args): 162 args = args[1:] 163 164 # Delete global configs that would override the mono builders' configuration. 165 if 'AWS_CREDENTIAL_FILE' in os.environ: 166 del os.environ['AWS_CREDENTIAL_FILE'] 167 if 'BOTO_CONFIG' in os.environ: 168 del os.environ['BOTO_CONFIG'] 169 170 buildbot_revision = os.environ.get('BUILDBOT_REVISION', '') 171 buildername = os.environ.get('BUILDBOT_BUILDERNAME', '') 172 173 os.chdir(SCRIPT_DIR) 174 175 if buildername == 'linux-sdk-mono32': 176 assert buildbot_revision 177 sdk_revision = buildbot_revision.split(':')[0] 178 pepper_revision = build_version.ChromeMajorVersion() 179 build_and_upload_mono(sdk_revision, pepper_revision, None, 180 'trunk.' + sdk_revision, args) 181 elif buildername == 'linux-sdk-mono64': 182 infos = get_sdk_build_info() 183 for info in infos: 184 # This will put the file in naclmono_19/1/naclmono_19.bz2 for example. 185 upload_path = info['naclmono_name'] + '/' + info['naclmono_rev'] 186 build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'], 187 upload_path, args) 188 update_mono_sdk_json(infos) 189 190 191 192 if __name__ == '__main__': 193 sys.exit(main(sys.argv)) 194
