1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import hashlib 6 import copy 7 import logging 8 import os 9 import subprocess 10 import sys 11 import urlparse 12 import urllib2 13 14 import command_common 15 import download 16 from sdk_update_common import Error 17 import sdk_update_common 18 19 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 20 PARENT_DIR = os.path.dirname(SCRIPT_DIR) 21 sys.path.append(PARENT_DIR) 22 try: 23 import cygtar 24 except ImportError: 25 # Try to find this in the Chromium repo. 26 CHROME_SRC_DIR = os.path.abspath( 27 os.path.join(PARENT_DIR, '..', '..', '..', '..')) 28 sys.path.append(os.path.join(CHROME_SRC_DIR, 'native_client', 'build')) 29 import cygtar 30 31 32 RECOMMENDED = 'recommended' 33 SDK_TOOLS = 'sdk_tools' 34 HTTP_CONTENT_LENGTH = 'Content-Length' # HTTP Header field for content length 35 DEFAULT_CACHE_SIZE = 512 * 1024 * 1024 # 1/2 Gb cache by default 36 37 38 class UpdateDelegate(object): 39 def BundleDirectoryExists(self, bundle_name): 40 raise NotImplementedError() 41 42 def DownloadToFile(self, url, dest_filename): 43 raise NotImplementedError() 44 45 def ExtractArchives(self, archives, extract_dir, rename_from_dir, 46 rename_to_dir): 47 raise NotImplementedError() 48 49 50 class RealUpdateDelegate(UpdateDelegate): 51 def __init__(self, user_data_dir, install_dir, cfg): 52 UpdateDelegate.__init__(self) 53 self.archive_cache = os.path.join(user_data_dir, 'archives') 54 self.install_dir = install_dir 55 self.cache_max = getattr(cfg, 'cache_max', DEFAULT_CACHE_SIZE) 56 57 def BundleDirectoryExists(self, bundle_name): 58 bundle_path = os.path.join(self.install_dir, bundle_name) 59 return os.path.isdir(bundle_path) 60 61 def VerifyDownload(self, filename, archive): 62 """Verify that a local filename in the cache matches the given 63 online archive. 64 65 Returns True if both size and sha1 match, False otherwise. 66 """ 67 filename = os.path.join(self.archive_cache, filename) 68 if not os.path.exists(filename): 69 logging.info('File does not exist: %s.' % filename) 70 return False 71 size = os.path.getsize(filename) 72 if size != archive.size: 73 logging.info('File size does not match (%d vs %d): %s.' % (size, 74 archive.size, filename)) 75 return False 76 sha1_hash = hashlib.sha1() 77 with open(filename) as f: 78 sha1_hash.update(f.read()) 79 if sha1_hash.hexdigest() != archive.GetChecksum(): 80 logging.info('File hash does not match: %s.' % filename) 81 return False 82 return True 83 84 def BytesUsedInCache(self): 85 """Determine number of bytes currently be in local archive cache.""" 86 total = 0 87 for root, _, files in os.walk(self.archive_cache): 88 for filename in files: 89 total += os.path.getsize(os.path.join(root, filename)) 90 return total 91 92 def CleanupCache(self): 93 """Remove archives from the local filesystem cache until the 94 total size is below cache_max. 95 96 This is done my deleting the oldest archive files until the 97 condition is satisfied. If cache_max is zero then the entire 98 cache will be removed. 99 """ 100 used = self.BytesUsedInCache() 101 logging.info('Cache usage: %d / %d' % (used, self.cache_max)) 102 if used <= self.cache_max: 103 return 104 clean_bytes = used - self.cache_max 105 106 logging.info('Clearing %d bytes in archive cache' % clean_bytes) 107 file_timestamps = [] 108 for root, _, files in os.walk(self.archive_cache): 109 for filename in files: 110 fullname = os.path.join(root, filename) 111 file_timestamps.append((os.path.getmtime(fullname), fullname)) 112 113 file_timestamps.sort() 114 while clean_bytes > 0: 115 assert(file_timestamps) 116 filename_to_remove = file_timestamps[0][1] 117 clean_bytes -= os.path.getsize(filename_to_remove) 118 logging.info('Removing from cache: %s' % filename_to_remove) 119 os.remove(filename_to_remove) 120 # Also remove resulting empty parent directory structure 121 while True: 122 filename_to_remove = os.path.dirname(filename_to_remove) 123 if not os.listdir(filename_to_remove): 124 os.rmdir(filename_to_remove) 125 else: 126 break 127 file_timestamps = file_timestamps[1:] 128 129 def DownloadToFile(self, url, dest_filename): 130 dest_path = os.path.join(self.archive_cache, dest_filename) 131 sdk_update_common.MakeDirs(os.path.dirname(dest_path)) 132 133 out_stream = None 134 url_stream = None 135 try: 136 out_stream = open(dest_path, 'wb') 137 url_stream = download.UrlOpen(url) 138 content_length = int(url_stream.info()[HTTP_CONTENT_LENGTH]) 139 progress = download.MakeProgressFunction(content_length) 140 sha1, size = download.DownloadAndComputeHash(url_stream, out_stream, 141 progress) 142 return sha1, size 143 except urllib2.URLError as e: 144 raise Error('Unable to read from URL "%s".\n %s' % (url, e)) 145 except IOError as e: 146 raise Error('Unable to write to file "%s".\n %s' % (dest_filename, e)) 147 finally: 148 if url_stream: 149 url_stream.close() 150 if out_stream: 151 out_stream.close() 152 153 def ExtractArchives(self, archives, extract_dir, rename_from_dir, 154 rename_to_dir): 155 tar_file = None 156 157 extract_path = os.path.join(self.install_dir, extract_dir) 158 rename_from_path = os.path.join(self.install_dir, rename_from_dir) 159 rename_to_path = os.path.join(self.install_dir, rename_to_dir) 160 161 # Extract to extract_dir, usually "<bundle name>_update". 162 # This way if the extraction fails, we haven't blown away the old bundle 163 # (if it exists). 164 sdk_update_common.RemoveDir(extract_path) 165 sdk_update_common.MakeDirs(extract_path) 166 curpath = os.getcwd() 167 tar_file = None 168 169 try: 170 try: 171 logging.info('Changing the directory to %s' % (extract_path,)) 172 os.chdir(extract_path) 173 except Exception as e: 174 raise Error('Unable to chdir into "%s".\n %s' % (extract_path, e)) 175 176 for i, archive in enumerate(archives): 177 archive_path = os.path.join(self.archive_cache, archive) 178 179 if len(archives) > 1: 180 print '(file %d/%d - "%s")' % ( 181 i + 1, len(archives), os.path.basename(archive_path)) 182 logging.info('Extracting to %s' % (extract_path,)) 183 184 if sys.platform == 'win32': 185 try: 186 logging.info('Opening file %s (%d/%d).' % (archive_path, i + 1, 187 len(archives))) 188 try: 189 tar_file = cygtar.CygTar(archive_path, 'r', verbose=True) 190 except Exception as e: 191 raise Error("Can't open archive '%s'.\n %s" % (archive_path, e)) 192 193 tar_file.Extract() 194 finally: 195 if tar_file: 196 tar_file.Close() 197 else: 198 try: 199 subprocess.check_call(['tar', 'xf', archive_path]) 200 except subprocess.CalledProcessError: 201 raise Error('Error extracting archive: %s' % archive_path) 202 203 logging.info('Changing the directory to %s' % (curpath,)) 204 os.chdir(curpath) 205 206 logging.info('Renaming %s->%s' % (rename_from_path, rename_to_path)) 207 sdk_update_common.RenameDir(rename_from_path, rename_to_path) 208 finally: 209 # Change the directory back so we can remove the update directory. 210 os.chdir(curpath) 211 212 # Clean up the ..._update directory. 213 try: 214 sdk_update_common.RemoveDir(extract_path) 215 except Exception as e: 216 logging.error('Failed to remove directory \"%s\". %s' % ( 217 extract_path, e)) 218 219 220 def Update(delegate, remote_manifest, local_manifest, bundle_names, force): 221 valid_bundles = set([bundle.name for bundle in remote_manifest.GetBundles()]) 222 requested_bundles = _GetRequestedBundleNamesFromArgs(remote_manifest, 223 bundle_names) 224 invalid_bundles = requested_bundles - valid_bundles 225 if invalid_bundles: 226 logging.warn('Ignoring unknown bundle(s): %s' % ( 227 ', '.join(invalid_bundles))) 228 requested_bundles -= invalid_bundles 229 230 if SDK_TOOLS in requested_bundles: 231 logging.warn('Updating sdk_tools happens automatically. ' 232 'Ignoring manual update request.') 233 requested_bundles.discard(SDK_TOOLS) 234 235 if requested_bundles: 236 for bundle_name in requested_bundles: 237 logging.info('Trying to update %s' % (bundle_name,)) 238 UpdateBundleIfNeeded(delegate, remote_manifest, local_manifest, 239 bundle_name, force) 240 else: 241 logging.warn('No bundles to update.') 242 243 244 def Reinstall(delegate, local_manifest, bundle_names): 245 valid_bundles, invalid_bundles = \ 246 command_common.GetValidBundles(local_manifest, bundle_names) 247 if invalid_bundles: 248 logging.warn('Unknown bundle(s): %s\n' % (', '.join(invalid_bundles))) 249 250 if not valid_bundles: 251 logging.warn('No bundles to reinstall.') 252 return 253 254 for bundle_name in valid_bundles: 255 bundle = copy.deepcopy(local_manifest.GetBundle(bundle_name)) 256 257 # HACK(binji): There was a bug where we'd merge the bundles from the old 258 # archive and the new archive when updating. As a result, some users may 259 # have a cache manifest that contains duplicate archives. Remove all 260 # archives with the same basename except for the most recent. 261 # Because the archives are added to a list, we know the most recent is at 262 # the end. 263 archives = {} 264 for archive in bundle.GetArchives(): 265 url = archive.url 266 path = urlparse.urlparse(url)[2] 267 basename = os.path.basename(path) 268 archives[basename] = archive 269 270 # Update the bundle with these new archives. 271 bundle.RemoveAllArchives() 272 for _, archive in archives.iteritems(): 273 bundle.AddArchive(archive) 274 275 _UpdateBundle(delegate, bundle, local_manifest) 276 277 278 def UpdateBundleIfNeeded(delegate, remote_manifest, local_manifest, 279 bundle_name, force): 280 bundle = remote_manifest.GetBundle(bundle_name) 281 if bundle: 282 if _BundleNeedsUpdate(delegate, local_manifest, bundle): 283 # TODO(binji): It would be nicer to detect whether the user has any 284 # modifications to the bundle. If not, we could update with impunity. 285 if not force and delegate.BundleDirectoryExists(bundle_name): 286 print ('%s already exists, but has an update available.\n' 287 'Run update with the --force option to overwrite the ' 288 'existing directory.\nWarning: This will overwrite any ' 289 'modifications you have made within this directory.' 290 % (bundle_name,)) 291 return 292 293 _UpdateBundle(delegate, bundle, local_manifest) 294 else: 295 print '%s is already up-to-date.' % (bundle.name,) 296 else: 297 logging.error('Bundle %s does not exist.' % (bundle_name,)) 298 299 300 def _GetRequestedBundleNamesFromArgs(remote_manifest, requested_bundles): 301 requested_bundles = set(requested_bundles) 302 if RECOMMENDED in requested_bundles: 303 requested_bundles.discard(RECOMMENDED) 304 requested_bundles |= set(_GetRecommendedBundleNames(remote_manifest)) 305 306 return requested_bundles 307 308 309 def _GetRecommendedBundleNames(remote_manifest): 310 result = [] 311 for bundle in remote_manifest.GetBundles(): 312 if bundle.recommended == 'yes' and bundle.name != SDK_TOOLS: 313 result.append(bundle.name) 314 return result 315 316 317 def _BundleNeedsUpdate(delegate, local_manifest, bundle): 318 # Always update the bundle if the directory doesn't exist; 319 # the user may have deleted it. 320 if not delegate.BundleDirectoryExists(bundle.name): 321 return True 322 323 return local_manifest.BundleNeedsUpdate(bundle) 324 325 326 def _UpdateBundle(delegate, bundle, local_manifest): 327 archives = bundle.GetHostOSArchives() 328 if not archives: 329 logging.warn('Bundle %s does not exist for this platform.' % (bundle.name,)) 330 return 331 332 archive_filenames = [] 333 334 shown_banner = False 335 for i, archive in enumerate(archives): 336 archive_filename = _GetFilenameFromURL(archive.url) 337 archive_filename = os.path.join(bundle.name, archive_filename) 338 339 if not delegate.VerifyDownload(archive_filename, archive): 340 if not shown_banner: 341 shown_banner = True 342 print 'Downloading bundle %s' % (bundle.name,) 343 if len(archives) > 1: 344 print '(file %d/%d - "%s")' % ( 345 i + 1, len(archives), os.path.basename(archive.url)) 346 sha1, size = delegate.DownloadToFile(archive.url, archive_filename) 347 _ValidateArchive(archive, sha1, size) 348 349 archive_filenames.append(archive_filename) 350 351 print 'Updating bundle %s to version %s, revision %s' % ( 352 bundle.name, bundle.version, bundle.revision) 353 extract_dir = bundle.name + '_update' 354 355 repath_dir = bundle.get('repath', None) 356 if repath_dir: 357 # If repath is specified: 358 # The files are extracted to nacl_sdk/<bundle.name>_update/<repath>/... 359 # The destination directory is nacl_sdk/<bundle.name>/... 360 rename_from_dir = os.path.join(extract_dir, repath_dir) 361 else: 362 # If no repath is specified: 363 # The files are extracted to nacl_sdk/<bundle.name>_update/... 364 # The destination directory is nacl_sdk/<bundle.name>/... 365 rename_from_dir = extract_dir 366 367 rename_to_dir = bundle.name 368 369 delegate.ExtractArchives(archive_filenames, extract_dir, rename_from_dir, 370 rename_to_dir) 371 372 logging.info('Updating local manifest to include bundle %s' % (bundle.name)) 373 local_manifest.RemoveBundle(bundle.name) 374 local_manifest.SetBundle(bundle) 375 delegate.CleanupCache() 376 377 378 def _GetFilenameFromURL(url): 379 path = urlparse.urlparse(url)[2] 380 return os.path.basename(path) 381 382 383 def _ValidateArchive(archive, actual_sha1, actual_size): 384 if actual_size != archive.size: 385 raise Error('Size mismatch on "%s". Expected %s but got %s bytes' % ( 386 archive.name, archive.size, actual_size)) 387 if actual_sha1 != archive.GetChecksum(): 388 raise Error('SHA1 checksum mismatch on "%s". Expected %s but got %s' % ( 389 archive.name, archive.GetChecksum(), actual_sha1)) 390