1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import os 6 import re 7 import sys 8 9 10 def LoadSupport(input_api): 11 if 'cloud_storage' not in globals(): 12 # Avoid leaking changes to global sys.path. 13 _old_sys_path = sys.path 14 try: 15 telemetry_path = os.path.join(os.path.dirname(os.path.dirname( 16 input_api.PresubmitLocalPath())), 'telemetry') 17 sys.path = [telemetry_path] + sys.path 18 from telemetry.page import cloud_storage 19 globals()['cloud_storage'] = cloud_storage 20 finally: 21 sys.path = _old_sys_path 22 23 return globals()['cloud_storage'] 24 25 26 def _GetFilesNotInCloud(input_api): 27 """Searches for .sha1 files and uploads them to Cloud Storage. 28 29 It validates all the hashes and skips upload if not necessary. 30 """ 31 hash_paths = [] 32 for affected_file in input_api.AffectedFiles(include_deletes=False): 33 hash_path = affected_file.AbsoluteLocalPath() 34 _, extension = os.path.splitext(hash_path) 35 if extension == '.sha1': 36 hash_paths.append(hash_path) 37 if not hash_paths: 38 return [] 39 40 cloud_storage = LoadSupport(input_api) 41 42 # Look in both buckets, in case the user uploaded the file manually. But this 43 # script focuses on WPR archives, so it only uploads to the internal bucket. 44 hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET) 45 try: 46 hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET) 47 except (cloud_storage.PermissionError, cloud_storage.CredentialsError): 48 pass 49 50 files = [] 51 for hash_path in hash_paths: 52 file_hash = cloud_storage.ReadHash(hash_path) 53 if file_hash not in hashes_in_cloud_storage: 54 files.append((hash_path, file_hash)) 55 56 return files 57 58 59 def _SyncFilesToCloud(input_api, output_api): 60 """Searches for .sha1 files and uploads them to Cloud Storage. 61 62 It validates all the hashes and skips upload if not necessary. 63 """ 64 65 cloud_storage = LoadSupport(input_api) 66 67 results = [] 68 for hash_path, file_hash in _GetFilesNotInCloud(input_api): 69 file_path, _ = os.path.splitext(hash_path) 70 71 if not re.match('^([A-Za-z0-9]{40})$', file_hash): 72 results.append(output_api.PresubmitError( 73 'Hash file does not contain a valid SHA-1 hash: %s' % hash_path)) 74 continue 75 if not os.path.exists(file_path): 76 results.append(output_api.PresubmitError( 77 'Hash file exists, but file not found: %s' % hash_path)) 78 continue 79 if cloud_storage.CalculateHash(file_path) != file_hash: 80 results.append(output_api.PresubmitError( 81 'Hash file does not match file\'s actual hash: %s' % hash_path)) 82 continue 83 84 try: 85 bucket_input = raw_input('Uploading to Cloud Storage: %s\n' 86 'Is this file [P]ublic or Google-[i]nternal?' 87 % file_path).lower() 88 if 'public'.startswith(bucket_input): 89 bucket = cloud_storage.PUBLIC_BUCKET 90 elif ('internal'.startswith(bucket_input) or 91 'google-internal'.startswith(bucket_input)): 92 bucket = cloud_storage.INTERNAL_BUCKET 93 else: 94 results.append(output_api.PresubmitError( 95 'Response was neither "public" nor "internal": %s' % bucket_input)) 96 return results 97 98 cloud_storage.Insert(bucket, file_hash, file_path) 99 results.append(output_api.PresubmitNotifyResult( 100 'Uploaded file to Cloud Storage: %s' % file_path)) 101 except cloud_storage.CloudStorageError, e: 102 results.append(output_api.PresubmitError( 103 'Unable to upload to Cloud Storage: %s\n\n%s' % (file_path, e))) 104 105 return results 106 107 108 def _VerifyFilesInCloud(input_api, output_api): 109 """Searches for .sha1 files and uploads them to Cloud Storage. 110 111 It validates all the hashes and skips upload if not necessary. 112 """ 113 results = [] 114 for hash_path, _ in _GetFilesNotInCloud(input_api): 115 results.append(output_api.PresubmitError( 116 'Attemping to commit hash file, but corresponding ' 117 'data file is not in Cloud Storage: %s' % hash_path)) 118 return results 119 120 121 def _IsNewJsonPageSet(affected_file): 122 return (affected_file.Action() == 'A' and 123 'page_sets/data/' not in affected_file.AbsoluteLocalPath() 124 and affected_file.AbsoluteLocalPath().endswith('.json')) 125 126 127 def _GetNewJsonPageSets(input_api): 128 return input_api.AffectedFiles(file_filter=_IsNewJsonPageSet) 129 130 def CheckChangeOnUpload(input_api, output_api): 131 results = _SyncFilesToCloud(input_api, output_api) 132 return results 133 134 135 def CheckChangeOnCommit(input_api, output_api): 136 results = _VerifyFilesInCloud(input_api, output_api) 137 return results 138