Home | History | Annotate | Download | only in page_sets 
      1 # Copyright 2013 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import os
      6 import re
      7 import sys
      8 
      9 
     10 def LoadSupport(input_api):
     11   if 'cloud_storage' not in globals():
     12     # Avoid leaking changes to global sys.path.
     13     _old_sys_path = sys.path
     14     try:
     15       telemetry_path = os.path.join(os.path.dirname(os.path.dirname(
     16           input_api.PresubmitLocalPath())), 'telemetry')
     17       sys.path = [telemetry_path] + sys.path
     18       from telemetry.page import cloud_storage
     19       globals()['cloud_storage'] = cloud_storage
     20     finally:
     21       sys.path = _old_sys_path
     22 
     23   return globals()['cloud_storage']
     24 
     25 
     26 def _GetFilesNotInCloud(input_api):
     27   """Searches for .sha1 files and uploads them to Cloud Storage.
     28 
     29   It validates all the hashes and skips upload if not necessary.
     30   """
     31   hash_paths = []
     32   for affected_file in input_api.AffectedFiles(include_deletes=False):
     33     hash_path = affected_file.AbsoluteLocalPath()
     34     _, extension = os.path.splitext(hash_path)
     35     if extension == '.sha1':
     36       hash_paths.append(hash_path)
     37   if not hash_paths:
     38     return []
     39 
     40   cloud_storage = LoadSupport(input_api)
     41 
     42   # Look in both buckets, in case the user uploaded the file manually. But this
     43   # script focuses on WPR archives, so it only uploads to the internal bucket.
     44   hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
     45   try:
     46     hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
     47   except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
     48     pass
     49 
     50   files = []
     51   for hash_path in hash_paths:
     52     file_hash = cloud_storage.ReadHash(hash_path)
     53     if file_hash not in hashes_in_cloud_storage:
     54       files.append((hash_path, file_hash))
     55 
     56   return files
     57 
     58 
     59 def _SyncFilesToCloud(input_api, output_api):
     60   """Searches for .sha1 files and uploads them to Cloud Storage.
     61 
     62   It validates all the hashes and skips upload if not necessary.
     63   """
     64 
     65   cloud_storage = LoadSupport(input_api)
     66 
     67   results = []
     68   for hash_path, file_hash in _GetFilesNotInCloud(input_api):
     69     file_path, _ = os.path.splitext(hash_path)
     70 
     71     if not re.match('^([A-Za-z0-9]{40})$', file_hash):
     72       results.append(output_api.PresubmitError(
     73           'Hash file does not contain a valid SHA-1 hash: %s' % hash_path))
     74       continue
     75     if not os.path.exists(file_path):
     76       results.append(output_api.PresubmitError(
     77           'Hash file exists, but file not found: %s' % hash_path))
     78       continue
     79     if cloud_storage.CalculateHash(file_path) != file_hash:
     80       results.append(output_api.PresubmitError(
     81           'Hash file does not match file\'s actual hash: %s' % hash_path))
     82       continue
     83 
     84     try:
     85       bucket_input = raw_input('Uploading to Cloud Storage: %s\n'
     86                                'Is this file [P]ublic or Google-[i]nternal?'
     87                                % file_path).lower()
     88       if 'public'.startswith(bucket_input):
     89         bucket = cloud_storage.PUBLIC_BUCKET
     90       elif ('internal'.startswith(bucket_input) or
     91             'google-internal'.startswith(bucket_input)):
     92         bucket = cloud_storage.INTERNAL_BUCKET
     93       else:
     94         results.append(output_api.PresubmitError(
     95             'Response was neither "public" nor "internal": %s' % bucket_input))
     96         return results
     97 
     98       cloud_storage.Insert(bucket, file_hash, file_path)
     99       results.append(output_api.PresubmitNotifyResult(
    100           'Uploaded file to Cloud Storage: %s' % file_path))
    101     except cloud_storage.CloudStorageError, e:
    102       results.append(output_api.PresubmitError(
    103           'Unable to upload to Cloud Storage: %s\n\n%s' % (file_path, e)))
    104 
    105   return results
    106 
    107 
    108 def _VerifyFilesInCloud(input_api, output_api):
    109   """Searches for .sha1 files and uploads them to Cloud Storage.
    110 
    111   It validates all the hashes and skips upload if not necessary.
    112   """
    113   results = []
    114   for hash_path, _ in _GetFilesNotInCloud(input_api):
    115     results.append(output_api.PresubmitError(
    116         'Attemping to commit hash file, but corresponding '
    117         'data file is not in Cloud Storage: %s' % hash_path))
    118   return results
    119 
    120 
    121 def _IsNewJsonPageSet(affected_file):
    122   return (affected_file.Action() == 'A' and
    123           'page_sets/data/' not in affected_file.AbsoluteLocalPath()
    124           and affected_file.AbsoluteLocalPath().endswith('.json'))
    125 
    126 
    127 def _GetNewJsonPageSets(input_api):
    128   return input_api.AffectedFiles(file_filter=_IsNewJsonPageSet)
    129 
    130 def CheckChangeOnUpload(input_api, output_api):
    131   results = _SyncFilesToCloud(input_api, output_api)
    132   return results
    133 
    134 
    135 def CheckChangeOnCommit(input_api, output_api):
    136   results = _VerifyFilesInCloud(input_api, output_api)
    137   return results
    138