Home | History | Annotate | Download | only in page_sets 
      1 # Copyright 2013 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import os
      6 import re
      7 import sys
      8 
      9 
     10 def LoadSupport(input_api):
     11   if 'cloud_storage' not in globals():
     12     # Avoid leaking changes to global sys.path.
     13     _old_sys_path = sys.path
     14     try:
     15       telemetry_path = os.path.join(os.path.dirname(os.path.dirname(
     16           input_api.PresubmitLocalPath())), 'telemetry')
     17       sys.path = [telemetry_path] + sys.path
     18       from telemetry.util import cloud_storage
     19       globals()['cloud_storage'] = cloud_storage
     20     finally:
     21       sys.path = _old_sys_path
     22 
     23   return globals()['cloud_storage']
     24 
     25 
     26 def _GetFilesNotInCloud(input_api):
     27   """Searches for .sha1 files and uploads them to Cloud Storage.
     28 
     29   It validates all the hashes and skips upload if not necessary.
     30   """
     31   hash_paths = []
     32   for affected_file in input_api.AffectedFiles(include_deletes=False):
     33     hash_path = affected_file.AbsoluteLocalPath()
     34     _, extension = os.path.splitext(hash_path)
     35     if extension == '.sha1':
     36       hash_paths.append(hash_path)
     37   if not hash_paths:
     38     return []
     39 
     40   cloud_storage = LoadSupport(input_api)
     41 
     42   # Look in both buckets, in case the user uploaded the file manually. But this
     43   # script focuses on WPR archives, so it only uploads to the internal bucket.
     44   hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
     45   try:
     46     hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
     47   except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
     48     pass
     49 
     50   files = []
     51   for hash_path in hash_paths:
     52     file_hash = cloud_storage.ReadHash(hash_path)
     53     if file_hash not in hashes_in_cloud_storage:
     54       files.append((hash_path, file_hash))
     55 
     56   return files
     57 
     58 
     59 def _SyncFilesToCloud(input_api, output_api):
     60   """Searches for .sha1 files and uploads them to Cloud Storage.
     61 
     62   It validates all the hashes and skips upload if not necessary.
     63   """
     64 
     65   cloud_storage = LoadSupport(input_api)
     66 
     67   results = []
     68   for hash_path, file_hash in _GetFilesNotInCloud(input_api):
     69     file_path, _ = os.path.splitext(hash_path)
     70 
     71     if not re.match('^([A-Za-z0-9]{40})$', file_hash):
     72       results.append(output_api.PresubmitError(
     73           'Hash file does not contain a valid SHA-1 hash: %s' % hash_path))
     74       continue
     75     if not os.path.exists(file_path):
     76       results.append(output_api.PresubmitError(
     77           'Hash file exists, but file not found: %s' % hash_path))
     78       continue
     79     if cloud_storage.CalculateHash(file_path) != file_hash:
     80       results.append(output_api.PresubmitError(
     81           'Hash file does not match file\'s actual hash: %s' % hash_path))
     82       continue
     83 
     84     try:
     85       bucket_aliases_string = ', '.join(cloud_storage.BUCKET_ALIASES)
     86       bucket_input = raw_input(
     87           'Uploading to Cloud Storage: %s\n'
     88           'Which bucket should this go in? (%s) '
     89           % (file_path, bucket_aliases_string)).lower()
     90       bucket = cloud_storage.BUCKET_ALIASES.get(bucket_input, None)
     91       if not bucket:
     92         results.append(output_api.PresubmitError(
     93             '"%s" was not one of %s' % (bucket_input, bucket_aliases_string)))
     94         return results
     95 
     96       cloud_storage.Insert(bucket, file_hash, file_path)
     97       results.append(output_api.PresubmitNotifyResult(
     98           'Uploaded file to Cloud Storage: %s' % file_path))
     99     except cloud_storage.CloudStorageError, e:
    100       results.append(output_api.PresubmitError(
    101           'Unable to upload to Cloud Storage: %s\n\n%s' % (file_path, e)))
    102 
    103   return results
    104 
    105 
    106 def _VerifyFilesInCloud(input_api, output_api):
    107   """Searches for .sha1 files and uploads them to Cloud Storage.
    108 
    109   It validates all the hashes and skips upload if not necessary.
    110   """
    111   results = []
    112   for hash_path, _ in _GetFilesNotInCloud(input_api):
    113     results.append(output_api.PresubmitError(
    114         'Attemping to commit hash file, but corresponding '
    115         'data file is not in Cloud Storage: %s' % hash_path))
    116   return results
    117 
    118 
    119 def _IsNewJsonPageSet(affected_file):
    120   return (affected_file.Action() == 'A' and
    121           'page_sets/data/' not in affected_file.AbsoluteLocalPath()
    122           and affected_file.AbsoluteLocalPath().endswith('.json'))
    123 
    124 
    125 def _GetNewJsonPageSets(input_api):
    126   return input_api.AffectedFiles(file_filter=_IsNewJsonPageSet)
    127 
    128 def CheckChangeOnUpload(input_api, output_api):
    129   results = _SyncFilesToCloud(input_api, output_api)
    130   return results
    131 
    132 
    133 def CheckChangeOnCommit(input_api, output_api):
    134   results = _VerifyFilesInCloud(input_api, output_api)
    135   return results
    136