Home | History | Annotate | Download | only in server2 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import traceback
      6 
      7 from app_yaml_helper import AppYamlHelper
      8 from appengine_wrappers import IsDeadlineExceededError, logservice
      9 from branch_utility import BranchUtility
     10 from compiled_file_system import CompiledFileSystem
     11 from custom_logger import CustomLogger
     12 from data_source_registry import CreateDataSource
     13 from environment import GetAppVersion
     14 from file_system import IsFileSystemThrottledError
     15 from future import Future
     16 from gcs_file_system_provider import CloudStorageFileSystemProvider
     17 from github_file_system_provider import GithubFileSystemProvider
     18 from host_file_system_provider import HostFileSystemProvider
     19 from object_store_creator import ObjectStoreCreator
     20 from server_instance import ServerInstance
     21 from servlet import Servlet, Request, Response
     22 from timer import Timer, TimerClosure
     23 
     24 
     25 
     26 _log = CustomLogger('refresh')
     27 
     28 
     29 class RefreshServlet(Servlet):
     30   '''Servlet which refreshes a single data source.
     31   '''
     32   def __init__(self, request, delegate_for_test=None):
     33     Servlet.__init__(self, request)
     34     self._delegate = delegate_for_test or RefreshServlet.Delegate()
     35 
     36   class Delegate(object):
     37     '''RefreshServlet's runtime dependencies. Override for testing.
     38     '''
     39     def CreateBranchUtility(self, object_store_creator):
     40       return BranchUtility.Create(object_store_creator)
     41 
     42     def CreateHostFileSystemProvider(self,
     43                                      object_store_creator,
     44                                      pinned_commit=None):
     45       return HostFileSystemProvider(object_store_creator,
     46                                     pinned_commit=pinned_commit)
     47 
     48     def CreateGithubFileSystemProvider(self, object_store_creator):
     49       return GithubFileSystemProvider(object_store_creator)
     50 
     51     def CreateGCSFileSystemProvider(self, object_store_creator):
     52       return CloudStorageFileSystemProvider(object_store_creator)
     53 
     54     def GetAppVersion(self):
     55       return GetAppVersion()
     56 
     57   def Get(self):
     58     # Manually flush logs at the end of the run. However, sometimes
     59     # even that isn't enough, which is why in this file we use the
     60     # custom logger and make it flush the log every time its used.
     61     logservice.AUTOFLUSH_ENABLED = False
     62     try:
     63       return self._GetImpl()
     64     except BaseException:
     65       _log.error('Caught top-level exception! %s', traceback.format_exc())
     66     finally:
     67       logservice.flush()
     68 
     69   def _GetImpl(self):
     70     path = self._request.path.strip('/')
     71     parts = self._request.path.split('/', 1)
     72     source_name = parts[0]
     73     if len(parts) == 2:
     74       source_path = parts[1]
     75     else:
     76       source_path = None
     77 
     78     _log.info('starting refresh of %s DataSource %s' %
     79         (source_name, '' if source_path is None else '[%s]' % source_path))
     80 
     81     if 'commit' in self._request.arguments:
     82       commit = self._request.arguments['commit']
     83     else:
     84       _log.warning('No commit given; refreshing from master. '
     85                    'This is probably NOT what you want.')
     86       commit = None
     87 
     88     server_instance = self._CreateServerInstance(commit)
     89     success = True
     90     try:
     91       if source_name == 'platform_bundle':
     92         data_source = server_instance.platform_bundle
     93       elif source_name == 'content_providers':
     94         data_source = server_instance.content_providers
     95       else:
     96         data_source = CreateDataSource(source_name, server_instance)
     97 
     98       class_name = data_source.__class__.__name__
     99       refresh_future = data_source.Refresh(source_path)
    100       assert isinstance(refresh_future, Future), (
    101           '%s.Refresh() did not return a Future' % class_name)
    102       timer = Timer()
    103       try:
    104         refresh_future.Get()
    105       except Exception as e:
    106         _log.error('%s: error %s' % (class_name, traceback.format_exc()))
    107         success = False
    108         if IsFileSystemThrottledError(e):
    109           return Response.ThrottledError('Throttled')
    110         raise
    111       finally:
    112         _log.info('Refreshing %s took %s' %
    113             (class_name, timer.Stop().FormatElapsed()))
    114 
    115     except:
    116       success = False
    117       # This should never actually happen.
    118       _log.error('uncaught error: %s' % traceback.format_exc())
    119       raise
    120     finally:
    121       _log.info('finished (%s)', 'success' if success else 'FAILED')
    122       return (Response.Ok('Success') if success else
    123               Response.InternalError('Failure'))
    124 
    125   def _CreateServerInstance(self, commit):
    126     '''Creates a ServerInstance pinned to |commit|, or HEAD if None.
    127     NOTE: If passed None it's likely that during the cron run patches will be
    128     submitted at HEAD, which may change data underneath the cron run.
    129     '''
    130     object_store_creator = ObjectStoreCreator(start_empty=True)
    131     branch_utility = self._delegate.CreateBranchUtility(object_store_creator)
    132     host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
    133         object_store_creator, pinned_commit=commit)
    134     github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
    135         object_store_creator)
    136     gcs_file_system_provider = self._delegate.CreateGCSFileSystemProvider(
    137         object_store_creator)
    138     return ServerInstance(object_store_creator,
    139                           CompiledFileSystem.Factory(object_store_creator),
    140                           branch_utility,
    141                           host_file_system_provider,
    142                           github_file_system_provider,
    143                           gcs_file_system_provider)
    144