1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import time 7 import traceback 8 9 from app_yaml_helper import AppYamlHelper 10 from appengine_wrappers import ( 11 GetAppVersion, DeadlineExceededError, IsDevServer, logservice) 12 from branch_utility import BranchUtility 13 from compiled_file_system import CompiledFileSystem 14 from empty_dir_file_system import EmptyDirFileSystem 15 from file_system_util import CreateURLsFromPaths 16 from github_file_system import GithubFileSystem 17 from host_file_system_creator import HostFileSystemCreator 18 from object_store_creator import ObjectStoreCreator 19 from render_servlet import RenderServlet 20 from server_instance import ServerInstance 21 from servlet import Servlet, Request, Response 22 import svn_constants 23 24 class _SingletonRenderServletDelegate(RenderServlet.Delegate): 25 def __init__(self, server_instance): 26 self._server_instance = server_instance 27 28 def CreateServerInstance(self): 29 return self._server_instance 30 31 class CronServlet(Servlet): 32 '''Servlet which runs a cron job. 33 ''' 34 def __init__(self, request, delegate_for_test=None): 35 Servlet.__init__(self, request) 36 self._delegate = delegate_for_test or CronServlet.Delegate() 37 38 class Delegate(object): 39 '''CronServlet's runtime dependencies. Override for testing. 40 ''' 41 def CreateBranchUtility(self, object_store_creator): 42 return BranchUtility.Create(object_store_creator) 43 44 def CreateHostFileSystemCreator(self, object_store_creator): 45 return HostFileSystemCreator(object_store_creator) 46 47 def CreateAppSamplesFileSystem(self, object_store_creator): 48 # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's 49 # not supported yet (see comment there). 50 return (EmptyDirFileSystem() if IsDevServer() else 51 GithubFileSystem.Create(object_store_creator)) 52 53 def GetAppVersion(self): 54 return GetAppVersion() 55 56 def Get(self): 57 # Crons often time out, and when they do *and* then eventually try to 58 # flush logs they die. Turn off autoflush and manually do so at the end. 59 logservice.AUTOFLUSH_ENABLED = False 60 try: 61 return self._GetImpl() 62 finally: 63 logservice.flush() 64 65 def _GetImpl(self): 66 # Cron strategy: 67 # 68 # Find all public template files and static files, and render them. Most of 69 # the time these won't have changed since the last cron run, so it's a 70 # little wasteful, but hopefully rendering is really fast (if it isn't we 71 # have a problem). 72 logging.info('cron: starting') 73 74 # This is returned every time RenderServlet wants to create a new 75 # ServerInstance. 76 server_instance = self._GetSafeServerInstance() 77 78 def get_via_render_servlet(path): 79 request = Request(path, self._request.host, self._request.headers) 80 delegate = _SingletonRenderServletDelegate(server_instance) 81 return RenderServlet(request, delegate).Get() 82 83 def run_cron_for_dir(d, path_prefix=''): 84 success = True 85 start_time = time.time() 86 files = dict( 87 CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix)) 88 logging.info('cron: rendering %s files from %s...' % (len(files), d)) 89 try: 90 for i, path in enumerate(files): 91 error = None 92 try: 93 response = get_via_render_servlet(path) 94 if response.status != 200: 95 error = 'Got %s response' % response.status 96 except DeadlineExceededError: 97 logging.error( 98 'cron: deadline exceeded rendering %s (%s of %s): %s' % ( 99 path, i + 1, len(files), traceback.format_exc())) 100 raise 101 except error: 102 pass 103 if error: 104 logging.error('cron: error rendering %s: %s' % (path, error)) 105 success = False 106 finally: 107 logging.info('cron: rendering %s files from %s took %s seconds' % ( 108 len(files), d, time.time() - start_time)) 109 return success 110 111 success = True 112 try: 113 # Render all of the publicly accessible files. 114 cron_runs = [ 115 # Note: rendering the public templates will pull in all of the private 116 # templates. 117 (svn_constants.PUBLIC_TEMPLATE_PATH, ''), 118 # Note: rendering the public templates will have pulled in the .js 119 # and manifest.json files (for listing examples on the API reference 120 # pages), but there are still images, CSS, etc. 121 (svn_constants.STATIC_PATH, 'static/'), 122 ] 123 if not IsDevServer(): 124 cron_runs.append( 125 (svn_constants.EXAMPLES_PATH, 'extensions/examples/')) 126 127 # Note: don't try to short circuit any of this stuff. We want to run 128 # the cron for all the directories regardless of intermediate 129 # failures. 130 for path, path_prefix in cron_runs: 131 success = run_cron_for_dir(path, path_prefix=path_prefix) and success 132 133 # TODO(kalman): Generic way for classes to request cron access. The next 134 # two special cases are ugly. It would potentially greatly speed up cron 135 # runs, too. 136 137 # Extension examples have zip files too. Well, so do apps, but the app 138 # file system doesn't get the Offline treatment so they don't need cron. 139 if not IsDevServer(): 140 manifest_json = 'manifest.json' 141 example_zips = [] 142 for root, _, files in server_instance.host_file_system.Walk( 143 svn_constants.EXAMPLES_PATH): 144 example_zips.extend( 145 root + '.zip' for name in files if name == manifest_json) 146 logging.info('cron: rendering %s example zips...' % len(example_zips)) 147 start_time = time.time() 148 try: 149 success = success and all( 150 get_via_render_servlet('extensions/examples/%s' % z).status == 200 151 for z in example_zips) 152 finally: 153 logging.info('cron: rendering %s example zips took %s seconds' % ( 154 len(example_zips), time.time() - start_time)) 155 156 except DeadlineExceededError: 157 success = False 158 159 logging.info('cron: running Redirector cron...') 160 server_instance.redirector.Cron() 161 162 logging.info('cron: finished (%s)' % ('success' if success else 'failure',)) 163 164 return (Response.Ok('Success') if success else 165 Response.InternalError('Failure')) 166 167 def _GetSafeServerInstance(self): 168 '''Returns a ServerInstance with a host file system at a safe revision, 169 meaning the last revision that the current running version of the server 170 existed. 171 ''' 172 delegate = self._delegate 173 server_instance_at_head = self._CreateServerInstance(None) 174 175 app_yaml_handler = AppYamlHelper( 176 svn_constants.APP_YAML_PATH, 177 server_instance_at_head.host_file_system, 178 server_instance_at_head.object_store_creator, 179 server_instance_at_head.host_file_system_creator) 180 181 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): 182 # TODO(kalman): return a new ServerInstance at an explicit revision in 183 # case the HEAD version changes underneath us. 184 return server_instance_at_head 185 186 # The version in app.yaml is greater than the currently running app's. 187 # The safe version is the one before it changed. 188 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( 189 delegate.GetAppVersion()) - 1 190 191 logging.info('cron: app version %s is out of date, safe is %s' % ( 192 delegate.GetAppVersion(), safe_revision)) 193 194 return self._CreateServerInstance(safe_revision) 195 196 def _CreateServerInstance(self, revision): 197 object_store_creator = ObjectStoreCreator(start_empty=True) 198 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) 199 host_file_system_creator = self._delegate.CreateHostFileSystemCreator( 200 object_store_creator) 201 host_file_system = host_file_system_creator.Create(revision=revision) 202 app_samples_file_system = self._delegate.CreateAppSamplesFileSystem( 203 object_store_creator) 204 compiled_host_fs_factory = CompiledFileSystem.Factory( 205 host_file_system, 206 object_store_creator) 207 return ServerInstance(object_store_creator, 208 host_file_system, 209 app_samples_file_system, 210 '', 211 compiled_host_fs_factory, 212 branch_utility, 213 host_file_system_creator) 214
