1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import time 30 import logging 31 import re 32 import urllib 33 import webapp2 34 35 from google.appengine.api import users 36 from google.appengine.ext.webapp import template 37 from google.appengine.ext import db 38 39 from model.jsonresults import JsonResults 40 from model.testfile import TestFile 41 42 PARAM_MASTER = "master" 43 PARAM_BUILDER = "builder" 44 PARAM_DIR = "dir" 45 PARAM_FILE = "file" 46 PARAM_NAME = "name" 47 PARAM_BEFORE = "before" 48 PARAM_NUM_FILES = "numfiles" 49 PARAM_KEY = "key" 50 PARAM_TEST_TYPE = "testtype" 51 PARAM_TEST_LIST_JSON = "testlistjson" 52 PARAM_CALLBACK = "callback" 53 54 55 def _replace_jsonp_callback(json, callback_name): 56 if callback_name and re.search(r"^[A-Za-z0-9_]+$", callback_name): 57 if re.search(r"^[A-Za-z0-9_]+[(]", json): 58 return re.sub(r"^[A-Za-z0-9_]+[(]", callback_name + "(", json) 59 return callback_name + "(" + json + ")" 60 61 return json 62 63 64 class DeleteFile(webapp2.RequestHandler): 65 """Delete test file for a given builder and name from datastore.""" 66 67 def get(self): 68 key = self.request.get(PARAM_KEY) 69 master = self.request.get(PARAM_MASTER) 70 builder = self.request.get(PARAM_BUILDER) 71 test_type = self.request.get(PARAM_TEST_TYPE) 72 name = self.request.get(PARAM_NAME) 73 num_files = self.request.get(PARAM_NUM_FILES) 74 before = self.request.get(PARAM_BEFORE) 75 76 logging.debug( 77 "Deleting File, master: %s, builder: %s, test_type: %s, name: %s, before: %s, key: %s.", 78 master, builder, test_type, name, before, key) 79 80 limit = int(num_files) if num_files else 1 81 num_deleted = TestFile.delete_file(key, master, builder, test_type, name, before, limit) 82 83 self.response.set_status(200) 84 self.response.out.write("Deleted %d files." % num_deleted) 85 86 87 class GetFile(webapp2.RequestHandler): 88 """Get file content or list of files for given builder and name.""" 89 90 def _get_file_list(self, master, builder, test_type, name, before, limit, callback_name=None): 91 """Get and display a list of files that matches builder and file name. 92 93 Args: 94 builder: builder name 95 test_type: type of the test 96 name: file name 97 """ 98 99 files = TestFile.get_files( 100 master, builder, test_type, name, before, load_data=False, limit=limit) 101 if not files: 102 logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.", 103 master, builder, test_type, name) 104 self.response.out.write("File not found") 105 return 106 107 template_values = { 108 "admin": users.is_current_user_admin(), 109 "master": master, 110 "builder": builder, 111 "test_type": test_type, 112 "name": name, 113 "files": files, 114 } 115 if callback_name: 116 json = template.render("templates/showfilelist.jsonp", template_values) 117 self._serve_json(_replace_jsonp_callback(json, callback_name), files[0].date) 118 return 119 self.response.out.write(template.render("templates/showfilelist.html", 120 template_values)) 121 122 def _get_file_content(self, master, builder, test_type, name): 123 """Return content of the file that matches builder and file name. 124 125 Args: 126 builder: builder name 127 test_type: type of the test 128 name: file name 129 """ 130 131 files = TestFile.get_files( 132 master, builder, test_type, name, load_data=True, limit=1) 133 if not files: 134 logging.info("File not found, master %s, builder: %s, test_type: %s, name: %s.", 135 master, builder, test_type, name) 136 return None, None 137 138 return files[0].data, files[0].date 139 140 def _get_file_content_from_key(self, key): 141 file = db.get(key) 142 143 if not file: 144 logging.info("File not found, key %s.", key) 145 return None 146 147 file.load_data() 148 return file.data, file.date 149 150 def _get_test_list_json(self, master, builder, test_type): 151 """Return json file with test name list only, do not include test 152 results and other non-test-data . 153 154 Args: 155 builder: builder name. 156 test_type: type of test results. 157 """ 158 159 json, date = self._get_file_content(master, builder, test_type, "results.json") 160 if not json: 161 return None 162 163 return JsonResults.get_test_list(builder, json), date 164 165 def _serve_json(self, json, modified_date): 166 if json: 167 if "If-Modified-Since" in self.request.headers: 168 old_date = self.request.headers["If-Modified-Since"] 169 if time.strptime(old_date, '%a, %d %b %Y %H:%M:%S %Z') == modified_date.utctimetuple(): 170 self.response.set_status(304) 171 return 172 173 # The appengine datetime objects are naive, so they lack a timezone. 174 # In practice, appengine seems to use GMT. 175 self.response.headers["Last-Modified"] = modified_date.strftime('%a, %d %b %Y %H:%M:%S') + ' GMT' 176 self.response.headers["Content-Type"] = "application/json" 177 self.response.headers["Access-Control-Allow-Origin"] = "*" 178 self.response.out.write(json) 179 else: 180 self.error(404) 181 182 def get(self): 183 key = self.request.get(PARAM_KEY) 184 master = self.request.get(PARAM_MASTER) 185 builder = self.request.get(PARAM_BUILDER) 186 test_type = self.request.get(PARAM_TEST_TYPE) 187 name = self.request.get(PARAM_NAME) 188 before = self.request.get(PARAM_BEFORE) 189 num_files = self.request.get(PARAM_NUM_FILES) 190 test_list_json = self.request.get(PARAM_TEST_LIST_JSON) 191 callback_name = self.request.get(PARAM_CALLBACK) 192 193 logging.debug( 194 "Getting files, master %s, builder: %s, test_type: %s, name: %s, before: %s.", 195 master, builder, test_type, name, before) 196 197 if key: 198 json, date = self._get_file_content_from_key(key) 199 elif test_list_json: 200 json, date = self._get_test_list_json(master, builder, test_type) 201 elif num_files or not master or not builder or not test_type or not name: 202 limit = int(num_files) if num_files else 100 203 self._get_file_list(master, builder, test_type, name, before, limit, callback_name) 204 return 205 else: 206 json, date = self._get_file_content(master, builder, test_type, name) 207 208 if json: 209 json = _replace_jsonp_callback(json, callback_name) 210 211 self._serve_json(json, date) 212 213 214 class Upload(webapp2.RequestHandler): 215 """Upload test results file to datastore.""" 216 217 def post(self): 218 file_params = self.request.POST.getall(PARAM_FILE) 219 if not file_params: 220 self.response.out.write("FAIL: missing upload file field.") 221 return 222 223 builder = self.request.get(PARAM_BUILDER) 224 if not builder: 225 self.response.out.write("FAIL: missing builder parameter.") 226 return 227 228 master = self.request.get(PARAM_MASTER) 229 test_type = self.request.get(PARAM_TEST_TYPE) 230 231 logging.debug( 232 "Processing upload request, master: %s, builder: %s, test_type: %s.", 233 master, builder, test_type) 234 235 # There are two possible types of each file_params in the request: 236 # one file item or a list of file items. 237 # Normalize file_params to a file item list. 238 files = [] 239 logging.debug("test: %s, type:%s", file_params, type(file_params)) 240 for item in file_params: 241 if not isinstance(item, list) and not isinstance(item, tuple): 242 item = [item] 243 files.extend(item) 244 245 errors = [] 246 final_status_code = 200 247 for file in files: 248 if file.filename == "incremental_results.json": 249 status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=False) 250 elif file.filename == "times_ms.json": 251 # We never look at historical times_ms.json files, so we can overwrite the existing one if it exists. 252 status_string, status_code = TestFile.overwrite_or_add_file(master, builder, test_type, file.filename, file.value) 253 else: 254 status_string, status_code = TestFile.add_file(master, builder, test_type, file.filename, file.value) 255 # FIXME: Upload full_results.json files for non-layout tests as well and stop supporting the 256 # incremental_results.json file format. 257 if status_code == 200 and file.filename == "full_results.json": 258 status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=True) 259 260 if status_code == 200: 261 logging.info(status_string) 262 else: 263 logging.error(status_string) 264 errors.append(status_string) 265 final_status_code = status_code 266 267 if errors: 268 messages = "FAIL: " + "; ".join(errors) 269 self.response.set_status(final_status_code, messages) 270 self.response.out.write(messages) 271 else: 272 self.response.set_status(200) 273 self.response.out.write("OK") 274 275 276 class UploadForm(webapp2.RequestHandler): 277 """Show a form so user can upload a file.""" 278 279 def get(self): 280 template_values = { 281 "upload_url": "/testfile/upload", 282 } 283 self.response.out.write(template.render("templates/uploadform.html", 284 template_values)) 285
