1 #!/usr/bin/env python 2 # Copyright 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # Run build_server so that files needed by tests are copied to the local 7 # third_party directory. 8 import build_server 9 build_server.main() 10 11 import json 12 import optparse 13 import os 14 import posixpath 15 import sys 16 import time 17 import unittest 18 19 from appengine_wrappers import SetTaskRunnerForTest 20 from branch_utility import BranchUtility 21 from chroot_file_system import ChrootFileSystem 22 from extensions_paths import ( 23 CONTENT_PROVIDERS, CHROME_EXTENSIONS, PUBLIC_TEMPLATES) 24 from fake_fetchers import ConfigureFakeFetchers 25 from special_paths import SITE_VERIFICATION_FILE 26 from handler import Handler 27 from link_error_detector import LinkErrorDetector, StringifyBrokenLinks 28 from local_file_system import LocalFileSystem 29 from local_renderer import LocalRenderer 30 from path_util import AssertIsValid 31 from servlet import Request 32 from third_party.json_schema_compiler import json_parse 33 from test_util import ( 34 ChromiumPath, DisableLogging, EnableLogging, ReadFile, Server2Path) 35 36 37 # Arguments set up if __main__ specifies them. 38 _EXPLICIT_TEST_FILES = None 39 _REBASE = False 40 _VERBOSE = False 41 42 43 def _ToPosixPath(os_path): 44 return os_path.replace(os.sep, '/') 45 46 47 def _FilterHidden(paths): 48 '''Returns a list of the non-hidden paths from |paths|. 49 ''' 50 # Hidden files start with a '.' but paths like './foo' and '../foo' are not 51 # hidden. 52 return [path for path in paths if (not path.startswith('.')) or 53 path.startswith('./') or 54 path.startswith('../')] 55 56 57 def _GetPublicFiles(): 58 '''Gets all public file paths mapped to their contents. 59 ''' 60 def walk(path, prefix=''): 61 path = ChromiumPath(path) 62 public_files = {} 63 for root, dirs, files in os.walk(path, topdown=True): 64 relative_root = root[len(path):].lstrip(os.path.sep) 65 dirs[:] = _FilterHidden(dirs) 66 for filename in _FilterHidden(files): 67 with open(os.path.join(root, filename), 'r') as f: 68 request_path = posixpath.join(prefix, relative_root, filename) 69 public_files[request_path] = f.read() 70 return public_files 71 72 # Public file locations are defined in content_providers.json, sort of. Epic 73 # hack to pull them out; list all the files from the directories that 74 # Chromium content providers ask for. 75 public_files = {} 76 content_providers = json_parse.Parse(ReadFile(CONTENT_PROVIDERS)) 77 for content_provider in content_providers.itervalues(): 78 if 'chromium' in content_provider: 79 public_files.update(walk(content_provider['chromium']['dir'], 80 prefix=content_provider['serveFrom'])) 81 return public_files 82 83 84 class IntegrationTest(unittest.TestCase): 85 def setUp(self): 86 ConfigureFakeFetchers() 87 88 @EnableLogging('info') 89 def testCronAndPublicFiles(self): 90 '''Runs cron then requests every public file. Cron needs to be run first 91 because the public file requests are offline. 92 ''' 93 if _EXPLICIT_TEST_FILES is not None: 94 return 95 96 97 def task_runner(url, commit=None): 98 arguments = { 'commit': commit } if commit else {} 99 Handler(Request.ForTest(url, arguments=arguments)).Get() 100 101 SetTaskRunnerForTest(task_runner) 102 103 print('Running cron...') 104 start_time = time.time() 105 try: 106 response = Handler(Request.ForTest('/_cron')).Get() 107 if response: 108 self.assertEqual(200, response.status) 109 self.assertEqual('Success', response.content.ToString()) 110 else: 111 self.fail('No response for _cron') 112 finally: 113 print('Took %s seconds' % (time.time() - start_time)) 114 115 # TODO(kalman): Re-enable this, but it takes about an hour at the moment, 116 # presumably because every page now has a lot of links on it from the 117 # topnav. 118 119 #print("Checking for broken links...") 120 #start_time = time.time() 121 #link_error_detector = LinkErrorDetector( 122 # # TODO(kalman): Use of ChrootFileSystem here indicates a hack. Fix. 123 # ChrootFileSystem(LocalFileSystem.Create(), CHROME_EXTENSIONS), 124 # lambda path: Handler(Request.ForTest(path)).Get(), 125 # 'templates/public', 126 # ('extensions/index.html', 'apps/about_apps.html')) 127 128 #broken_links = link_error_detector.GetBrokenLinks() 129 #if broken_links: 130 # print('Found %d broken links.' % ( 131 # len(broken_links))) 132 # if _VERBOSE: 133 # print(StringifyBrokenLinks(broken_links)) 134 135 #broken_links_set = set(broken_links) 136 137 #known_broken_links_path = os.path.join( 138 # Server2Path('known_broken_links.json')) 139 #try: 140 # with open(known_broken_links_path, 'r') as f: 141 # # The JSON file converts tuples and sets into lists, and for this 142 # # set union/difference logic they need to be converted back. 143 # known_broken_links = set(tuple(item) for item in json.load(f)) 144 #except IOError: 145 # known_broken_links = set() 146 147 #newly_broken_links = broken_links_set - known_broken_links 148 #fixed_links = known_broken_links - broken_links_set 149 150 #print('Took %s seconds.' % (time.time() - start_time)) 151 152 #print('Searching for orphaned pages...') 153 #start_time = time.time() 154 #orphaned_pages = link_error_detector.GetOrphanedPages() 155 #if orphaned_pages: 156 # # TODO(jshumway): Test should fail when orphaned pages are detected. 157 # print('Found %d orphaned pages:' % len(orphaned_pages)) 158 # for page in orphaned_pages: 159 # print(page) 160 #print('Took %s seconds.' % (time.time() - start_time)) 161 162 public_files = _GetPublicFiles() 163 164 print('Rendering %s public files...' % len(public_files.keys())) 165 start_time = time.time() 166 try: 167 for path, content in public_files.iteritems(): 168 AssertIsValid(path) 169 if path.endswith('redirects.json'): 170 continue 171 172 # The non-example html and md files are served without their file 173 # extensions. 174 path_without_ext, ext = posixpath.splitext(path) 175 if (ext in ('.html', '.md') and 176 '/examples/' not in path and 177 path != SITE_VERIFICATION_FILE): 178 path = path_without_ext 179 180 def check_result(response): 181 self.assertEqual(200, response.status, 182 'Got %s when rendering %s' % (response.status, path)) 183 184 # This is reaaaaally rough since usually these will be tiny templates 185 # that render large files. At least it'll catch zero-length responses. 186 self.assertTrue(len(response.content) >= len(content), 187 'Rendered content length was %s vs template content length %s ' 188 'when rendering %s' % (len(response.content), len(content), path)) 189 190 check_result(Handler(Request.ForTest(path)).Get()) 191 192 if path.startswith(('apps/', 'extensions/')): 193 # Make sure that adding the .html will temporarily redirect to 194 # the path without the .html for APIs and articles. 195 if '/examples/' not in path: 196 redirect_response = Handler(Request.ForTest(path + '.html')).Get() 197 self.assertEqual( 198 ('/' + path, False), redirect_response.GetRedirect(), 199 '%s.html did not (temporarily) redirect to %s (status %s)' % 200 (path, path, redirect_response.status)) 201 202 # Make sure including a channel will permanently redirect to the same 203 # path without a channel. 204 for channel in BranchUtility.GetAllChannelNames(): 205 redirect_response = Handler( 206 Request.ForTest(posixpath.join(channel, path))).Get() 207 self.assertEqual( 208 ('/' + path, True), 209 redirect_response.GetRedirect(), 210 '%s/%s did not (permanently) redirect to %s (status %s)' % 211 (channel, path, path, redirect_response.status)) 212 213 # Samples are internationalized, test some locales. 214 if path.endswith('/samples'): 215 for lang in ('en-US', 'es', 'ar'): 216 check_result(Handler(Request.ForTest( 217 path, 218 headers={'Accept-Language': '%s;q=0.8' % lang})).Get()) 219 finally: 220 print('Took %s seconds' % (time.time() - start_time)) 221 222 #if _REBASE: 223 # print('Rebasing broken links with %s newly broken and %s fixed links.' % 224 # (len(newly_broken_links), len(fixed_links))) 225 # with open(known_broken_links_path, 'w') as f: 226 # json.dump(broken_links, f, 227 # indent=2, separators=(',', ': '), sort_keys=True) 228 #else: 229 # if fixed_links or newly_broken_links: 230 # print('**********************************************\n' 231 # 'CHANGE DETECTED IN BROKEN LINKS WITHOUT REBASE\n' 232 # '**********************************************') 233 # print('Found %s broken links, and some have changed. ' 234 # 'If this is acceptable or expected then run %s with the --rebase ' 235 # 'option.' % (len(broken_links), os.path.split(__file__)[-1])) 236 # elif broken_links: 237 # print('%s existing broken links' % len(broken_links)) 238 # if fixed_links: 239 # print('%s broken links have been fixed:' % len(fixed_links)) 240 # print(StringifyBrokenLinks(fixed_links)) 241 # if newly_broken_links: 242 # print('There are %s new broken links:' % len(newly_broken_links)) 243 # print(StringifyBrokenLinks(newly_broken_links)) 244 # self.fail('See logging for details.') 245 246 # TODO(kalman): Move this test elsewhere, it's not an integration test. 247 # Perhaps like "presubmit_tests" or something. 248 def testExplicitFiles(self): 249 '''Tests just the files in _EXPLICIT_TEST_FILES. 250 ''' 251 if _EXPLICIT_TEST_FILES is None: 252 return 253 for filename in _EXPLICIT_TEST_FILES: 254 print('Rendering %s...' % filename) 255 start_time = time.time() 256 try: 257 response = LocalRenderer.Render(_ToPosixPath(filename)) 258 self.assertEqual(200, response.status) 259 self.assertTrue(response.content != '') 260 finally: 261 print('Took %s seconds' % (time.time() - start_time)) 262 263 # TODO(jshumway): Check page for broken links (currently prohibited by the 264 # time it takes to render the pages). 265 266 @DisableLogging('warning') 267 def testFileNotFound(self): 268 response = LocalRenderer.Render('/extensions/notfound') 269 self.assertEqual(404, response.status) 270 271 def testSiteVerificationFile(self): 272 response = LocalRenderer.Render('/' + SITE_VERIFICATION_FILE) 273 self.assertEqual(200, response.status) 274 275 if __name__ == '__main__': 276 parser = optparse.OptionParser() 277 parser.add_option('-a', '--all', action='store_true', default=False, 278 help='Render all pages, not just the one specified') 279 parser.add_option('-r', '--rebase', action='store_true', default=False, 280 help='Rewrites the known_broken_links.json file with ' 281 'the current set of broken links') 282 parser.add_option('-v', '--verbose', action='store_true', default=False, 283 help='Show verbose output like currently broken links') 284 (opts, args) = parser.parse_args() 285 if not opts.all: 286 _EXPLICIT_TEST_FILES = args 287 _REBASE = opts.rebase 288 _VERBOSE = opts.verbose 289 # Kill sys.argv because we have our own flags. 290 sys.argv = [sys.argv[0]] 291 unittest.main() 292
