1 #!/usr/bin/python 2 3 import cgi, os, socket, sys, urllib2 4 import common 5 from multiprocessing import pool 6 from autotest_lib.frontend import setup_django_environment 7 8 from autotest_lib.client.common_lib import global_config 9 from autotest_lib.client.bin import utils 10 from autotest_lib.frontend.afe.json_rpc import serviceHandler 11 from autotest_lib.server import system_utils 12 from autotest_lib.server import utils as server_utils 13 14 15 _PAGE = """\ 16 Status: 302 Found 17 Content-Type: text/plain 18 Location: %s\r\n\r 19 """ 20 21 VIEWER_PREFIX = 'stainless.corp.google.com/browse/' 22 23 # Define function for retrieving logs 24 def _retrieve_logs_dummy(job_path): 25 pass 26 27 site_retrieve_logs = utils.import_site_function(__file__, 28 "autotest_lib.tko.site_retrieve_logs", "site_retrieve_logs", 29 _retrieve_logs_dummy) 30 31 site_find_repository_host = utils.import_site_function(__file__, 32 "autotest_lib.tko.site_retrieve_logs", "site_find_repository_host", 33 _retrieve_logs_dummy) 34 35 form = cgi.FieldStorage(keep_blank_values=True) 36 # determine if this is a JSON-RPC request. we support both so that the new TKO 37 # client can use its RPC client code, but the old TKO can still use simple GET 38 # params. 39 _is_json_request = form.has_key('callback') 40 41 # if this key exists, we check if requested log exists in local machine, 42 # and do not return Google Storage URL when the log doesn't exist. 43 _local_only = form.has_key('localonly') 44 45 46 def _get_requested_path(): 47 if _is_json_request: 48 request_data = form['request'].value 49 request = serviceHandler.ServiceHandler.translateRequest(request_data) 50 parameters = request['params'][0] 51 return parameters['path'] 52 53 return form['job'].value 54 55 56 def _check_result(args): 57 host = args['host'] 58 job_path = args['job_path'] 59 shard = args['shard'] 60 if shard: 61 http_path = 'http://%s/tko/retrieve_logs.cgi?localonly&job=%s' % ( 62 host, job_path) 63 else: 64 http_path = 'http://%s%s' % (host, job_path) 65 66 try: 67 # HACK: This urlopen call isn't forwarding HTTP headers correctly. This 68 # leads to uberproxy sitting between master (orignator of this request) 69 # and shard (target of the request) to redirect to the the login page. 70 # We detect this condition and reject the target shard as a viable 71 # redirect. The implication is that we will not redirect to the shard 72 # even if the user could themselves access the shard with the correct 73 # credentials. 74 u = utils.urlopen(http_path) 75 redirected_url = u.geturl() 76 if 'accounts.google.com' in redirected_url: 77 return None 78 79 # On Vms the shard name is set to the default gateway but the 80 # browser used to navigate frontends (that runs on the host of 81 # the vms) is immune to the same NAT routing the vms have, so we 82 # need to replace the gateway with 'localhost'. 83 if utils.DEFAULT_VM_GATEWAY in host: 84 normalized_host = host.replace(utils.DEFAULT_VM_GATEWAY, 'localhost') 85 else: 86 try: 87 normalized_host = utils.normalize_hostname(host) 88 except socket.herror: 89 # Ignore error: 'socket.herror: [Errno 1] Unknown host' 90 # This can happen when reverse name lookup is not stable. 91 normalized_host = host 92 return 'http', normalized_host, job_path 93 except urllib2.URLError: 94 return None 95 96 97 def _get_tpool_args(hosts, job_path, is_shard, host_set): 98 """Get a list of arguments to be passed to multiprocessing.pool.ThreadPool. 99 100 @param hosts: a list of host names. 101 @param job_path: a requested job path. 102 @param is_shard: True if hosts are shards, False otherwise. 103 @param host_set: a Set to filter out duplicated hosts. 104 105 @return: a list of dictionaries to be used as input of _check_result(). 106 """ 107 args = [] 108 for host in hosts: 109 host = host.strip() 110 if host and host != 'localhost' and host not in host_set: 111 host_set.add(host) 112 arg = {'host': host, 'job_path': job_path, 'shard': is_shard} 113 args.append(arg) 114 return args 115 116 117 def find_repository_host(job_path): 118 """Find the machine holding the given logs and return a URL to the logs""" 119 site_repo_info = site_find_repository_host(job_path) 120 if site_repo_info is not None: 121 return site_repo_info 122 123 # This cgi script is run only in master (cautotest) and shards. 124 # Drones do not run this script when receiving '/results/...' request. 125 # Only master should check drones and shards for the requested log. 126 # Also restricted users do not have access to drones or shards, 127 # always point them to localhost or google storage. 128 if (not server_utils.is_shard() and 129 not server_utils.is_restricted_user(os.environ.get('REMOTE_USER'))): 130 drones = system_utils.get_drones() 131 shards = system_utils.get_shards() 132 133 host_set = set() 134 tpool_args = _get_tpool_args(drones, job_path, False, host_set) 135 tpool_args += _get_tpool_args(shards, job_path, True, host_set) 136 137 tpool = pool.ThreadPool() 138 for result_path in tpool.imap_unordered(_check_result, tpool_args): 139 if result_path: 140 return result_path 141 142 # If the URL requested is a test result, it is now either on the local 143 # host or in Google Storage. 144 if job_path.startswith('/results/'): 145 # We only care about the path after '/results/'. 146 job_relative_path = job_path[9:] 147 if not _local_only and not os.path.exists( 148 os.path.join('/usr/local/autotest/results', 149 job_relative_path)): 150 gsuri = utils.get_offload_gsuri().split('gs://')[1] 151 return ['https', VIEWER_PREFIX, gsuri + job_relative_path] 152 153 154 def get_full_url(info, log_path): 155 if info is not None: 156 protocol, host, path = info 157 prefix = '%s://%s' % (protocol, host) 158 else: 159 prefix = '' 160 path = log_path 161 162 if _is_json_request: 163 return '%s/tko/jsonp_fetcher.cgi?%s' % (prefix, 164 os.environ['QUERY_STRING']) 165 else: 166 return prefix + path 167 168 169 log_path = _get_requested_path() 170 info = find_repository_host(log_path) 171 site_retrieve_logs(log_path) 172 print _PAGE % get_full_url(info, log_path) 173
