1 #!/usr/bin/python 2 3 import cgi, os, sys, urllib2 4 import common 5 from multiprocessing import pool 6 from autotest_lib.frontend import setup_django_environment 7 8 from autotest_lib.client.common_lib import global_config 9 from autotest_lib.client.bin import utils 10 from autotest_lib.frontend.afe.json_rpc import serviceHandler 11 from autotest_lib.server import system_utils 12 from autotest_lib.server import utils as server_utils 13 14 15 _PAGE = """\ 16 Status: 302 Found 17 Content-Type: text/plain 18 Location: %s\r\n\r 19 """ 20 21 GOOGLE_STORAGE_PATTERN = 'pantheon.corp.google.com/storage/browser/' 22 23 # Define function for retrieving logs 24 def _retrieve_logs_dummy(job_path): 25 pass 26 27 site_retrieve_logs = utils.import_site_function(__file__, 28 "autotest_lib.tko.site_retrieve_logs", "site_retrieve_logs", 29 _retrieve_logs_dummy) 30 31 site_find_repository_host = utils.import_site_function(__file__, 32 "autotest_lib.tko.site_retrieve_logs", "site_find_repository_host", 33 _retrieve_logs_dummy) 34 35 form = cgi.FieldStorage(keep_blank_values=True) 36 # determine if this is a JSON-RPC request. we support both so that the new TKO 37 # client can use its RPC client code, but the old TKO can still use simple GET 38 # params. 39 _is_json_request = form.has_key('callback') 40 41 # if this key exists, we check if requested log exists in local machine, 42 # and do not return Google Storage URL when the log doesn't exist. 43 _local_only = form.has_key('localonly') 44 45 46 def _get_requested_path(): 47 if _is_json_request: 48 request_data = form['request'].value 49 request = serviceHandler.ServiceHandler.translateRequest(request_data) 50 parameters = request['params'][0] 51 return parameters['path'] 52 53 return form['job'].value 54 55 56 def _check_result(args): 57 host = args['host'] 58 job_path = args['job_path'] 59 shard = args['shard'] 60 if shard: 61 http_path = 'http://%s/tko/retrieve_logs.cgi?localonly&job=%s' % ( 62 host, job_path) 63 else: 64 http_path = 'http://%s%s' % (host, job_path) 65 66 try: 67 utils.urlopen(http_path) 68 69 # On Vms the shard name is set to the default gateway but the 70 # browser used to navigate frontends (that runs on the host of 71 # the vms) is immune to the same NAT routing the vms have, so we 72 # need to replace the gateway with 'localhost'. 73 if utils.DEFAULT_VM_GATEWAY in host: 74 normalized_host = host.replace(utils.DEFAULT_VM_GATEWAY, 'localhost') 75 else: 76 normalized_host = utils.normalize_hostname(host) 77 return 'http', normalized_host, job_path 78 except urllib2.URLError: 79 return None 80 81 82 def _get_tpool_args(hosts, job_path, is_shard, host_set): 83 """Get a list of arguments to be passed to multiprocessing.pool.ThreadPool. 84 85 @param hosts: a list of host names. 86 @param job_path: a requested job path. 87 @param is_shard: True if hosts are shards, False otherwise. 88 @param host_set: a Set to filter out duplicated hosts. 89 90 @return: a list of dictionaries to be used as input of _check_result(). 91 """ 92 args = [] 93 for host in hosts: 94 host = host.strip() 95 if host and host != 'localhost' and host not in host_set: 96 host_set.add(host) 97 arg = {'host': host, 'job_path': job_path, 'shard': is_shard} 98 args.append(arg) 99 return args 100 101 102 def find_repository_host(job_path): 103 """Find the machine holding the given logs and return a URL to the logs""" 104 site_repo_info = site_find_repository_host(job_path) 105 if site_repo_info is not None: 106 return site_repo_info 107 108 # This cgi script is run only in master (cautotest) and shards. 109 # Drones do not run this script when receiving '/results/...' request. 110 # Only master should check drones and shards for the requested log. 111 # Also restricted users do not have access to drones or shards, 112 # always point them to localhost or google storage. 113 if (not server_utils.is_shard() and 114 not server_utils.is_restricted_user(os.environ.get('REMOTE_USER'))): 115 drones = system_utils.get_drones() 116 shards = system_utils.get_shards() 117 118 host_set = set() 119 tpool_args = _get_tpool_args(drones, job_path, False, host_set) 120 tpool_args += _get_tpool_args(shards, job_path, True, host_set) 121 122 tpool = pool.ThreadPool() 123 for result_path in tpool.imap_unordered(_check_result, tpool_args): 124 if result_path: 125 return result_path 126 127 # If the URL requested is a test result, it is now either on the local 128 # host or in Google Storage. 129 if job_path.startswith('/results/'): 130 # We only care about the path after '/results/'. 131 job_relative_path = job_path[9:] 132 if not _local_only and not os.path.exists( 133 os.path.join('/usr/local/autotest/results', 134 job_relative_path)): 135 gsuri = utils.get_offload_gsuri().split('gs://')[1] 136 return ['https', GOOGLE_STORAGE_PATTERN, gsuri + job_relative_path] 137 138 139 def get_full_url(info, log_path): 140 if info is not None: 141 protocol, host, path = info 142 prefix = '%s://%s' % (protocol, host) 143 else: 144 prefix = '' 145 path = log_path 146 147 if _is_json_request: 148 return '%s/tko/jsonp_fetcher.cgi?%s' % (prefix, 149 os.environ['QUERY_STRING']) 150 else: 151 return prefix + path 152 153 154 log_path = _get_requested_path() 155 info = find_repository_host(log_path) 156 site_retrieve_logs(log_path) 157 print _PAGE % get_full_url(info, log_path) 158
