1 import os, shutil, tempfile, logging 2 3 import common 4 from autotest_lib.client.common_lib import utils, error, profiler_manager 5 from autotest_lib.server import profiler, autotest, standalone_profiler, hosts 6 7 8 PROFILER_TMPDIR = '/tmp/profilers' 9 10 11 def get_profiler_results_dir(autodir): 12 """ 13 Given the directory of the autotest client used to run a profiler, 14 return the remote path where profiler results will be stored. 15 """ 16 return os.path.join(autodir, 'results', 'default', 'profiler_sync', 17 'profiling') 18 19 20 def get_profiler_log_path(autodir): 21 """ 22 Given the directory of a profiler client, find the client log path. 23 """ 24 return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG') 25 26 27 class profilers(profiler_manager.profiler_manager): 28 def __init__(self, job): 29 super(profilers, self).__init__(job) 30 self.add_log = {} 31 self.start_delay = 0 32 # maps hostname to (host object, autotest.Autotest object, Autotest 33 # install dir), where the host object is the one created specifically 34 # for profiling 35 self.installed_hosts = {} 36 self.current_test = None 37 38 39 def set_start_delay(self, start_delay): 40 self.start_delay = start_delay 41 42 43 def load_profiler(self, profiler_name, args, dargs): 44 newprofiler = profiler.profiler_proxy(profiler_name) 45 newprofiler.initialize(*args, **dargs) 46 newprofiler.setup(*args, **dargs) # lazy setup is done client-side 47 return newprofiler 48 49 50 def add(self, profiler, *args, **dargs): 51 super(profilers, self).add(profiler, *args, **dargs) 52 self.add_log[profiler] = (args, dargs) 53 54 55 def delete(self, profiler): 56 super(profilers, self).delete(profiler) 57 if profiler in self.add_log: 58 del self.add_log[profiler] 59 60 61 def _install_clients(self): 62 """ 63 Install autotest on any current job hosts. 64 """ 65 in_use_hosts = set() 66 # find hosts in use but not used by us 67 for host in self.job.hosts: 68 if host.hostname not in self.job.machines: 69 # job.hosts include all host instances created on the fly. 70 # We only care DUTs in job.machines which are 71 # piped in from autoserv -m option. 72 continue 73 autodir = host.get_autodir() 74 if not (autodir and autodir.startswith(PROFILER_TMPDIR)): 75 in_use_hosts.add(host.hostname) 76 logging.debug('Hosts currently in use: %s', in_use_hosts) 77 78 # determine what valid host objects we already have installed 79 profiler_hosts = set() 80 for host, at, profiler_dir in self.installed_hosts.values(): 81 if host.path_exists(profiler_dir): 82 profiler_hosts.add(host.hostname) 83 else: 84 # the profiler was wiped out somehow, drop this install 85 logging.warning('The profiler client on %s at %s was deleted', 86 host.hostname, profiler_dir) 87 host.close() 88 del self.installed_hosts[host.hostname] 89 logging.debug('Hosts with profiler clients already installed: %s', 90 profiler_hosts) 91 92 # install autotest on any new hosts in use 93 for hostname in in_use_hosts - profiler_hosts: 94 host = hosts.create_host(hostname) 95 tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR) 96 at = autotest.Autotest(host) 97 at.install_no_autoserv(autodir=tmp_dir) 98 self.installed_hosts[host.hostname] = (host, at, tmp_dir) 99 100 # drop any installs from hosts no longer in job.hosts 101 hostnames_to_drop = profiler_hosts - in_use_hosts 102 hosts_to_drop = [self.installed_hosts[hostname][0] 103 for hostname in hostnames_to_drop] 104 for host in hosts_to_drop: 105 host.close() 106 del self.installed_hosts[host.hostname] 107 108 109 def _get_hosts(self, host=None): 110 """ 111 Returns a list of (Host, Autotest, install directory) tuples for hosts 112 currently supported by this profiler. The returned Host object is always 113 the one created by this profiler, regardless of what's passed in. If 114 'host' is not None, all entries not matching that host object are 115 filtered out of the list. 116 """ 117 if host is None: 118 return self.installed_hosts.values() 119 if host.hostname in self.installed_hosts: 120 return [self.installed_hosts[host.hostname]] 121 return [] 122 123 124 def _get_local_profilers_dir(self, test, hostname): 125 in_machine_dir = ( 126 os.path.basename(test.job.resultdir) in test.job.machines) 127 if len(test.job.machines) > 1 and not in_machine_dir: 128 local_dir = os.path.join(test.profdir, hostname) 129 if not os.path.exists(local_dir): 130 os.makedirs(local_dir) 131 else: 132 local_dir = test.profdir 133 134 return local_dir 135 136 137 def _get_failure_logs(self, autodir, test, host): 138 """ 139 Collect the client logs from a profiler run and put them in a 140 file named failure-*.log. 141 """ 142 try: 143 fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-', 144 dir=self._get_local_profilers_dir(test, host.hostname)) 145 os.close(fd) 146 host.get_file(get_profiler_log_path(autodir), path) 147 # try to collect any partial profiler logs 148 self._get_profiler_logs(autodir, test, host) 149 except (error.AutotestError, error.AutoservError): 150 logging.exception('Profiler failure log collection failed') 151 # swallow the exception so that we don't override an existing 152 # exception being thrown 153 154 155 def _get_all_failure_logs(self, test, hosts): 156 for host, at, autodir in hosts: 157 self._get_failure_logs(autodir, test, host) 158 159 160 def _get_profiler_logs(self, autodir, test, host): 161 results_dir = get_profiler_results_dir(autodir) 162 local_dir = self._get_local_profilers_dir(test, host.hostname) 163 164 self.job.remove_client_log(host.hostname, results_dir, local_dir) 165 166 tempdir = tempfile.mkdtemp(dir=self.job.tmpdir) 167 try: 168 host.get_file(results_dir + '/', tempdir) 169 except error.AutoservRunError: 170 pass # no files to pull back, nothing we can do 171 utils.merge_trees(tempdir, local_dir) 172 shutil.rmtree(tempdir, ignore_errors=True) 173 174 175 def _run_clients(self, test, hosts): 176 """ 177 We initialize the profilers just before start because only then we 178 know all the hosts involved. 179 """ 180 181 hostnames = [host_info[0].hostname for host_info in hosts] 182 profilers_args = [(p.name, p.args, p.dargs) 183 for p in self.list] 184 185 for host, at, autodir in hosts: 186 control_script = standalone_profiler.generate_test(hostnames, 187 host.hostname, 188 profilers_args, 189 180, None) 190 try: 191 at.run(control_script, background=True) 192 except Exception: 193 self._get_failure_logs(autodir, test, host) 194 raise 195 196 remote_results_dir = get_profiler_results_dir(autodir) 197 local_results_dir = self._get_local_profilers_dir(test, 198 host.hostname) 199 self.job.add_client_log(host.hostname, remote_results_dir, 200 local_results_dir) 201 202 try: 203 # wait for the profilers to be added 204 standalone_profiler.wait_for_profilers(hostnames) 205 except Exception: 206 self._get_all_failure_logs(test, hosts) 207 raise 208 209 210 def before_start(self, test, host=None): 211 # create host objects and install the needed clients 212 # so later in start() we don't spend too much time 213 self._install_clients() 214 self._run_clients(test, self._get_hosts(host)) 215 216 217 def start(self, test, host=None): 218 hosts = self._get_hosts(host) 219 220 # wait for the profilers to start 221 hostnames = [host_info[0].hostname for host_info in hosts] 222 try: 223 standalone_profiler.start_profilers(hostnames) 224 except Exception: 225 self._get_all_failure_logs(test, hosts) 226 raise 227 228 self.current_test = test 229 230 231 def stop(self, test): 232 assert self.current_test == test 233 234 hosts = self._get_hosts() 235 # wait for the profilers to stop 236 hostnames = [host_info[0].hostname for host_info in hosts] 237 try: 238 standalone_profiler.stop_profilers(hostnames) 239 except Exception: 240 self._get_all_failure_logs(test, hosts) 241 raise 242 243 244 def report(self, test, host=None): 245 assert self.current_test == test 246 247 hosts = self._get_hosts(host) 248 # when running on specific hosts we cannot wait for the other 249 # hosts to sync with us 250 if not host: 251 hostnames = [host_info[0].hostname for host_info in hosts] 252 try: 253 standalone_profiler.finish_profilers(hostnames) 254 except Exception: 255 self._get_all_failure_logs(test, hosts) 256 raise 257 258 # pull back all the results 259 for host, at, autodir in hosts: 260 self._get_profiler_logs(autodir, test, host) 261 262 263 def handle_reboot(self, host): 264 if self.current_test: 265 test = self.current_test 266 for profiler in self.list: 267 if not profiler.supports_reboot: 268 msg = 'profiler %s does not support rebooting during tests' 269 msg %= profiler.name 270 self.job.record('WARN', os.path.basename(test.outputdir), 271 None, msg) 272 273 self.report(test, host) 274 self.before_start(test, host) 275 self.start(test, host) 276
