1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import csv 7 import cStringIO 8 import random 9 import re 10 import collections 11 12 from autotest_lib.client.common_lib.cros import path_utils 13 14 class ResourceMonitorRawResult(object): 15 """Encapsulates raw resource_monitor results.""" 16 17 def __init__(self, raw_results_filename): 18 self._raw_results_filename = raw_results_filename 19 20 21 def get_parsed_results(self): 22 """Constructs parsed results from the raw ones. 23 24 @return ResourceMonitorParsedResult object 25 26 """ 27 return ResourceMonitorParsedResult(self.raw_results_filename) 28 29 30 @property 31 def raw_results_filename(self): 32 """@return string filename storing the raw top command output.""" 33 return self._raw_results_filename 34 35 36 class IncorrectTopFormat(Exception): 37 """Thrown if top output format is not as expected""" 38 pass 39 40 41 def _extract_value_before_single_keyword(line, keyword): 42 """Extract word occurring immediately before the specified keyword. 43 44 @param line string the line in which to search for the keyword. 45 @param keyword string the keyword to look for. Can be a regexp. 46 @return string the word just before the keyword. 47 48 """ 49 pattern = ".*?(\S+) " + keyword 50 matches = re.match(pattern, line) 51 if matches is None or len(matches.groups()) != 1: 52 raise IncorrectTopFormat 53 54 return matches.group(1) 55 56 57 def _extract_values_before_keywords(line, *args): 58 """Extract the words occuring immediately before each specified 59 keyword in args. 60 61 @param line string the string to look for the keywords. 62 @param args variable number of string args the keywords to look for. 63 @return string list the words occuring just before each keyword. 64 65 """ 66 line_nocomma = re.sub(",", " ", line) 67 line_singlespace = re.sub("\s+", " ", line_nocomma) 68 69 return [_extract_value_before_single_keyword( 70 line_singlespace, arg) for arg in args] 71 72 73 def _find_top_output_identifying_pattern(line): 74 """Return true iff the line looks like the first line of top output. 75 76 @param line string to look for the pattern 77 @return boolean 78 79 """ 80 pattern ="\s*top\s*-.*up.*users.*" 81 matches = re.match(pattern, line) 82 return matches is not None 83 84 85 class ResourceMonitorParsedResult(object): 86 """Encapsulates logic to parse and represent top command results.""" 87 88 _columns = ["Time", "UserCPU", "SysCPU", "NCPU", "Idle", 89 "IOWait", "IRQ", "SoftIRQ", "Steal", 90 "MemUnits", "UsedMem", "FreeMem", 91 "SwapUnits", "UsedSwap", "FreeSwap"] 92 UtilValues = collections.namedtuple('UtilValues', ' '.join(_columns)) 93 94 def __init__(self, raw_results_filename): 95 """Construct a ResourceMonitorResult. 96 97 @param raw_results_filename string filename of raw batch top output. 98 99 """ 100 self._raw_results_filename = raw_results_filename 101 self.parse_resource_monitor_results() 102 103 104 def parse_resource_monitor_results(self): 105 """Extract utilization metrics from output file.""" 106 self._utils_over_time = [] 107 108 with open(self._raw_results_filename, "r") as results_file: 109 while True: 110 curr_line = '\n' 111 while curr_line != '' and \ 112 not _find_top_output_identifying_pattern(curr_line): 113 curr_line = results_file.readline() 114 if curr_line == '': 115 break 116 try: 117 time, = _extract_values_before_keywords(curr_line, "up") 118 119 # Ignore one line. 120 _ = results_file.readline() 121 122 # Get the cpu usage. 123 curr_line = results_file.readline() 124 (cpu_user, cpu_sys, cpu_nice, cpu_idle, io_wait, irq, sirq, 125 steal) = _extract_values_before_keywords(curr_line, 126 "us", "sy", "ni", "id", "wa", "hi", "si", "st") 127 128 # Get memory usage. 129 curr_line = results_file.readline() 130 (mem_units, mem_free, 131 mem_used) = _extract_values_before_keywords( 132 curr_line, "Mem", "free", "used") 133 134 # Get swap usage. 135 curr_line = results_file.readline() 136 (swap_units, swap_free, 137 swap_used) = _extract_values_before_keywords( 138 curr_line, "Swap", "free", "used") 139 140 curr_util_values = ResourceMonitorParsedResult.UtilValues( 141 Time=time, UserCPU=cpu_user, 142 SysCPU=cpu_sys, NCPU=cpu_nice, Idle=cpu_idle, 143 IOWait=io_wait, IRQ=irq, SoftIRQ=sirq, Steal=steal, 144 MemUnits=mem_units, UsedMem=mem_used, 145 FreeMem=mem_free, 146 SwapUnits=swap_units, UsedSwap=swap_used, 147 FreeSwap=swap_free) 148 self._utils_over_time.append(curr_util_values) 149 except IncorrectTopFormat: 150 logging.error( 151 "Top output format incorrect. Aborting parse.") 152 return 153 154 155 def __repr__(self): 156 output_stringfile = cStringIO.StringIO() 157 self.save_to_file(output_stringfile) 158 return output_stringfile.getvalue() 159 160 161 def save_to_file(self, file): 162 """Save parsed top results to file 163 164 @param file file object to write to 165 166 """ 167 if len(self._utils_over_time) < 1: 168 logging.warning("Tried to save parsed results, but they were " 169 "empty. Skipping the save.") 170 return 171 csvwriter = csv.writer(file, delimiter=',') 172 csvwriter.writerow(self._utils_over_time[0]._fields) 173 for row in self._utils_over_time: 174 csvwriter.writerow(row) 175 176 177 def save_to_filename(self, filename): 178 """Save parsed top results to filename 179 180 @param filename string filepath to write to 181 182 """ 183 out_file = open(filename, "wb") 184 self.save_to_file(out_file) 185 out_file.close() 186 187 188 class ResourceMonitorConfig(object): 189 """Defines a single top run.""" 190 191 DEFAULT_MONITOR_PERIOD = 3 192 193 def __init__(self, monitor_period=DEFAULT_MONITOR_PERIOD, 194 rawresult_output_filename=None): 195 """Construct a ResourceMonitorConfig. 196 197 @param monitor_period float seconds between successive top refreshes. 198 @param rawresult_output_filename string filename to output the raw top 199 results to 200 201 """ 202 if monitor_period < 0.1: 203 logging.info('Monitor period must be at least 0.1s.' 204 ' Given: %r. Defaulting to 0.1s', monitor_period) 205 monitor_period = 0.1 206 207 self._monitor_period = monitor_period 208 self._server_outfile = rawresult_output_filename 209 210 211 class ResourceMonitor(object): 212 """Delegate to run top on a client. 213 214 Usage example (call from a test): 215 rmc = resource_monitor.ResourceMonitorConfig(monitor_period=1, 216 rawresult_output_filename=os.path.join(self.resultsdir, 217 'topout.txt')) 218 with resource_monitor.ResourceMonitor(self.context.client.host, rmc) as rm: 219 rm.start() 220 <operation_to_monitor> 221 rm_raw_res = rm.stop() 222 rm_res = rm_raw_res.get_parsed_results() 223 rm_res.save_to_filename( 224 os.path.join(self.resultsdir, 'resource_mon.csv')) 225 226 """ 227 228 def __init__(self, client_host, config): 229 """Construct a ResourceMonitor. 230 231 @param client_host: SSHHost object representing a remote ssh host 232 233 """ 234 self._client_host = client_host 235 self._config = config 236 self._command_top = path_utils.must_be_installed( 237 'top', host=self._client_host) 238 self._top_pid = None 239 240 241 def __enter__(self): 242 return self 243 244 245 def __exit__(self, exc_type, exc_value, traceback): 246 if self._top_pid is not None: 247 self._client_host.run('kill %s && rm %s' % 248 (self._top_pid, self._client_outfile), ignore_status=True) 249 return True 250 251 252 def start(self): 253 """Run top and save results to a temp file on the client.""" 254 if self._top_pid is not None: 255 logging.debug("Tried to start monitoring before stopping. " 256 "Ignoring request.") 257 return 258 259 # Decide where to write top's output to (on the client). 260 random_suffix = random.random() 261 self._client_outfile = '/tmp/topcap-%r' % random_suffix 262 263 # Run top on the client. 264 top_command = '%s -b -d%d > %s' % (self._command_top, 265 self._config._monitor_period, self._client_outfile) 266 logging.info('Running top.') 267 self._top_pid = self._client_host.run_background(top_command) 268 logging.info('Top running with pid %s', self._top_pid) 269 270 271 def stop(self): 272 """Stop running top and return the results. 273 274 @return ResourceMonitorRawResult object 275 276 """ 277 logging.debug("Stopping monitor") 278 if self._top_pid is None: 279 logging.debug("Tried to stop monitoring before starting. " 280 "Ignoring request.") 281 return 282 283 # Stop top on the client. 284 self._client_host.run('kill %s' % self._top_pid, ignore_status=True) 285 286 # Get the top output file from the client onto the server. 287 if self._config._server_outfile is None: 288 self._config._server_outfile = self._client_outfile 289 self._client_host.get_file( 290 self._client_outfile, self._config._server_outfile) 291 292 # Delete the top output file from client. 293 self._client_host.run('rm %s' % self._client_outfile, 294 ignore_status=True) 295 296 self._top_pid = None 297 logging.info("Saved resource monitor results at %s", 298 self._config._server_outfile) 299 return ResourceMonitorRawResult(self._config._server_outfile) 300
