1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 8 from autotest_lib.client.common_lib import log 9 from autotest_lib.client.common_lib import error, utils, global_config 10 from autotest_lib.client.bin import base_sysinfo, utils 11 from autotest_lib.client.cros import constants, tpm_dam 12 13 get_value = global_config.global_config.get_config_value 14 collect_corefiles = get_value('CLIENT', 'collect_corefiles', 15 type=bool, default=True) 16 17 18 logfile = base_sysinfo.logfile 19 command = base_sysinfo.command 20 21 22 class logdir(base_sysinfo.loggable): 23 """Represents a log directory.""" 24 def __init__(self, directory, additional_exclude=None): 25 super(logdir, self).__init__(directory, log_in_keyval=False) 26 self.dir = directory 27 self.additional_exclude = additional_exclude 28 29 30 def __repr__(self): 31 return "site_sysinfo.logdir(%r, %s)" % (self.dir, 32 self.additional_exclude) 33 34 35 def __eq__(self, other): 36 if isinstance(other, logdir): 37 return (self.dir == other.dir and 38 self.additional_exclude == other.additional_exclude) 39 elif isinstance(other, base_sysinfo.loggable): 40 return False 41 return NotImplemented 42 43 44 def __ne__(self, other): 45 result = self.__eq__(other) 46 if result is NotImplemented: 47 return result 48 return not result 49 50 51 def __hash__(self): 52 return hash(self.dir) + hash(self.additional_exclude) 53 54 55 def run(self, log_dir): 56 """Copies this log directory to the specified directory. 57 58 @param log_dir: The destination log directory. 59 """ 60 if os.path.exists(self.dir): 61 parent_dir = os.path.dirname(self.dir) 62 utils.system("mkdir -p %s%s" % (log_dir, parent_dir)) 63 # Take source permissions and add ugo+r so files are accessible via 64 # archive server. 65 additional_exclude_str = "" 66 if self.additional_exclude: 67 additional_exclude_str = "--exclude=" + self.additional_exclude 68 69 utils.system("rsync --no-perms --chmod=ugo+r -a --exclude=autoserv*" 70 " %s %s %s%s" % (additional_exclude_str, self.dir, 71 log_dir, parent_dir)) 72 73 74 class file_stat(object): 75 """Store the file size and inode, used for retrieving new data in file.""" 76 def __init__(self, file_path): 77 """Collect the size and inode information of a file. 78 79 @param file_path: full path to the file. 80 81 """ 82 stat = os.stat(file_path) 83 # Start size of the file, skip that amount of bytes when do diff. 84 self.st_size = stat.st_size 85 # inode of the file. If inode is changed, treat this as a new file and 86 # copy the whole file. 87 self.st_ino = stat.st_ino 88 89 90 class diffable_logdir(logdir): 91 """Represents a log directory that only new content will be copied. 92 93 An instance of this class should be added in both 94 before_iteration_loggables and after_iteration_loggables. This is to 95 guarantee the file status information is collected when run method is 96 called in before_iteration_loggables, and diff is executed when run 97 method is called in after_iteration_loggables. 98 99 """ 100 def __init__(self, directory, additional_exclude=None, 101 keep_file_hierarchy=True, append_diff_in_name=True): 102 """ 103 Constructor of a diffable_logdir instance. 104 105 @param directory: directory to be diffed after an iteration finished. 106 @param additional_exclude: additional dir to be excluded, not used. 107 @param keep_file_hierarchy: True if need to preserve full path, e.g., 108 sysinfo/var/log/sysstat, v.s. sysinfo/sysstat if it's False. 109 @param append_diff_in_name: True if you want to append '_diff' to the 110 folder name to indicate it's a diff, e.g., var/log_diff. Option 111 keep_file_hierarchy must be True for this to take effect. 112 113 """ 114 super(diffable_logdir, self).__init__(directory, additional_exclude) 115 self.additional_exclude = additional_exclude 116 self.keep_file_hierarchy = keep_file_hierarchy 117 self.append_diff_in_name = append_diff_in_name 118 # Init dictionary to store all file status for files in the directory. 119 self._log_stats = {} 120 121 122 def _get_init_status_of_src_dir(self, src_dir): 123 """Get initial status of files in src_dir folder. 124 125 @param src_dir: directory to be diff-ed. 126 127 """ 128 # Dictionary used to store the initial status of files in src_dir. 129 for file_path in self._get_all_files(src_dir): 130 self._log_stats[file_path] = file_stat(file_path) 131 self.file_stats_collected = True 132 133 134 def _get_all_files(self, path): 135 """Iterate through files in given path including subdirectories. 136 137 @param path: root directory. 138 @return: an iterator that iterates through all files in given path 139 including subdirectories. 140 141 """ 142 if not os.path.exists(path): 143 yield [] 144 for root, dirs, files in os.walk(path): 145 for f in files: 146 if f.startswith('autoserv'): 147 continue 148 yield os.path.join(root, f) 149 150 151 def _copy_new_data_in_file(self, file_path, src_dir, dest_dir): 152 """Copy all new data in a file to target directory. 153 154 @param file_path: full path to the file to be copied. 155 @param src_dir: source directory to do the diff. 156 @param dest_dir: target directory to store new data of src_dir. 157 158 """ 159 bytes_to_skip = 0 160 if self._log_stats.has_key(file_path): 161 prev_stat = self._log_stats[file_path] 162 new_stat = os.stat(file_path) 163 if new_stat.st_ino == prev_stat.st_ino: 164 bytes_to_skip = prev_stat.st_size 165 if new_stat.st_size == bytes_to_skip: 166 return 167 elif new_stat.st_size < prev_stat.st_size: 168 # File is modified to a smaller size, copy whole file. 169 bytes_to_skip = 0 170 try: 171 with open(file_path, 'r') as in_log: 172 if bytes_to_skip > 0: 173 in_log.seek(bytes_to_skip) 174 # Skip src_dir in path, e.g., src_dir/[sub_dir]/file_name. 175 target_path = os.path.join(dest_dir, 176 os.path.relpath(file_path, src_dir)) 177 target_dir = os.path.dirname(target_path) 178 if not os.path.exists(target_dir): 179 os.makedirs(target_dir) 180 with open(target_path, "w") as out_log: 181 out_log.write(in_log.read()) 182 except IOError as e: 183 logging.error('Diff %s failed with error: %s', file_path, e) 184 185 186 def _log_diff(self, src_dir, dest_dir): 187 """Log all of the new data in src_dir to dest_dir. 188 189 @param src_dir: source directory to do the diff. 190 @param dest_dir: target directory to store new data of src_dir. 191 192 """ 193 if self.keep_file_hierarchy: 194 dir = src_dir.lstrip('/') 195 if self.append_diff_in_name: 196 dir = dir.rstrip('/') + '_diff' 197 dest_dir = os.path.join(dest_dir, dir) 198 199 if not os.path.exists(dest_dir): 200 os.makedirs(dest_dir) 201 202 for src_file in self._get_all_files(src_dir): 203 self._copy_new_data_in_file(src_file, src_dir, dest_dir) 204 205 206 def run(self, log_dir, collect_init_status=True, collect_all=False): 207 """Copies new content from self.dir to the destination log_dir. 208 209 @param log_dir: The destination log directory. 210 @param collect_init_status: Set to True if run method is called to 211 collect the initial status of files. 212 @param collect_all: Set to True to force to collect all files. 213 214 """ 215 if collect_init_status: 216 self._get_init_status_of_src_dir(self.dir) 217 elif os.path.exists(self.dir): 218 if not collect_all: 219 self._log_diff(self.dir, log_dir) 220 else: 221 logdir_temp = logdir(self.dir) 222 logdir_temp.run(log_dir) 223 224 225 class purgeable_logdir(logdir): 226 """Represents a log directory that will be purged.""" 227 def __init__(self, directory, additional_exclude=None): 228 super(purgeable_logdir, self).__init__(directory, additional_exclude) 229 self.additional_exclude = additional_exclude 230 231 def run(self, log_dir): 232 """Copies this log dir to the destination dir, then purges the source. 233 234 @param log_dir: The destination log directory. 235 """ 236 super(purgeable_logdir, self).run(log_dir) 237 238 if os.path.exists(self.dir): 239 utils.system("rm -rf %s/*" % (self.dir)) 240 241 242 class site_sysinfo(base_sysinfo.base_sysinfo): 243 """Represents site system info.""" 244 def __init__(self, job_resultsdir): 245 super(site_sysinfo, self).__init__(job_resultsdir) 246 crash_exclude_string = None 247 if not collect_corefiles: 248 crash_exclude_string = "*.core" 249 250 # This is added in before and after_iteration_loggables. When run is 251 # called in before_iteration_loggables, it collects file status in 252 # the directory. When run is called in after_iteration_loggables, diff 253 # is executed. 254 # self.diffable_loggables is only initialized if the instance does not 255 # have this attribute yet. The sysinfo instance could be loaded 256 # from an earlier pickle dump, which has already initialized attribute 257 # self.diffable_loggables. 258 if not hasattr(self, 'diffable_loggables'): 259 diffable_log = diffable_logdir(constants.LOG_DIR) 260 self.diffable_loggables = set() 261 self.diffable_loggables.add(diffable_log) 262 263 # add in some extra command logging 264 self.boot_loggables.add(command("ls -l /boot", 265 "boot_file_list")) 266 self.before_iteration_loggables.add( 267 command(constants.CHROME_VERSION_COMMAND, "chrome_version")) 268 self.boot_loggables.add(command("crossystem", "crossystem")) 269 self.test_loggables.add( 270 purgeable_logdir( 271 os.path.join(constants.CRYPTOHOME_MOUNT_PT, "log"))) 272 # We only want to gather and purge crash reports after the client test 273 # runs in case a client test is checking that a crash found at boot 274 # (such as a kernel crash) is handled. 275 self.after_iteration_loggables.add( 276 purgeable_logdir( 277 os.path.join(constants.CRYPTOHOME_MOUNT_PT, "crash"), 278 additional_exclude=crash_exclude_string)) 279 self.after_iteration_loggables.add( 280 purgeable_logdir(constants.CRASH_DIR, 281 additional_exclude=crash_exclude_string)) 282 self.test_loggables.add( 283 logfile(os.path.join(constants.USER_DATA_DIR, 284 ".Google/Google Talk Plugin/gtbplugin.log"))) 285 self.test_loggables.add(purgeable_logdir( 286 constants.CRASH_DIR, 287 additional_exclude=crash_exclude_string)) 288 # Collect files under /tmp/crash_reporter, which contain the procfs 289 # copy of those crashed processes whose core file didn't get converted 290 # into minidump. We need these additional files for post-mortem analysis 291 # of the conversion failure. 292 self.test_loggables.add( 293 purgeable_logdir(constants.CRASH_REPORTER_RESIDUE_DIR)) 294 295 296 @log.log_and_ignore_errors("pre-test sysinfo error:") 297 def log_before_each_test(self, test): 298 """Logging hook called before a test starts. 299 300 @param test: A test object. 301 """ 302 super(site_sysinfo, self).log_before_each_test(test) 303 304 for log in self.diffable_loggables: 305 log.run(log_dir=None, collect_init_status=True) 306 307 # Start each log with the board name for orientation. 308 logging.info("ChromeOS BOARD = %s", 309 utils.get_board_with_frequency_and_memory()) 310 311 @log.log_and_ignore_errors("post-test sysinfo error:") 312 def log_after_each_test(self, test): 313 """Logging hook called after a test finishs. 314 315 @param test: A test object. 316 """ 317 super(site_sysinfo, self).log_after_each_test(test) 318 319 test_sysinfodir = self._get_sysinfodir(test.outputdir) 320 321 for log in self.diffable_loggables: 322 log.run(log_dir=test_sysinfodir, collect_init_status=False, 323 collect_all=not test.success) 324 325 326 def _get_chrome_version(self): 327 """Gets the Chrome version number and milestone as strings. 328 329 Invokes "chrome --version" to get the version number and milestone. 330 331 @return A tuple (chrome_ver, milestone) where "chrome_ver" is the 332 current Chrome version number as a string (in the form "W.X.Y.Z") 333 and "milestone" is the first component of the version number 334 (the "W" from "W.X.Y.Z"). If the version number cannot be parsed 335 in the "W.X.Y.Z" format, the "chrome_ver" will be the full output 336 of "chrome --version" and the milestone will be the empty string. 337 338 """ 339 version_string = utils.system_output(constants.CHROME_VERSION_COMMAND, 340 ignore_status=True) 341 return utils.parse_chrome_version(version_string) 342 343 344 def log_test_keyvals(self, test_sysinfodir): 345 keyval = super(site_sysinfo, self).log_test_keyvals(test_sysinfodir) 346 347 lsb_lines = utils.system_output( 348 "cat /etc/lsb-release", 349 ignore_status=True).splitlines() 350 lsb_dict = dict(item.split("=") for item in lsb_lines) 351 352 for lsb_key in lsb_dict.keys(): 353 # Special handling for build number 354 if lsb_key == "CHROMEOS_RELEASE_DESCRIPTION": 355 keyval["CHROMEOS_BUILD"] = ( 356 lsb_dict[lsb_key].rstrip(")").split(" ")[3]) 357 keyval[lsb_key] = lsb_dict[lsb_key] 358 359 # Get the hwid (hardware ID), if applicable. 360 try: 361 keyval["hwid"] = utils.system_output('crossystem hwid') 362 except error.CmdError: 363 # The hwid may not be available (e.g, when running on a VM). 364 # If the output of 'crossystem mainfw_type' is 'nonchrome', then 365 # we expect the hwid to not be avilable, and we can proceed in this 366 # case. Otherwise, the hwid is missing unexpectedly. 367 mainfw_type = utils.system_output('crossystem mainfw_type') 368 if mainfw_type == 'nonchrome': 369 logging.info( 370 'HWID not available; not logging it as a test keyval.') 371 else: 372 logging.exception('HWID expected but could not be identified; ' 373 'output of "crossystem mainfw_type" is "%s"', 374 mainfw_type) 375 raise 376 377 # Get the chrome version and milestone numbers. 378 keyval["CHROME_VERSION"], keyval["MILESTONE"] = ( 379 self._get_chrome_version()) 380 381 # Get the dictionary attack counter. 382 keyval["TPM_DICTIONARY_ATTACK_COUNTER"] = ( 383 tpm_dam.get_dictionary_attack_counter()) 384 385 # Return the updated keyvals. 386 return keyval 387 388 389 def add_logdir(self, log_path): 390 """Collect files in log_path to sysinfo folder. 391 392 This method can be called from a control file for test to collect files 393 in a specified folder. autotest creates a folder 394 [test result dir]/sysinfo folder with the full path of log_path and copy 395 all files in log_path to that folder. 396 397 @param log_path: Full path of a folder that test needs to collect files 398 from, e.g., 399 /mnt/stateful_partition/unencrypted/preserve/log 400 """ 401 self.test_loggables.add(logdir(log_path)) 402
