1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 8 from autotest_lib.client.common_lib import log 9 from autotest_lib.client.common_lib import error, utils, global_config 10 from autotest_lib.client.bin import base_sysinfo, utils 11 from autotest_lib.client.cros import constants 12 13 get_value = global_config.global_config.get_config_value 14 collect_corefiles = get_value('CLIENT', 'collect_corefiles', 15 type=bool, default=False) 16 17 18 logfile = base_sysinfo.logfile 19 command = base_sysinfo.command 20 21 22 class logdir(base_sysinfo.loggable): 23 """Represents a log directory.""" 24 def __init__(self, directory, additional_exclude=None): 25 super(logdir, self).__init__(directory, log_in_keyval=False) 26 self.dir = directory 27 self.additional_exclude = additional_exclude 28 29 30 def __repr__(self): 31 return "site_sysinfo.logdir(%r, %s)" % (self.dir, 32 self.additional_exclude) 33 34 35 def __eq__(self, other): 36 if isinstance(other, logdir): 37 return (self.dir == other.dir and 38 self.additional_exclude == other.additional_exclude) 39 elif isinstance(other, base_sysinfo.loggable): 40 return False 41 return NotImplemented 42 43 44 def __ne__(self, other): 45 result = self.__eq__(other) 46 if result is NotImplemented: 47 return result 48 return not result 49 50 51 def __hash__(self): 52 return hash(self.dir) + hash(self.additional_exclude) 53 54 55 def run(self, log_dir): 56 """Copies this log directory to the specified directory. 57 58 @param log_dir: The destination log directory. 59 """ 60 if os.path.exists(self.dir): 61 parent_dir = os.path.dirname(self.dir) 62 utils.system("mkdir -p %s%s" % (log_dir, parent_dir)) 63 # Take source permissions and add ugo+r so files are accessible via 64 # archive server. 65 additional_exclude_str = "" 66 if self.additional_exclude: 67 additional_exclude_str = "--exclude=" + self.additional_exclude 68 69 utils.system("rsync --no-perms --chmod=ugo+r -a --exclude=autoserv*" 70 " --safe-links" 71 " %s %s %s%s" % (additional_exclude_str, self.dir, 72 log_dir, parent_dir)) 73 74 75 class file_stat(object): 76 """Store the file size and inode, used for retrieving new data in file.""" 77 def __init__(self, file_path): 78 """Collect the size and inode information of a file. 79 80 @param file_path: full path to the file. 81 82 """ 83 stat = os.stat(file_path) 84 # Start size of the file, skip that amount of bytes when do diff. 85 self.st_size = stat.st_size 86 # inode of the file. If inode is changed, treat this as a new file and 87 # copy the whole file. 88 self.st_ino = stat.st_ino 89 90 91 class diffable_logdir(logdir): 92 """Represents a log directory that only new content will be copied. 93 94 An instance of this class should be added in both 95 before_iteration_loggables and after_iteration_loggables. This is to 96 guarantee the file status information is collected when run method is 97 called in before_iteration_loggables, and diff is executed when run 98 method is called in after_iteration_loggables. 99 100 """ 101 def __init__(self, directory, additional_exclude=None, 102 keep_file_hierarchy=True, append_diff_in_name=True): 103 """ 104 Constructor of a diffable_logdir instance. 105 106 @param directory: directory to be diffed after an iteration finished. 107 @param additional_exclude: additional dir to be excluded, not used. 108 @param keep_file_hierarchy: True if need to preserve full path, e.g., 109 sysinfo/var/log/sysstat, v.s. sysinfo/sysstat if it's False. 110 @param append_diff_in_name: True if you want to append '_diff' to the 111 folder name to indicate it's a diff, e.g., var/log_diff. Option 112 keep_file_hierarchy must be True for this to take effect. 113 114 """ 115 super(diffable_logdir, self).__init__(directory, additional_exclude) 116 self.additional_exclude = additional_exclude 117 self.keep_file_hierarchy = keep_file_hierarchy 118 self.append_diff_in_name = append_diff_in_name 119 # Init dictionary to store all file status for files in the directory. 120 self._log_stats = {} 121 122 123 def _get_init_status_of_src_dir(self, src_dir): 124 """Get initial status of files in src_dir folder. 125 126 @param src_dir: directory to be diff-ed. 127 128 """ 129 # Dictionary used to store the initial status of files in src_dir. 130 for file_path in self._get_all_files(src_dir): 131 self._log_stats[file_path] = file_stat(file_path) 132 self.file_stats_collected = True 133 134 135 def _get_all_files(self, path): 136 """Iterate through files in given path including subdirectories. 137 138 @param path: root directory. 139 @return: an iterator that iterates through all files in given path 140 including subdirectories. 141 142 """ 143 if not os.path.exists(path): 144 yield [] 145 for root, dirs, files in os.walk(path): 146 for f in files: 147 if f.startswith('autoserv'): 148 continue 149 yield os.path.join(root, f) 150 151 152 def _copy_new_data_in_file(self, file_path, src_dir, dest_dir): 153 """Copy all new data in a file to target directory. 154 155 @param file_path: full path to the file to be copied. 156 @param src_dir: source directory to do the diff. 157 @param dest_dir: target directory to store new data of src_dir. 158 159 """ 160 bytes_to_skip = 0 161 if self._log_stats.has_key(file_path): 162 prev_stat = self._log_stats[file_path] 163 new_stat = os.stat(file_path) 164 if new_stat.st_ino == prev_stat.st_ino: 165 bytes_to_skip = prev_stat.st_size 166 if new_stat.st_size == bytes_to_skip: 167 return 168 elif new_stat.st_size < prev_stat.st_size: 169 # File is modified to a smaller size, copy whole file. 170 bytes_to_skip = 0 171 try: 172 with open(file_path, 'r') as in_log: 173 if bytes_to_skip > 0: 174 in_log.seek(bytes_to_skip) 175 # Skip src_dir in path, e.g., src_dir/[sub_dir]/file_name. 176 target_path = os.path.join(dest_dir, 177 os.path.relpath(file_path, src_dir)) 178 target_dir = os.path.dirname(target_path) 179 if not os.path.exists(target_dir): 180 os.makedirs(target_dir) 181 with open(target_path, "w") as out_log: 182 out_log.write(in_log.read()) 183 except IOError as e: 184 logging.error('Diff %s failed with error: %s', file_path, e) 185 186 187 def _log_diff(self, src_dir, dest_dir): 188 """Log all of the new data in src_dir to dest_dir. 189 190 @param src_dir: source directory to do the diff. 191 @param dest_dir: target directory to store new data of src_dir. 192 193 """ 194 if self.keep_file_hierarchy: 195 dir = src_dir.lstrip('/') 196 if self.append_diff_in_name: 197 dir = dir.rstrip('/') + '_diff' 198 dest_dir = os.path.join(dest_dir, dir) 199 200 if not os.path.exists(dest_dir): 201 os.makedirs(dest_dir) 202 203 for src_file in self._get_all_files(src_dir): 204 self._copy_new_data_in_file(src_file, src_dir, dest_dir) 205 206 207 def run(self, log_dir, collect_init_status=True, collect_all=False): 208 """Copies new content from self.dir to the destination log_dir. 209 210 @param log_dir: The destination log directory. 211 @param collect_init_status: Set to True if run method is called to 212 collect the initial status of files. 213 @param collect_all: Set to True to force to collect all files. 214 215 """ 216 if collect_init_status: 217 self._get_init_status_of_src_dir(self.dir) 218 elif os.path.exists(self.dir): 219 # Always create a copy of the new logs to help debugging. 220 self._log_diff(self.dir, log_dir) 221 if collect_all: 222 logdir_temp = logdir(self.dir) 223 logdir_temp.run(log_dir) 224 225 226 class purgeable_logdir(logdir): 227 """Represents a log directory that will be purged.""" 228 def __init__(self, directory, additional_exclude=None): 229 super(purgeable_logdir, self).__init__(directory, additional_exclude) 230 self.additional_exclude = additional_exclude 231 232 def run(self, log_dir): 233 """Copies this log dir to the destination dir, then purges the source. 234 235 @param log_dir: The destination log directory. 236 """ 237 super(purgeable_logdir, self).run(log_dir) 238 239 if os.path.exists(self.dir): 240 utils.system("rm -rf %s/*" % (self.dir)) 241 242 243 class site_sysinfo(base_sysinfo.base_sysinfo): 244 """Represents site system info.""" 245 def __init__(self, job_resultsdir): 246 super(site_sysinfo, self).__init__(job_resultsdir) 247 crash_exclude_string = None 248 if not collect_corefiles: 249 crash_exclude_string = "*.core" 250 251 # This is added in before and after_iteration_loggables. When run is 252 # called in before_iteration_loggables, it collects file status in 253 # the directory. When run is called in after_iteration_loggables, diff 254 # is executed. 255 # self.diffable_loggables is only initialized if the instance does not 256 # have this attribute yet. The sysinfo instance could be loaded 257 # from an earlier pickle dump, which has already initialized attribute 258 # self.diffable_loggables. 259 if not hasattr(self, 'diffable_loggables'): 260 diffable_log = diffable_logdir(constants.LOG_DIR) 261 self.diffable_loggables = set() 262 self.diffable_loggables.add(diffable_log) 263 264 # add in some extra command logging 265 self.boot_loggables.add(command("ls -l /boot", 266 "boot_file_list")) 267 self.before_iteration_loggables.add( 268 command(constants.CHROME_VERSION_COMMAND, "chrome_version")) 269 self.boot_loggables.add(command("crossystem", "crossystem")) 270 self.test_loggables.add( 271 purgeable_logdir( 272 os.path.join(constants.CRYPTOHOME_MOUNT_PT, "log"))) 273 # We only want to gather and purge crash reports after the client test 274 # runs in case a client test is checking that a crash found at boot 275 # (such as a kernel crash) is handled. 276 self.after_iteration_loggables.add( 277 purgeable_logdir( 278 os.path.join(constants.CRYPTOHOME_MOUNT_PT, "crash"), 279 additional_exclude=crash_exclude_string)) 280 self.after_iteration_loggables.add( 281 purgeable_logdir(constants.CRASH_DIR, 282 additional_exclude=crash_exclude_string)) 283 self.test_loggables.add( 284 logfile(os.path.join(constants.USER_DATA_DIR, 285 ".Google/Google Talk Plugin/gtbplugin.log"))) 286 self.test_loggables.add(purgeable_logdir( 287 constants.CRASH_DIR, 288 additional_exclude=crash_exclude_string)) 289 # Collect files under /tmp/crash_reporter, which contain the procfs 290 # copy of those crashed processes whose core file didn't get converted 291 # into minidump. We need these additional files for post-mortem analysis 292 # of the conversion failure. 293 self.test_loggables.add( 294 purgeable_logdir(constants.CRASH_REPORTER_RESIDUE_DIR)) 295 296 297 @log.log_and_ignore_errors("pre-test sysinfo error:") 298 def log_before_each_test(self, test): 299 """Logging hook called before a test starts. 300 301 @param test: A test object. 302 """ 303 super(site_sysinfo, self).log_before_each_test(test) 304 305 for log in self.diffable_loggables: 306 log.run(log_dir=None, collect_init_status=True) 307 308 309 @log.log_and_ignore_errors("post-test sysinfo error:") 310 def log_after_each_test(self, test): 311 """Logging hook called after a test finishs. 312 313 @param test: A test object. 314 """ 315 super(site_sysinfo, self).log_after_each_test(test) 316 317 test_sysinfodir = self._get_sysinfodir(test.outputdir) 318 319 for log in self.diffable_loggables: 320 log.run(log_dir=test_sysinfodir, collect_init_status=False, 321 collect_all=not test.success) 322 323 324 def _get_chrome_version(self): 325 """Gets the Chrome version number and milestone as strings. 326 327 Invokes "chrome --version" to get the version number and milestone. 328 329 @return A tuple (chrome_ver, milestone) where "chrome_ver" is the 330 current Chrome version number as a string (in the form "W.X.Y.Z") 331 and "milestone" is the first component of the version number 332 (the "W" from "W.X.Y.Z"). If the version number cannot be parsed 333 in the "W.X.Y.Z" format, the "chrome_ver" will be the full output 334 of "chrome --version" and the milestone will be the empty string. 335 336 """ 337 version_string = utils.system_output(constants.CHROME_VERSION_COMMAND, 338 ignore_status=True) 339 return utils.parse_chrome_version(version_string) 340 341 342 def log_test_keyvals(self, test_sysinfodir): 343 keyval = super(site_sysinfo, self).log_test_keyvals(test_sysinfodir) 344 345 lsb_lines = utils.system_output( 346 "cat /etc/lsb-release", 347 ignore_status=True).splitlines() 348 lsb_dict = dict(item.split("=") for item in lsb_lines) 349 350 for lsb_key in lsb_dict.keys(): 351 # Special handling for build number 352 if lsb_key == "CHROMEOS_RELEASE_DESCRIPTION": 353 keyval["CHROMEOS_BUILD"] = ( 354 lsb_dict[lsb_key].rstrip(")").split(" ")[3]) 355 keyval[lsb_key] = lsb_dict[lsb_key] 356 357 # Get the hwid (hardware ID), if applicable. 358 try: 359 keyval["hwid"] = utils.system_output('crossystem hwid') 360 except error.CmdError: 361 # The hwid may not be available (e.g, when running on a VM). 362 # If the output of 'crossystem mainfw_type' is 'nonchrome', then 363 # we expect the hwid to not be avilable, and we can proceed in this 364 # case. Otherwise, the hwid is missing unexpectedly. 365 mainfw_type = utils.system_output('crossystem mainfw_type') 366 if mainfw_type == 'nonchrome': 367 logging.info( 368 'HWID not available; not logging it as a test keyval.') 369 else: 370 logging.exception('HWID expected but could not be identified; ' 371 'output of "crossystem mainfw_type" is "%s"', 372 mainfw_type) 373 raise 374 375 # Get the chrome version and milestone numbers. 376 keyval["CHROME_VERSION"], keyval["MILESTONE"] = ( 377 self._get_chrome_version()) 378 379 # TODO(kinaba): crbug.com/707448 Import at the head of this file. 380 # Currently a server-side script server/server_job.py is indirectly 381 # importing this file, so we cannot globaly import cryptohome that 382 # has dependency to a client-only library. 383 from autotest_lib.client.cros import cryptohome 384 # Get the dictionary attack counter. 385 keyval["TPM_DICTIONARY_ATTACK_COUNTER"] = ( 386 cryptohome.get_tpm_more_status().get( 387 'dictionary_attack_counter', 'Failed to query cryptohome')) 388 389 # Return the updated keyvals. 390 return keyval 391 392 393 def add_logdir(self, log_path): 394 """Collect files in log_path to sysinfo folder. 395 396 This method can be called from a control file for test to collect files 397 in a specified folder. autotest creates a folder 398 [test result dir]/sysinfo folder with the full path of log_path and copy 399 all files in log_path to that folder. 400 401 @param log_path: Full path of a folder that test needs to collect files 402 from, e.g., 403 /mnt/stateful_partition/unencrypted/preserve/log 404 """ 405 self.test_loggables.add(logdir(log_path)) 406
