1 """This module gives the mkfs creation options for an existing filesystem. 2 3 tune2fs or xfs_growfs is called according to the filesystem. The results, 4 filesystem tunables, are parsed and mapped to corresponding mkfs options. 5 """ 6 import os, re, tempfile 7 import common 8 from autotest_lib.client.common_lib import error, utils 9 10 11 def opt_string2dict(opt_string): 12 """Breaks the mkfs.ext* option string into dictionary.""" 13 # Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces. 14 opt_dict = {} 15 16 for item in opt_string.split('-'): 17 item = item.strip() 18 if ' ' in item: 19 (opt, value) = item.split(' ', 1) 20 opt_dict['-%s' % opt] = value 21 elif item != '': 22 opt_dict['-%s' % item] = None 23 # Convert all the digit strings to int. 24 for key, value in opt_dict.iteritems(): 25 if value and value.isdigit(): 26 opt_dict[key] = int(value) 27 28 return opt_dict 29 30 31 def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'): 32 """Parses mke2fs config file for default settings.""" 33 # Please see /ect/mke2fs.conf for an example. 34 default_opt = {} 35 fs_opt = {} 36 current_fs_type = '' 37 current_section = '' 38 f = open(conf_file, 'r') 39 for line in f: 40 if '[defaults]' == line.strip(): 41 current_section = '[defaults]' 42 elif '[fs_types]' == line.strip(): 43 current_section = '[fs_types]' 44 elif current_section == '[defaults]': 45 components = line.split('=', 1) 46 if len(components) == 2: 47 default_opt[components[0].strip()] = components[1].strip() 48 elif current_section == '[fs_types]': 49 m = re.search('(\w+) = {', line) 50 if m: 51 current_fs_type = m.group(1) 52 else: 53 components = line.split('=', 1) 54 if len(components) == 2 and current_fs_type == fs_type: 55 default_opt[components[0].strip()] = components[1].strip() 56 f.close() 57 58 # fs_types options override the defaults options 59 for key, value in fs_opt.iteritems(): 60 default_opt[key] = value 61 62 # Convert all the digit strings to int. 63 for key, value in default_opt.iteritems(): 64 if value and value.isdigit(): 65 default_opt[key] = int(value) 66 67 return default_opt 68 69 70 def convert_conf_opt(default_opt): 71 conf_opt_mapping = {'blocksize': '-b', 72 'inode_ratio': '-i', 73 'inode_size': '-I'} 74 mkfs_opt = {} 75 76 # Here we simply concatenate the feature string while we really need 77 # to do the better and/or operations. 78 if 'base_features' in default_opt: 79 mkfs_opt['-O'] = default_opt['base_features'] 80 if 'default_features' in default_opt: 81 mkfs_opt['-O'] += ',%s' % default_opt['default_features'] 82 if 'features' in default_opt: 83 mkfs_opt['-O'] += ',%s' % default_opt['features'] 84 85 for key, value in conf_opt_mapping.iteritems(): 86 if key in default_opt: 87 mkfs_opt[value] = default_opt[key] 88 89 if '-O' in mkfs_opt: 90 mkfs_opt['-O'] = mkfs_opt['-O'].split(',') 91 92 return mkfs_opt 93 94 95 def merge_ext_features(conf_feature, user_feature): 96 user_feature_list = user_feature.split(',') 97 98 merged_feature = [] 99 # Removes duplicate entries in conf_list. 100 for item in conf_feature: 101 if item not in merged_feature: 102 merged_feature.append(item) 103 104 # User options override config options. 105 for item in user_feature_list: 106 if item[0] == '^': 107 if item[1:] in merged_feature: 108 merged_feature.remove(item[1:]) 109 else: 110 merged_feature.append(item) 111 elif item not in merged_feature: 112 merged_feature.append(item) 113 return merged_feature 114 115 116 def ext_tunables(dev): 117 """Call tune2fs -l and parse the result.""" 118 cmd = 'tune2fs -l %s' % dev 119 try: 120 out = utils.system_output(cmd) 121 except error.CmdError: 122 tools_dir = os.path.join(os.environ['AUTODIR'], 'tools') 123 cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev) 124 out = utils.system_output(cmd) 125 # Load option mappings 126 tune2fs_dict = {} 127 for line in out.splitlines(): 128 components = line.split(':', 1) 129 if len(components) == 2: 130 value = components[1].strip() 131 option = components[0] 132 if value.isdigit(): 133 tune2fs_dict[option] = int(value) 134 else: 135 tune2fs_dict[option] = value 136 137 return tune2fs_dict 138 139 140 def ext_mkfs_options(tune2fs_dict, mkfs_option): 141 """Map the tune2fs options to mkfs options.""" 142 143 def __inode_count(tune_dict, k): 144 return (tune_dict['Block count']/tune_dict[k] + 1) * ( 145 tune_dict['Block size']) 146 147 def __block_count(tune_dict, k): 148 return int(100*tune_dict[k]/tune_dict['Block count'] + 1) 149 150 def __volume_name(tune_dict, k): 151 if tune_dict[k] != '<none>': 152 return tune_dict[k] 153 else: 154 return '' 155 156 # mappings between fs features and mkfs options 157 ext_mapping = {'Blocks per group': '-g', 158 'Block size': '-b', 159 'Filesystem features': '-O', 160 'Filesystem OS type': '-o', 161 'Filesystem revision #': '-r', 162 'Filesystem volume name': '-L', 163 'Flex block group size': '-G', 164 'Fragment size': '-f', 165 'Inode count': '-i', 166 'Inode size': '-I', 167 'Journal inode': '-j', 168 'Reserved block count': '-m'} 169 170 conversions = { 171 'Journal inode': lambda d, k: None, 172 'Filesystem volume name': __volume_name, 173 'Reserved block count': __block_count, 174 'Inode count': __inode_count, 175 'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]), 176 'Filesystem revision #': lambda d, k: d[k][0]} 177 178 for key, value in ext_mapping.iteritems(): 179 if key not in tune2fs_dict: 180 continue 181 if key in conversions: 182 mkfs_option[value] = conversions[key](tune2fs_dict, key) 183 else: 184 mkfs_option[value] = tune2fs_dict[key] 185 186 187 def xfs_tunables(dev): 188 """Call xfs_grow -n to get filesystem tunables.""" 189 # Have to mount the filesystem to call xfs_grow. 190 tmp_mount_dir = tempfile.mkdtemp() 191 cmd = 'mount %s %s' % (dev, tmp_mount_dir) 192 utils.system_output(cmd) 193 xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs') 194 cmd = '%s -n %s' % (xfs_growfs, dev) 195 try: 196 out = utils.system_output(cmd) 197 finally: 198 # Clean. 199 cmd = 'umount %s' % dev 200 utils.system_output(cmd, ignore_status=True) 201 os.rmdir(tmp_mount_dir) 202 203 ## The output format is given in report_info (xfs_growfs.c) 204 ## "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n" 205 ## " =%-22s sectsz=%-5u attr=%u\n" 206 ## "data =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n" 207 ## " =%-22s sunit=%-6u swidth=%u blks\n" 208 ## "naming =version %-14u bsize=%-6u\n" 209 ## "log =%-22s bsize=%-6u blocks=%u, version=%u\n" 210 ## " =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n" 211 ## "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n" 212 213 tune2fs_dict = {} 214 # Flag for extracting naming version number 215 keep_version = False 216 for line in out.splitlines(): 217 m = re.search('^([-\w]+)', line) 218 if m: 219 main_tag = m.group(1) 220 pairs = line.split() 221 for pair in pairs: 222 # naming: version needs special treatment 223 if pair == '=version': 224 # 1 means the next pair is the version number we want 225 keep_version = True 226 continue 227 if keep_version: 228 tune2fs_dict['naming: version'] = pair 229 # Resets the flag since we have logged the version 230 keep_version = False 231 continue 232 # Ignores the strings without '=', such as 'blks' 233 if '=' not in pair: 234 continue 235 key, value = pair.split('=') 236 tagged_key = '%s: %s' % (main_tag, key) 237 if re.match('[0-9]+', value): 238 tune2fs_dict[tagged_key] = int(value.rstrip(',')) 239 else: 240 tune2fs_dict[tagged_key] = value.rstrip(',') 241 242 return tune2fs_dict 243 244 245 def xfs_mkfs_options(tune2fs_dict, mkfs_option): 246 """Maps filesystem tunables to their corresponding mkfs options.""" 247 248 # Mappings 249 xfs_mapping = {'meta-data: isize': '-i size', 250 'meta-data: agcount': '-d agcount', 251 'meta-data: sectsz': '-s size', 252 'meta-data: attr': '-i attr', 253 'data: bsize': '-b size', 254 'data: imaxpct': '-i maxpct', 255 'data: sunit': '-d sunit', 256 'data: swidth': '-d swidth', 257 'data: unwritten': '-d unwritten', 258 'naming: version': '-n version', 259 'naming: bsize': '-n size', 260 'log: version': '-l version', 261 'log: sectsz': '-l sectsize', 262 'log: sunit': '-l sunit', 263 'log: lazy-count': '-l lazy-count', 264 'realtime: extsz': '-r extsize', 265 'realtime: blocks': '-r size', 266 'realtime: rtextents': '-r rtdev'} 267 268 mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * ( 269 tune2fs_dict['log: blocks']) 270 271 for key, value in xfs_mapping.iteritems(): 272 mkfs_option[value] = tune2fs_dict[key] 273 274 275 def compare_features(needed_feature, current_feature): 276 """Compare two ext* feature lists.""" 277 if len(needed_feature) != len(current_feature): 278 return False 279 for feature in current_feature: 280 if feature not in needed_feature: 281 return False 282 return True 283 284 285 def match_ext_options(fs_type, dev, needed_options): 286 """Compare the current ext* filesystem tunables with needed ones.""" 287 # mkfs.ext* will load default options from /etc/mke2fs.conf 288 conf_opt = parse_mke2fs_conf(fs_type) 289 # We need to convert the conf options to mkfs options. 290 conf_mkfs_opt = convert_conf_opt(conf_opt) 291 # Breaks user mkfs option string to dictionary. 292 needed_opt_dict = opt_string2dict(needed_options) 293 # Removes ignored options. 294 ignored_option = ['-c', '-q', '-E', '-F'] 295 for opt in ignored_option: 296 if opt in needed_opt_dict: 297 del needed_opt_dict[opt] 298 299 # User options override config options. 300 needed_opt = conf_mkfs_opt 301 for key, value in needed_opt_dict.iteritems(): 302 if key == '-N' or key == '-T': 303 raise Exception('-N/T is not allowed.') 304 elif key == '-O': 305 needed_opt[key] = merge_ext_features(needed_opt[key], value) 306 else: 307 needed_opt[key] = value 308 309 # '-j' option will add 'has_journal' feature. 310 if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']: 311 needed_opt['-O'].append('has_journal') 312 # 'extents' will be shown as 'extent' in the outcome of tune2fs 313 if 'extents' in needed_opt['-O']: 314 needed_opt['-O'].append('extent') 315 needed_opt['-O'].remove('extents') 316 # large_file is a byproduct of resize_inode. 317 if 'large_file' not in needed_opt['-O'] and ( 318 'resize_inode' in needed_opt['-O']): 319 needed_opt['-O'].append('large_file') 320 321 current_opt = {} 322 tune2fs_dict = ext_tunables(dev) 323 ext_mkfs_options(tune2fs_dict, current_opt) 324 325 # Does the match 326 for key, value in needed_opt.iteritems(): 327 if key == '-O': 328 if not compare_features(value, current_opt[key].split(',')): 329 return False 330 elif key not in current_opt or value != current_opt[key]: 331 return False 332 return True 333 334 335 def match_xfs_options(dev, needed_options): 336 """Compare the current ext* filesystem tunables with needed ones.""" 337 tmp_mount_dir = tempfile.mkdtemp() 338 cmd = 'mount %s %s' % (dev, tmp_mount_dir) 339 utils.system_output(cmd) 340 xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs') 341 cmd = '%s -n %s' % (xfs_growfs, dev) 342 try: 343 current_option = utils.system_output(cmd) 344 finally: 345 # Clean. 346 cmd = 'umount %s' % dev 347 utils.system_output(cmd, ignore_status=True) 348 os.rmdir(tmp_mount_dir) 349 350 # '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details. 351 cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev) 352 needed_out = utils.system_output(cmd) 353 # 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n' 354 needed_out = re.sub('internal log', 'internal ', needed_out) 355 if current_option == needed_out: 356 return True 357 else: 358 return False 359 360 361 def match_mkfs_option(fs_type, dev, needed_options): 362 """Compare the current filesystem tunables with needed ones.""" 363 if fs_type.startswith('ext'): 364 ret = match_ext_options(fs_type, dev, needed_options) 365 elif fs_type == 'xfs': 366 ret = match_xfs_options(dev, needed_options) 367 else: 368 ret = False 369 370 return ret 371
