Home | History | Annotate | Download | only in toolchain-utils 
      1 #!/usr/bin/env python2
      2 """A crontab script to delete night test data."""
      3 
      4 from __future__ import print_function
      5 
      6 __author__ = 'shenhan (at] google.com (Han Shen)'
      7 
      8 import argparse
      9 import datetime
     10 import os
     11 import re
     12 import sys
     13 
     14 from cros_utils import command_executer
     15 from cros_utils import constants
     16 from cros_utils import misc
     17 
     18 DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
     19 
     20 
     21 def CleanNumberedDir(s, dry_run=False):
     22   """Deleted directories under each dated_dir."""
     23   chromeos_dirs = [
     24       os.path.join(s, x) for x in os.listdir(s)
     25       if misc.IsChromeOsTree(os.path.join(s, x))
     26   ]
     27   ce = command_executer.GetCommandExecuter(log_level='none')
     28   all_succeeded = True
     29   for cd in chromeos_dirs:
     30     if misc.DeleteChromeOsTree(cd, dry_run=dry_run):
     31       print('Successfully removed chromeos tree "{0}".'.format(cd))
     32     else:
     33       all_succeeded = False
     34       print('Failed to remove chromeos tree "{0}", please check.'.format(cd))
     35 
     36   if not all_succeeded:
     37     print('Failed to delete at least one chromeos tree, please check.')
     38     return False
     39 
     40   ## Now delete the numbered dir Before forcibly removing the directory, just
     41   ## check 's' to make sure it is sane.  A valid dir to be removed must be
     42   ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'.
     43   valid_dir_pattern = (
     44       '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')')
     45   if not re.search(valid_dir_pattern, s):
     46     print('Trying to delete an invalid dir "{0}" (must match "{1}"), '
     47           'please check.'.format(s, valid_dir_pattern))
     48     return False
     49 
     50   cmd = 'rm -fr {0}'.format(s)
     51   if dry_run:
     52     print(cmd)
     53   else:
     54     if ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) == 0:
     55       print('Successfully removed "{0}".'.format(s))
     56     else:
     57       all_succeeded = False
     58       print('Failed to remove "{0}", please check.'.format(s))
     59   return all_succeeded
     60 
     61 
     62 def CleanDatedDir(dated_dir, dry_run=False):
     63   # List subdirs under dir
     64   subdirs = [
     65       os.path.join(dated_dir, x) for x in os.listdir(dated_dir)
     66       if os.path.isdir(os.path.join(dated_dir, x))
     67   ]
     68   all_succeeded = True
     69   for s in subdirs:
     70     if not CleanNumberedDir(s, dry_run):
     71       all_succeeded = False
     72   return all_succeeded
     73 
     74 
     75 def ProcessArguments(argv):
     76   """Process arguments."""
     77   parser = argparse.ArgumentParser(
     78       description='Automatically delete nightly test data directories.',
     79       usage='auto_delete_nightly_test_data.py options')
     80   parser.add_argument(
     81       '-d',
     82       '--dry_run',
     83       dest='dry_run',
     84       default=False,
     85       action='store_true',
     86       help='Only print command line, do not execute anything.')
     87   parser.add_argument(
     88       '--days_to_preserve',
     89       dest='days_to_preserve',
     90       default=3,
     91       help=('Specify the number of days (not including today),'
     92             ' test data generated on these days will *NOT* be '
     93             'deleted. Defaults to 3.'))
     94   options = parser.parse_args(argv)
     95   return options
     96 
     97 
     98 def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run):
     99   rv = 0
    100   ce = command_executer.GetCommandExecuter()
    101   # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last
    102   # accessed more than specified time.
    103   minutes = 1440 * days_to_preserve
    104   cmd = (r'find {0} -maxdepth 1 -type d '
    105          r'\( -name "test_that_*"  -amin +{1} -o '
    106          r'   -name "cros-update*" -amin +{1} -o '
    107          r' -regex "{0}/tmp......" -amin +{1} \) '
    108          r'-exec bash -c "echo rm -fr {{}}" \; '
    109          r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, minutes)
    110   if dry_run:
    111     print('Going to execute:\n%s' % cmd)
    112   else:
    113     rv = ce.RunCommand(cmd, print_to_console=False)
    114     if rv == 0:
    115       print('Successfully cleaned chromeos tree tmp directory '
    116             '"{0}".'.format(chroot_tmp))
    117     else:
    118       print('Some directories were not removed under chromeos tree '
    119             'tmp directory -"{0}".'.format(chroot_tmp))
    120 
    121   return rv
    122 
    123 
    124 def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve,
    125                             dry_run):
    126   rv = 0
    127   rv2 = 0
    128   ce = command_executer.GetCommandExecuter()
    129   minutes = 1440 * days_to_preserve
    130   # Clean files that were last accessed more than the specified time.
    131   rv2 = 0
    132   cmd = (r'find {0}/*{1}/* -maxdepth 1 -type d '
    133          r'-amin +{2} '
    134          r'-exec bash -c "echo rm -fr {{}}" \; '
    135          r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, subdir_suffix,
    136                                                    minutes)
    137   if dry_run:
    138     print('Going to execute:\n%s' % cmd)
    139   else:
    140     rv2 = ce.RunCommand(cmd, print_to_console=False)
    141     if rv2 == 0:
    142       print('Successfully cleaned chromeos image autotest directories from '
    143             '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
    144     else:
    145       print('Some image autotest directories were not removed from '
    146             '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix))
    147 
    148   rv += rv2
    149   return rv
    150 
    151 
    152 def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False):
    153   """Delete temporaries, images under crostc/chromeos."""
    154   chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos',
    155                                      'chroot', 'tmp')
    156   # Clean files in tmp directory
    157   rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run)
    158   # Clean image files in *-tryjob directories
    159   rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-tryjob',
    160                                 days_to_preserve, dry_run)
    161   # Clean image files in *-release directories
    162   rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-release',
    163                                 days_to_preserve, dry_run)
    164   # Clean image files in *-pfq directories
    165   rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve,
    166                                 dry_run)
    167 
    168   return rv
    169 
    170 
    171 def Main(argv):
    172   """Delete nightly test data directories, tmps and test images."""
    173   options = ProcessArguments(argv)
    174   # Function 'isoweekday' returns 1(Monday) - 7 (Sunday).
    175   d = datetime.datetime.today().isoweekday()
    176   # We go back 1 week, delete from that day till we are
    177   # options.days_to_preserve away from today.
    178   s = d - 7
    179   e = d - int(options.days_to_preserve)
    180   rv = 0
    181   for i in range(s + 1, e):
    182     if i <= 0:
    183       ## Wrap around if index is negative.  6 is from i + 7 - 1, because
    184       ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7.
    185       dated_dir = DIR_BY_WEEKDAY[i + 6]
    186     else:
    187       dated_dir = DIR_BY_WEEKDAY[i - 1]
    188 
    189     rv += 0 if CleanDatedDir(
    190         os.path.join(constants.CROSTC_WORKSPACE, dated_dir),
    191         options.dry_run) else 1
    192 
    193 ## Finally clean temporaries, images under crostc/chromeos
    194   rv2 = CleanChromeOsTmpAndImages(
    195       int(options.days_to_preserve), options.dry_run)
    196 
    197   return rv + rv2
    198 
    199 if __name__ == '__main__':
    200   retval = Main(sys.argv[1:])
    201   sys.exit(retval)
    202