1 #!/usr/bin/python -u 2 3 """ 4 Utility to upload or remove the packages from the packages repository. 5 """ 6 7 import logging, optparse, os, shutil, sys, tempfile 8 import common 9 from autotest_lib.client.common_lib import utils as client_utils 10 from autotest_lib.client.common_lib import global_config, error 11 from autotest_lib.client.common_lib import packages 12 from autotest_lib.server import utils as server_utils 13 14 c = global_config.global_config 15 logging.basicConfig(level=logging.DEBUG) 16 17 ACTION_REMOVE = 'remove' 18 ACTION_UPLOAD = 'upload' 19 ACTION_TAR_ONLY = 'tar_only' 20 21 def get_exclude_string(client_dir): 22 ''' 23 Get the exclude string for the tar command to exclude specific 24 subdirectories inside client_dir. 25 For profilers we need to exclude everything except the __init__.py 26 file so that the profilers can be imported. 27 ''' 28 exclude_string = ('--exclude=deps/* --exclude=tests/* ' 29 '--exclude=site_tests/* --exclude=**.pyc') 30 31 # Get the profilers directory 32 prof_dir = os.path.join(client_dir, 'profilers') 33 34 # Include the __init__.py file for the profilers and exclude all its 35 # subdirectories 36 for f in os.listdir(prof_dir): 37 if os.path.isdir(os.path.join(prof_dir, f)): 38 exclude_string += ' --exclude=profilers/%s' % f 39 40 # The '.' here is needed to zip the files in the current 41 # directory. We use '-C' for tar to change to the required 42 # directory i.e. src_dir and then zip up the files in that 43 # directory(which is '.') excluding the ones in the exclude_dirs 44 exclude_string += " ." 45 46 # TODO(milleral): This is sad and ugly. http://crbug.com/258161 47 # Surprisingly, |exclude_string| actually means argument list, and 48 # we'd like to package up the current global_config.ini also, so let's 49 # just tack it on here. 50 # Also note that this only works because tar prevents us from un-tarring 51 # files into parent directories. 52 exclude_string += " ../global_config.ini" 53 54 return exclude_string 55 56 57 def parse_args(): 58 parser = optparse.OptionParser() 59 parser.add_option("-d", "--dependency", help="package the dependency" 60 " from client/deps directory and upload to the repo", 61 dest="dep") 62 parser.add_option("-p", "--profiler", help="package the profiler " 63 "from client/profilers directory and upload to the repo", 64 dest="prof") 65 parser.add_option("-t", "--test", help="package the test from client/tests" 66 " or client/site_tests and upload to the repo.", 67 dest="test") 68 parser.add_option("-c", "--client", help="package the client " 69 "directory alone without the tests, deps and profilers", 70 dest="client", action="store_true", default=False) 71 parser.add_option("-f", "--file", help="simply uploads the specified" 72 "file on to the repo", dest="file") 73 parser.add_option("-r", "--repository", help="the URL of the packages" 74 "repository location to upload the packages to.", 75 dest="repo", default=None) 76 parser.add_option("-o", "--output_dir", help="the output directory" 77 "to place tarballs and md5sum files in.", 78 dest="output_dir", default=None) 79 parser.add_option("-a", "--action", help="the action to perform", 80 dest="action", choices=(ACTION_UPLOAD, ACTION_REMOVE, 81 ACTION_TAR_ONLY), default=None) 82 parser.add_option("--all", help="Upload all the files locally " 83 "to all the repos specified in global_config.ini. " 84 "(includes the client, tests, deps and profilers)", 85 dest="all", action="store_true", default=False) 86 87 options, args = parser.parse_args() 88 return options, args 89 90 def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, 91 action, dest_dir=None): 92 """Method to upload or remove package depending on the flag passed to it. 93 94 If tar_only is set to True, this routine is solely used to generate a 95 tarball and compute the md5sum from that tarball. 96 If the tar_only flag is True, then the remove flag is ignored. 97 """ 98 exclude_string = ' .' 99 names = [p.strip() for p in pkg_names.split(',')] 100 for name in names: 101 print "Processing %s ... " % name 102 if pkg_type == 'client': 103 pkg_dir = src_dir 104 exclude_string = get_exclude_string(pkg_dir) 105 elif pkg_type == 'test': 106 # if the package is a test then look whether it is in client/tests 107 # or client/site_tests 108 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 109 else: 110 # for the profilers and deps 111 pkg_dir = os.path.join(src_dir, name) 112 113 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 114 115 if action == ACTION_TAR_ONLY: 116 # If the package is a test, then the work-dir should have 'client' 117 # appended to it. 118 base_test_dir = os.path.join(dest_dir, 'client') 119 build_dir = os.path.join(get_test_dir(name, base_test_dir), name) 120 121 try: 122 packages.check_diskspace(build_dir) 123 except error.RepoDiskFullError as e: 124 msg = ("Work_dir directory for packages %s does not have " 125 "enough space available: %s" % (build_dir, e)) 126 raise error.RepoDiskFullError(msg) 127 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 128 build_dir, exclude_string) 129 130 # Create the md5 hash too. 131 md5sum = pkgmgr.compute_checksum(tarball_path) 132 md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum') 133 with open(md5sum_filepath, "w") as f: 134 f.write(md5sum) 135 136 elif action == ACTION_UPLOAD: 137 # Tar the source and upload 138 temp_dir = tempfile.mkdtemp() 139 try: 140 try: 141 packages.check_diskspace(temp_dir) 142 except error.RepoDiskFullError, e: 143 msg = ("Temporary directory for packages %s does not have " 144 "enough space available: %s" % (temp_dir, e)) 145 raise error.RepoDiskFullError(msg) 146 147 # Check if tarball already exists. If it does, don't duplicate 148 # the effort. 149 tarball_path = os.path.join(pkg_dir, pkg_name); 150 if os.path.exists(tarball_path): 151 print("Tarball %s already exists" % tarball_path); 152 else: 153 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 154 temp_dir, exclude_string) 155 pkgmgr.upload_pkg(tarball_path, update_checksum=True) 156 finally: 157 # remove the temporary directory 158 shutil.rmtree(temp_dir) 159 elif action == ACTION_REMOVE: 160 pkgmgr.remove_pkg(pkg_name, remove_checksum=True) 161 print "Done." 162 163 164 def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir): 165 """Tar all packages up and return a list of each tar created""" 166 tarballs = [] 167 exclude_string = ' .' 168 names = [p.strip() for p in pkg_names.split(',')] 169 for name in names: 170 print "Processing %s ... " % name 171 if pkg_type == 'client': 172 pkg_dir = src_dir 173 exclude_string = get_exclude_string(pkg_dir) 174 elif pkg_type == 'test': 175 # if the package is a test then look whether it is in client/tests 176 # or client/site_tests 177 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 178 else: 179 # for the profilers and deps 180 pkg_dir = os.path.join(src_dir, name) 181 182 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 183 184 # Check if tarball already exists. If it does, don't duplicate 185 # the effort. 186 tarball_path = os.path.join(pkg_dir, pkg_name); 187 if os.path.exists(tarball_path): 188 print("Tarball %s already exists" % tarball_path); 189 else: 190 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 191 temp_dir, exclude_string) 192 tarballs.append(tarball_path) 193 return tarballs 194 195 196 def process_all_packages(pkgmgr, client_dir, action): 197 """Process a full upload of packages as a directory upload.""" 198 dep_dir = os.path.join(client_dir, "deps") 199 prof_dir = os.path.join(client_dir, "profilers") 200 # Directory where all are kept 201 temp_dir = tempfile.mkdtemp() 202 try: 203 packages.check_diskspace(temp_dir) 204 except error.RepoDiskFullError, e: 205 print ("Temp destination for packages is full %s, aborting upload: %s" 206 % (temp_dir, e)) 207 os.rmdir(temp_dir) 208 sys.exit(1) 209 210 # process tests 211 tests_list = get_subdir_list('tests', client_dir) 212 tests = ','.join(tests_list) 213 214 # process site_tests 215 site_tests_list = get_subdir_list('site_tests', client_dir) 216 site_tests = ','.join(site_tests_list) 217 218 # process deps 219 deps_list = get_subdir_list('deps', client_dir) 220 deps = ','.join(deps_list) 221 222 # process profilers 223 profilers_list = get_subdir_list('profilers', client_dir) 224 profilers = ','.join(profilers_list) 225 226 # Update md5sum 227 if action == ACTION_UPLOAD: 228 all_packages = [] 229 all_packages.extend(tar_packages(pkgmgr, 'profiler', profilers, 230 prof_dir, temp_dir)) 231 all_packages.extend(tar_packages(pkgmgr, 'dep', deps, dep_dir, 232 temp_dir)) 233 all_packages.extend(tar_packages(pkgmgr, 'test', site_tests, 234 client_dir, temp_dir)) 235 all_packages.extend(tar_packages(pkgmgr, 'test', tests, client_dir, 236 temp_dir)) 237 all_packages.extend(tar_packages(pkgmgr, 'client', 'autotest', 238 client_dir, temp_dir)) 239 for package in all_packages: 240 pkgmgr.upload_pkg(package, update_checksum=True) 241 client_utils.run('rm -rf ' + temp_dir) 242 elif action == ACTION_REMOVE: 243 process_packages(pkgmgr, 'test', tests, client_dir, action=action) 244 process_packages(pkgmgr, 'test', site_tests, client_dir, action=action) 245 process_packages(pkgmgr, 'client', 'autotest', client_dir, 246 action=action) 247 process_packages(pkgmgr, 'dep', deps, dep_dir, action=action) 248 process_packages(pkgmgr, 'profiler', profilers, prof_dir, 249 action=action) 250 251 252 # Get the list of sub directories present in a directory 253 def get_subdir_list(name, client_dir): 254 dir_name = os.path.join(client_dir, name) 255 return [f for f in 256 os.listdir(dir_name) 257 if os.path.isdir(os.path.join(dir_name, f)) ] 258 259 260 # Look whether the test is present in client/tests and client/site_tests dirs 261 def get_test_dir(name, client_dir): 262 names_test = os.listdir(os.path.join(client_dir, 'tests')) 263 names_site_test = os.listdir(os.path.join(client_dir, 'site_tests')) 264 if name in names_test: 265 src_dir = os.path.join(client_dir, 'tests') 266 elif name in names_site_test: 267 src_dir = os.path.join(client_dir, 'site_tests') 268 else: 269 print "Test %s not found" % name 270 sys.exit(0) 271 return src_dir 272 273 274 def main(): 275 # get options and args 276 options, args = parse_args() 277 278 server_dir = server_utils.get_server_dir() 279 autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) 280 281 # extract the pkg locations from global config 282 repo_urls = c.get_config_value('PACKAGES', 'fetch_location', 283 type=list, default=[]) 284 upload_paths = c.get_config_value('PACKAGES', 'upload_location', 285 type=list, default=[]) 286 287 if options.repo: 288 upload_paths.append(options.repo) 289 290 # Having no upload paths basically means you're not using packaging. 291 if not upload_paths: 292 print("No upload locations found. Please set upload_location under" 293 " PACKAGES in the global_config.ini or provide a location using" 294 " the --repository option.") 295 return 296 297 client_dir = os.path.join(autotest_dir, "client") 298 299 # Bail out if the client directory does not exist 300 if not os.path.exists(client_dir): 301 sys.exit(0) 302 303 dep_dir = os.path.join(client_dir, "deps") 304 prof_dir = os.path.join(client_dir, "profilers") 305 306 # Due to the delayed uprev-ing of certain ebuilds, we need to support 307 # both the legacy command line and the new one. 308 # So, if the new "action" option isn't specified, try looking for the 309 # old style remove/upload argument 310 if options.action is None: 311 if len(args) == 0 or args[0] not in ['upload', 'remove']: 312 print("Either 'upload' or 'remove' needs to be specified " 313 "for the package") 314 sys.exit(0) 315 cur_action = args[0] 316 else: 317 cur_action = options.action 318 319 if cur_action == ACTION_TAR_ONLY and options.output_dir is None: 320 print("An output dir has to be specified") 321 sys.exit(0) 322 323 pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, 324 upload_paths=upload_paths, 325 run_function_dargs={'timeout':600}) 326 327 if options.all: 328 process_all_packages(pkgmgr, client_dir, action=cur_action) 329 330 if options.client: 331 process_packages(pkgmgr, 'client', 'autotest', client_dir, 332 action=cur_action) 333 334 if options.dep: 335 process_packages(pkgmgr, 'dep', options.dep, dep_dir, 336 action=cur_action) 337 338 if options.test: 339 process_packages(pkgmgr, 'test', options.test, client_dir, 340 action=cur_action, dest_dir=options.output_dir) 341 342 if options.prof: 343 process_packages(pkgmgr, 'profiler', options.prof, prof_dir, 344 action=cur_action) 345 346 if options.file: 347 if cur_action == ACTION_REMOVE: 348 pkgmgr.remove_pkg(options.file, remove_checksum=True) 349 elif cur_action == ACTION_UPLOAD: 350 pkgmgr.upload_pkg(options.file, update_checksum=True) 351 352 353 if __name__ == "__main__": 354 main() 355
