1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 # 6 """Logic to generate lists of DEPS used by various parts of 7 the android_webview continuous integration (buildbot) infrastructure. 8 9 Note: The root Chromium project (which is not explicitly listed here) 10 has a couple of third_party libraries checked in directly into it. This means 11 that the list of third parties present in this file is not a comprehensive 12 list of third party android_webview dependencies. 13 """ 14 15 import argparse 16 import json 17 import logging 18 import os 19 import sys 20 21 # Add android_webview/tools to path to get at known_issues. 22 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools')) 23 import known_issues 24 25 26 class DepsWhitelist(object): 27 def __init__(self): 28 # If a new DEPS entry is needed for the AOSP bot to compile please add it 29 # here first. 30 # This is a staging area for deps that are accepted by the android_webview 31 # team and are in the process of having the required branches being created 32 # in the Android tree. 33 self._compile_but_not_snapshot_dependencies = [ 34 ] 35 36 # Dependencies that need to be merged into the Android tree. 37 self._snapshot_into_android_dependencies = [ 38 'sdch/open-vcdiff', 39 'testing/gtest', 40 'third_party/WebKit', 41 'third_party/angle', 42 'third_party/boringssl/src', 43 'third_party/brotli/src', 44 ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/' 45 'braille'), 46 'third_party/freetype', 47 'third_party/icu', 48 'third_party/leveldatabase/src', 49 'third_party/libaddressinput/src', 50 'third_party/libjingle/source/talk', 51 'third_party/libjpeg_turbo', 52 'third_party/libphonenumber/src/phonenumbers', 53 'third_party/libphonenumber/src/resources', 54 'third_party/libsrtp', 55 'third_party/libvpx', 56 'third_party/libyuv', 57 'third_party/mesa/src', 58 'third_party/openmax_dl', 59 'third_party/opus/src', 60 'third_party/ots', 61 'third_party/sfntly/cpp/src', 62 'third_party/skia', 63 'third_party/smhasher/src', 64 'third_party/usrsctp/usrsctplib', 65 'third_party/webrtc', 66 'third_party/yasm/source/patched-yasm', 67 'tools/grit', 68 'tools/gyp', 69 'v8', 70 ] 71 72 # We can save some time by not rsyncing code we don't use. 73 self._prune_from_rsync_build = [ 74 'third_party/WebKit/LayoutTests', 75 ] 76 77 # Dependencies required to build android_webview. 78 self._compile_dependencies = (self._snapshot_into_android_dependencies + 79 self._compile_but_not_snapshot_dependencies) 80 81 # Dependencies required to run android_webview tests but not required to 82 # compile. 83 self._test_data_dependencies = [ 84 'chrome/test/data/perf/third_party/octane', 85 ] 86 87 @staticmethod 88 def _read_deps_file(deps_file_path): 89 class FileImplStub(object): 90 """Stub for the File syntax.""" 91 def __init__(self, file_location): 92 pass 93 94 @staticmethod 95 def GetPath(): 96 return '' 97 98 @staticmethod 99 def GetFilename(): 100 return '' 101 102 @staticmethod 103 def GetRevision(): 104 return None 105 106 def from_stub(__, _=None): 107 """Stub for the From syntax.""" 108 return '' 109 110 class VarImpl(object): 111 def __init__(self, custom_vars, local_scope): 112 self._custom_vars = custom_vars 113 self._local_scope = local_scope 114 115 def Lookup(self, var_name): 116 """Implements the Var syntax.""" 117 if var_name in self._custom_vars: 118 return self._custom_vars[var_name] 119 elif var_name in self._local_scope.get("vars", {}): 120 return self._local_scope["vars"][var_name] 121 raise Exception("Var is not defined: %s" % var_name) 122 123 local_scope = {} 124 var = VarImpl({}, local_scope) 125 global_scope = { 126 'File': FileImplStub, 127 'From': from_stub, 128 'Var': var.Lookup, 129 'deps_os': {}, 130 } 131 execfile(deps_file_path, global_scope, local_scope) 132 deps = local_scope.get('deps', {}) 133 deps_os = local_scope.get('deps_os', {}) 134 for os_specific_deps in deps_os.itervalues(): 135 deps.update(os_specific_deps) 136 return deps.keys() 137 138 def _get_known_issues(self): 139 issues = [] 140 for root, paths in known_issues.KNOWN_INCOMPATIBLE.items(): 141 for path in paths: 142 issues.append(os.path.normpath(os.path.join(root, path))) 143 return issues 144 145 def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps): 146 """Calculates the list of deps that need to be excluded from the deps_file 147 so that the only deps left are the one in the whitelist.""" 148 all_deps = self._read_deps_file(deps_file_path) 149 # The list of deps read from the DEPS file are prefixed with the source 150 # tree root, which is 'src' for Chromium. 151 def prepend_root(path): 152 return os.path.join('src', path) 153 whitelisted_deps = map(prepend_root, whitelisted_deps) 154 deps_blacklist = set(all_deps).difference(set(whitelisted_deps)) 155 return dict(map(lambda(x): (x, None), deps_blacklist)) 156 157 def _make_blacklist(self, deps_file_path, whitelisted_deps): 158 """Calculates the list of paths we should exclude """ 159 all_deps = self._read_deps_file(deps_file_path) 160 def remove_src_prefix(path): 161 return path.replace('src/', '', 1) 162 all_deps = map(remove_src_prefix, all_deps) 163 # Ignore all deps except those whitelisted. 164 blacklist = set(all_deps).difference(whitelisted_deps) 165 # Ignore the 'known issues'. Typically these are the licence incompatible 166 # things checked directly into Chromium. 167 blacklist = blacklist.union(self._get_known_issues()) 168 # Ignore any other non-deps, non-licence paths we don't like. 169 blacklist = blacklist.union(self._prune_from_rsync_build) 170 return list(blacklist) 171 172 def get_deps_for_android_build(self, deps_file_path): 173 """This is used to calculate the custom_deps list for the Android bot. 174 """ 175 if not deps_file_path: 176 raise Exception('You need to specify a DEPS file path.') 177 return self._make_gclient_blacklist(deps_file_path, 178 self._compile_dependencies) 179 180 def get_deps_for_android_build_and_test(self, deps_file_path): 181 """This is used to calculate the custom_deps list for the Android perf bot. 182 """ 183 if not deps_file_path: 184 raise Exception('You need to specify a DEPS file path.') 185 return self._make_gclient_blacklist(deps_file_path, 186 self._compile_dependencies + 187 self._test_data_dependencies) 188 189 def get_blacklist_for_android_rsync_build(self, deps_file_path): 190 """Calculates the list of paths we should exclude when building Android 191 either because of license compatibility or because they are large and 192 uneeded. 193 """ 194 if not deps_file_path: 195 raise Exception('You need to specify a DEPS file path.') 196 return self._make_blacklist(deps_file_path, self._compile_dependencies) 197 198 def get_deps_for_android_merge(self, _): 199 """Calculates the list of deps that need to be merged into the Android tree 200 in order to build the C++ and Java android_webview code.""" 201 return self._snapshot_into_android_dependencies 202 203 def get_deps_for_license_check(self, _): 204 """Calculates the list of deps that need to be checked for Android license 205 compatibility""" 206 return self._compile_dependencies 207 208 209 def execute_method(self, method_name, deps_file_path): 210 methods = { 211 'android_build': self.get_deps_for_android_build, 212 'android_build_and_test': 213 self.get_deps_for_android_build_and_test, 214 'android_merge': self.get_deps_for_android_merge, 215 'license_check': self.get_deps_for_license_check, 216 'android_rsync_build': self.get_blacklist_for_android_rsync_build, 217 } 218 if not method_name in methods: 219 raise Exception('Method name %s is not valid. Valid choices are %s' % 220 (method_name, methods.keys())) 221 return methods[method_name](deps_file_path) 222 223 def main(): 224 parser = argparse.ArgumentParser() 225 parser.add_argument('--method', help='Method to use to fetch from whitelist.', 226 required=True) 227 parser.add_argument('--path-to-deps', help='Path to DEPS file.') 228 parser.add_argument('--output-json', help='Name of file to write output to.') 229 parser.add_argument('verbose', action='store_true', default=False) 230 opts = parser.parse_args() 231 232 logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN) 233 234 deps_whitelist = DepsWhitelist() 235 blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps) 236 237 if (opts.output_json): 238 output_dict = { 239 'blacklist' : blacklist 240 } 241 with open(opts.output_json, 'w') as output_json_file: 242 json.dump(output_dict, output_json_file) 243 else: 244 print blacklist 245 246 return 0 247 248 249 if __name__ == '__main__': 250 sys.exit(main()) 251
