1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Utility for checking and processing licensing information in third_party 7 directories. 8 9 Usage: licenses.py <command> 10 11 Commands: 12 scan scan third_party directories, verifying that we have licensing info 13 credits generate about:credits on stdout 14 15 (You can also import this as a module.) 16 """ 17 18 import cgi 19 import os 20 import sys 21 22 # Paths from the root of the tree to directories to skip. 23 PRUNE_PATHS = set([ 24 # Same module occurs in crypto/third_party/nss and net/third_party/nss, so 25 # skip this one. 26 os.path.join('third_party','nss'), 27 28 # Placeholder directory only, not third-party code. 29 os.path.join('third_party','adobe'), 30 31 # Build files only, not third-party code. 32 os.path.join('third_party','widevine'), 33 34 # Only binaries, used during development. 35 os.path.join('third_party','valgrind'), 36 37 # Used for development and test, not in the shipping product. 38 os.path.join('build','secondary'), 39 os.path.join('third_party','bison'), 40 os.path.join('third_party','blanketjs'), 41 os.path.join('third_party','cygwin'), 42 os.path.join('third_party','gnu_binutils'), 43 os.path.join('third_party','gold'), 44 os.path.join('third_party','gperf'), 45 os.path.join('third_party','lighttpd'), 46 os.path.join('third_party','llvm'), 47 os.path.join('third_party','llvm-build'), 48 os.path.join('third_party','mingw-w64'), 49 os.path.join('third_party','nacl_sdk_binaries'), 50 os.path.join('third_party','pefile'), 51 os.path.join('third_party','perl'), 52 os.path.join('third_party','psyco_win32'), 53 os.path.join('third_party','pylib'), 54 os.path.join('third_party','pywebsocket'), 55 os.path.join('third_party','qunit'), 56 os.path.join('third_party','sinonjs'), 57 os.path.join('third_party','syzygy'), 58 os.path.join('tools', 'profile_chrome', 'third_party'), 59 60 # Chromium code in third_party. 61 os.path.join('third_party','fuzzymatch'), 62 os.path.join('tools', 'swarming_client'), 63 64 # Stuff pulled in from chrome-internal for official builds/tools. 65 os.path.join('third_party', 'clear_cache'), 66 os.path.join('third_party', 'gnu'), 67 os.path.join('third_party', 'googlemac'), 68 os.path.join('third_party', 'pcre'), 69 os.path.join('third_party', 'psutils'), 70 os.path.join('third_party', 'sawbuck'), 71 72 # Redistribution does not require attribution in documentation. 73 os.path.join('third_party','directxsdk'), 74 os.path.join('third_party','platformsdk_win2008_6_1'), 75 os.path.join('third_party','platformsdk_win7'), 76 ]) 77 78 # Directories we don't scan through. 79 VCS_METADATA_DIRS = ('.svn', '.git') 80 PRUNE_DIRS = (VCS_METADATA_DIRS + 81 ('out', 'Debug', 'Release', # build files 82 'layout_tests')) # lots of subdirs 83 84 ADDITIONAL_PATHS = ( 85 os.path.join('breakpad'), 86 os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'), 87 os.path.join('chrome', 'test', 'chromeos', 'autotest'), 88 os.path.join('chrome', 'test', 'data'), 89 os.path.join('native_client'), 90 os.path.join('net', 'tools', 'spdyshark'), 91 os.path.join('sdch', 'open-vcdiff'), 92 os.path.join('testing', 'gmock'), 93 os.path.join('testing', 'gtest'), 94 os.path.join('tools', 'grit'), 95 os.path.join('tools', 'gyp'), 96 os.path.join('tools', 'page_cycler', 'acid3'), 97 os.path.join('url', 'third_party', 'mozilla'), 98 os.path.join('v8'), 99 # Fake directory so we can include the strongtalk license. 100 os.path.join('v8', 'strongtalk'), 101 os.path.join('v8', 'third_party', 'fdlibm'), 102 ) 103 104 105 # Directories where we check out directly from upstream, and therefore 106 # can't provide a README.chromium. Please prefer a README.chromium 107 # wherever possible. 108 SPECIAL_CASES = { 109 os.path.join('native_client'): { 110 "Name": "native client", 111 "URL": "http://code.google.com/p/nativeclient", 112 "License": "BSD", 113 }, 114 os.path.join('sdch', 'open-vcdiff'): { 115 "Name": "open-vcdiff", 116 "URL": "http://code.google.com/p/open-vcdiff", 117 "License": "Apache 2.0, MIT, GPL v2 and custom licenses", 118 "License Android Compatible": "yes", 119 }, 120 os.path.join('testing', 'gmock'): { 121 "Name": "gmock", 122 "URL": "http://code.google.com/p/googlemock", 123 "License": "BSD", 124 "License File": "NOT_SHIPPED", 125 }, 126 os.path.join('testing', 'gtest'): { 127 "Name": "gtest", 128 "URL": "http://code.google.com/p/googletest", 129 "License": "BSD", 130 "License File": "NOT_SHIPPED", 131 }, 132 os.path.join('third_party', 'angle'): { 133 "Name": "Almost Native Graphics Layer Engine", 134 "URL": "http://code.google.com/p/angleproject/", 135 "License": "BSD", 136 }, 137 os.path.join('third_party', 'cros_system_api'): { 138 "Name": "Chromium OS system API", 139 "URL": "http://www.chromium.org/chromium-os", 140 "License": "BSD", 141 # Absolute path here is resolved as relative to the source root. 142 "License File": "/LICENSE.chromium_os", 143 }, 144 os.path.join('third_party', 'lss'): { 145 "Name": "linux-syscall-support", 146 "URL": "http://code.google.com/p/linux-syscall-support/", 147 "License": "BSD", 148 "License File": "/LICENSE", 149 }, 150 os.path.join('third_party', 'ots'): { 151 "Name": "OTS (OpenType Sanitizer)", 152 "URL": "http://code.google.com/p/ots/", 153 "License": "BSD", 154 }, 155 os.path.join('third_party', 'pdfium'): { 156 "Name": "PDFium", 157 "URL": "http://code.google.com/p/pdfium/", 158 "License": "BSD", 159 }, 160 os.path.join('third_party', 'pdfsqueeze'): { 161 "Name": "pdfsqueeze", 162 "URL": "http://code.google.com/p/pdfsqueeze/", 163 "License": "Apache 2.0", 164 "License File": "COPYING", 165 }, 166 os.path.join('third_party', 'ppapi'): { 167 "Name": "ppapi", 168 "URL": "http://code.google.com/p/ppapi/", 169 }, 170 os.path.join('third_party', 'scons-2.0.1'): { 171 "Name": "scons-2.0.1", 172 "URL": "http://www.scons.org", 173 "License": "MIT", 174 "License File": "NOT_SHIPPED", 175 }, 176 os.path.join('third_party', 'trace-viewer'): { 177 "Name": "trace-viewer", 178 "URL": "http://code.google.com/p/trace-viewer", 179 "License": "BSD", 180 "License File": "NOT_SHIPPED", 181 }, 182 os.path.join('third_party', 'v8-i18n'): { 183 "Name": "Internationalization Library for v8", 184 "URL": "http://code.google.com/p/v8-i18n/", 185 "License": "Apache 2.0", 186 }, 187 os.path.join('third_party', 'WebKit'): { 188 "Name": "WebKit", 189 "URL": "http://webkit.org/", 190 "License": "BSD and GPL v2", 191 # Absolute path here is resolved as relative to the source root. 192 "License File": "/webkit/LICENSE", 193 }, 194 os.path.join('third_party', 'webpagereplay'): { 195 "Name": "webpagereplay", 196 "URL": "http://code.google.com/p/web-page-replay", 197 "License": "Apache 2.0", 198 "License File": "NOT_SHIPPED", 199 }, 200 os.path.join('tools', 'grit'): { 201 "Name": "grit", 202 "URL": "http://code.google.com/p/grit-i18n", 203 "License": "BSD", 204 "License File": "NOT_SHIPPED", 205 }, 206 os.path.join('tools', 'gyp'): { 207 "Name": "gyp", 208 "URL": "http://code.google.com/p/gyp", 209 "License": "BSD", 210 "License File": "NOT_SHIPPED", 211 }, 212 os.path.join('v8'): { 213 "Name": "V8 JavaScript Engine", 214 "URL": "http://code.google.com/p/v8", 215 "License": "BSD", 216 }, 217 os.path.join('v8', 'strongtalk'): { 218 "Name": "Strongtalk", 219 "URL": "http://www.strongtalk.org/", 220 "License": "BSD", 221 # Absolute path here is resolved as relative to the source root. 222 "License File": "/v8/LICENSE.strongtalk", 223 }, 224 os.path.join('v8', 'third_party', 'fdlibm'): { 225 "Name": "fdlibm", 226 "URL": "http://www.netlib.org/fdlibm/", 227 "License": "Freely Distributable", 228 # Absolute path here is resolved as relative to the source root. 229 "License File" : "/v8/third_party/fdlibm/LICENSE", 230 "License Android Compatible" : "yes", 231 }, 232 } 233 234 # Special value for 'License File' field used to indicate that the license file 235 # should not be used in about:credits. 236 NOT_SHIPPED = "NOT_SHIPPED" 237 238 239 class LicenseError(Exception): 240 """We raise this exception when a directory's licensing info isn't 241 fully filled out.""" 242 pass 243 244 def AbsolutePath(path, filename, root): 245 """Convert a path in README.chromium to be absolute based on the source 246 root.""" 247 if filename.startswith('/'): 248 # Absolute-looking paths are relative to the source root 249 # (which is the directory we're run from). 250 absolute_path = os.path.join(root, filename[1:]) 251 else: 252 absolute_path = os.path.join(root, path, filename) 253 if os.path.exists(absolute_path): 254 return absolute_path 255 return None 256 257 def ParseDir(path, root, require_license_file=True): 258 """Examine a third_party/foo component and extract its metadata.""" 259 260 # Parse metadata fields out of README.chromium. 261 # We examine "LICENSE" for the license file by default. 262 metadata = { 263 "License File": "LICENSE", # Relative path to license text. 264 "Name": None, # Short name (for header on about:credits). 265 "URL": None, # Project home page. 266 "License": None, # Software license. 267 } 268 269 # Relative path to a file containing some html we're required to place in 270 # about:credits. 271 optional_keys = ["Required Text", "License Android Compatible"] 272 273 if path in SPECIAL_CASES: 274 metadata.update(SPECIAL_CASES[path]) 275 else: 276 # Try to find README.chromium. 277 readme_path = os.path.join(root, path, 'README.chromium') 278 if not os.path.exists(readme_path): 279 raise LicenseError("missing README.chromium or licenses.py " 280 "SPECIAL_CASES entry") 281 282 for line in open(readme_path): 283 line = line.strip() 284 if not line: 285 break 286 for key in metadata.keys() + optional_keys: 287 field = key + ": " 288 if line.startswith(field): 289 metadata[key] = line[len(field):] 290 291 # Check that all expected metadata is present. 292 for key, value in metadata.iteritems(): 293 if not value: 294 raise LicenseError("couldn't find '" + key + "' line " 295 "in README.chromium or licences.py " 296 "SPECIAL_CASES") 297 298 # Special-case modules that aren't in the shipping product, so don't need 299 # their license in about:credits. 300 if metadata["License File"] != NOT_SHIPPED: 301 # Check that the license file exists. 302 for filename in (metadata["License File"], "COPYING"): 303 license_path = AbsolutePath(path, filename, root) 304 if license_path is not None: 305 break 306 307 if require_license_file and not license_path: 308 raise LicenseError("License file not found. " 309 "Either add a file named LICENSE, " 310 "import upstream's COPYING if available, " 311 "or add a 'License File:' line to " 312 "README.chromium with the appropriate path.") 313 metadata["License File"] = license_path 314 315 if "Required Text" in metadata: 316 required_path = AbsolutePath(path, metadata["Required Text"], root) 317 if required_path is not None: 318 metadata["Required Text"] = required_path 319 else: 320 raise LicenseError("Required text file listed but not found.") 321 322 return metadata 323 324 325 def ContainsFiles(path, root): 326 """Determines whether any files exist in a directory or in any of its 327 subdirectories.""" 328 for _, dirs, files in os.walk(os.path.join(root, path)): 329 if files: 330 return True 331 for vcs_metadata in VCS_METADATA_DIRS: 332 if vcs_metadata in dirs: 333 dirs.remove(vcs_metadata) 334 return False 335 336 337 def FilterDirsWithFiles(dirs_list, root): 338 # If a directory contains no files, assume it's a DEPS directory for a 339 # project not used by our current configuration and skip it. 340 return [x for x in dirs_list if ContainsFiles(x, root)] 341 342 343 def FindThirdPartyDirs(prune_paths, root): 344 """Find all third_party directories underneath the source root.""" 345 third_party_dirs = set() 346 for path, dirs, files in os.walk(root): 347 path = path[len(root)+1:] # Pretty up the path. 348 349 if path in prune_paths: 350 dirs[:] = [] 351 continue 352 353 # Prune out directories we want to skip. 354 # (Note that we loop over PRUNE_DIRS so we're not iterating over a 355 # list that we're simultaneously mutating.) 356 for skip in PRUNE_DIRS: 357 if skip in dirs: 358 dirs.remove(skip) 359 360 if os.path.basename(path) == 'third_party': 361 # Add all subdirectories that are not marked for skipping. 362 for dir in dirs: 363 dirpath = os.path.join(path, dir) 364 if dirpath not in prune_paths: 365 third_party_dirs.add(dirpath) 366 367 # Don't recurse into any subdirs from here. 368 dirs[:] = [] 369 continue 370 371 # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular 372 # third_party/foo paths. 373 if path in ADDITIONAL_PATHS: 374 dirs[:] = [] 375 376 for dir in ADDITIONAL_PATHS: 377 if dir not in prune_paths: 378 third_party_dirs.add(dir) 379 380 return third_party_dirs 381 382 383 def ScanThirdPartyDirs(root=None): 384 """Scan a list of directories and report on any problems we find.""" 385 if root is None: 386 root = os.getcwd() 387 third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root) 388 third_party_dirs = FilterDirsWithFiles(third_party_dirs, root) 389 390 errors = [] 391 for path in sorted(third_party_dirs): 392 try: 393 metadata = ParseDir(path, root) 394 except LicenseError, e: 395 errors.append((path, e.args[0])) 396 continue 397 398 for path, error in sorted(errors): 399 print path + ": " + error 400 401 return len(errors) == 0 402 403 404 def GenerateCredits(): 405 """Generate about:credits.""" 406 407 if len(sys.argv) not in (2, 3): 408 print 'usage: licenses.py credits [output_file]' 409 return False 410 411 def EvaluateTemplate(template, env, escape=True): 412 """Expand a template with variables like {{foo}} using a 413 dictionary of expansions.""" 414 for key, val in env.items(): 415 if escape and not key.endswith("_unescaped"): 416 val = cgi.escape(val) 417 template = template.replace('{{%s}}' % key, val) 418 return template 419 420 root = os.path.join(os.path.dirname(__file__), '..') 421 third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root) 422 423 entry_template = open(os.path.join(root, 'chrome', 'browser', 'resources', 424 'about_credits_entry.tmpl'), 'rb').read() 425 entries = [] 426 for path in sorted(third_party_dirs): 427 try: 428 metadata = ParseDir(path, root) 429 except LicenseError: 430 # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240). 431 continue 432 if metadata['License File'] == NOT_SHIPPED: 433 continue 434 env = { 435 'name': metadata['Name'], 436 'url': metadata['URL'], 437 'license': open(metadata['License File'], 'rb').read(), 438 'license_unescaped': '', 439 } 440 if 'Required Text' in metadata: 441 required_text = open(metadata['Required Text'], 'rb').read() 442 env["license_unescaped"] = required_text 443 entries.append(EvaluateTemplate(entry_template, env)) 444 445 file_template = open(os.path.join(root, 'chrome', 'browser', 'resources', 446 'about_credits.tmpl'), 'rb').read() 447 template_contents = "<!-- Generated by licenses.py; do not edit. -->" 448 template_contents += EvaluateTemplate(file_template, 449 {'entries': '\n'.join(entries)}, 450 escape=False) 451 452 if len(sys.argv) == 3: 453 with open(sys.argv[2], 'w') as output_file: 454 output_file.write(template_contents) 455 elif len(sys.argv) == 2: 456 print template_contents 457 458 return True 459 460 461 def main(): 462 command = 'help' 463 if len(sys.argv) > 1: 464 command = sys.argv[1] 465 466 if command == 'scan': 467 if not ScanThirdPartyDirs(): 468 return 1 469 elif command == 'credits': 470 if not GenerateCredits(): 471 return 1 472 else: 473 print __doc__ 474 return 1 475 476 477 if __name__ == '__main__': 478 sys.exit(main()) 479
