1 ############################################################################ 2 # Copyright 2016-2017 Intel Corporation 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 ############################################################################ 16 17 """use scons -k to invoke all builds regardless of unit test failures 18 """ 19 import string 20 import sys 21 import SCons.Script 22 import os.path 23 import subprocess 24 from subprocess import Popen, PIPE 25 from parts import * 26 import re 27 import tempfile 28 import shutil 29 from collections import OrderedDict 30 31 def get_parts_versions(env): 32 """Get Parts related versions given SCons environment env""" 33 return OrderedDict({'python': string.split(sys.version, " ", 1)[0], 34 'scons': str(SCons.__version__), 35 'parts': str(PartsExtensionVersion())}) 36 37 def get_toolchain_versions(env): 38 """Get version of compilation toolchain given SCons environment env""" 39 versions = OrderedDict() 40 if 'MSVC_VERSION' in env: 41 versions['compiler'] = 'MSVC ' + env['MSVC_VERSION'] 42 cmd = env.subst('echo int main(){return 0;} > a.cpp' 43 ' | $CXX $CCFLAGS a.cpp /link /verbose') 44 defaultlib_regexp = r'.*Searching (.*\.lib).*' 45 elif 'GCC_VERSION' in env: 46 versions['compiler'] = 'GCC ' + env['GCC_VERSION'] 47 if 'GXX_VERSION' in env: 48 versions['compiler'] += ' and GXX ' + env['GXX_VERSION'] 49 if os.name == 'nt': 50 cmd = env.subst('echo int main(){return 0;}' 51 ' | $CXX $CCFLAGS -xc++ -Wl,--verbose -') 52 else: 53 cmd = env.subst('echo "int main(){return 0;}"' 54 ' | $CXX $CCFLAGS -xc++ -Wl,--verbose -') 55 else: 56 if os.name == 'nt': 57 cmd = env.subst('echo int main(){return 0;}' 58 ' | $CXX $CCFLAGS -xc++ -Wl,--verbose -') 59 else: 60 cmd = env.subst('echo "int main(){return 0;}"' 61 ' | $CC $CCFLAGS -xc -Wl,--verbose -') 62 if os.name == 'nt': 63 defaultlib_regexp = r'\n.* open (.*) succeeded' 64 else: 65 defaultlib_regexp = r'[\n(](/.*\.so[-.\da-fA-F]*).*' 66 67 # Intel C compiler always depends from base toolchain 68 if 'INTELC_VERSION' in env: 69 versions['compiler'] = 'INTELC {0} with {1}'.format( 70 env['INTELC_VERSION'], 71 versions['compiler']) 72 73 env['ENV']['PATH'] = str(env['ENV']['PATH']) 74 temp_dir = tempfile.mkdtemp() 75 try: 76 proc = subprocess.Popen(cmd, 77 cwd=temp_dir, 78 env=env['ENV'], 79 shell=True, 80 stdout=subprocess.PIPE, stderr=subprocess.PIPE) 81 stdout, _ = proc.communicate() 82 if proc.returncode != 0: 83 versions['default_libs'] = 'failure executing: "{0}"'.format(cmd) 84 else: 85 default_libs = list( 86 set(re.findall(defaultlib_regexp, stdout, re.M))) 87 if 'MSVC_VERSION' in env: 88 # for windows additionally report versions of Windows Kit used 89 runtime_version_set = set() 90 for lib_path in default_libs: 91 path_components = os.path.realpath(lib_path).split(os.sep) 92 if 'Windows Kits' in path_components: 93 i = path_components.index('Windows Kits') 94 runtime_version_set.add( 95 'Windows Kits {0} {1}'.format(path_components[i + 1], 96 path_components[i + 3])) 97 versions['sdk_or_libc'] = '; '.join(list(runtime_version_set)) 98 else: 99 # for posix additionally report versions of libc used 100 versions['sdk_or_libc'] = os.path.split(os.path.realpath( 101 next((lib for lib in default_libs if 102 'libc' in lib.lower() and 'libcilk' not in lib.lower()), None)))[1] 103 versions['default_libs'] = default_libs 104 finally: 105 shutil.rmtree(temp_dir) 106 107 return versions 108 109 110 def log_versions(env, include_toolchain=True): 111 """Log tools and libraries versions given SCons environment env 112 113 Args: 114 env: Scons environment. 115 include_toolchain: Log version of compilation toolchain if True. 116 """ 117 118 versions = get_parts_versions(env) 119 if include_toolchain: 120 versions.update(get_toolchain_versions(env)) 121 122 print "**************** VERSIONS *************" 123 long_names = { 124 'python': 'Python Version', 125 'scons': 'SCons Version', 126 'parts': 'Parts Version', 127 'compiler': 'Compiler Version', 128 'sdk_or_libc': 'Libc/SDK', 129 'default_libs': 'Default Libs' 130 } 131 for name, value in versions.iteritems(): 132 if not isinstance(value, list): 133 print '* {0}: {1}'.format(long_names.get(name, name), value) 134 else: 135 print '* {0}:\n* \t{1}'.format(long_names.get(name, name), 136 '\n* \t'.join(sorted(value))) 137 print "***************************************" 138 139 140 def include_parts(part_list, **kwargs): 141 for parts_file in part_list: 142 if os.path.isfile(DefaultEnvironment().subst(parts_file)): 143 Part(parts_file=parts_file, **kwargs) 144 145 146 ######## Part groups #################################################### 147 ipp_parts = ['ext/ipp/ippcp.parts'] 148 utest_parts = ['ext/gtest/gtest.parts', 149 'epid/common-testhelper/common-testhelper.parts'] 150 common_parts = ['epid/common/common.parts'] 151 member_parts = ['epid/member/member.parts'] 152 verifier_parts = ['epid/verifier/verifier.parts'] 153 util_parts = ['example/util/util.parts'] 154 example_parts = ['ext/argtable3/argtable3.parts', 155 'example/verifysig/verifysig.parts', 156 'example/signmsg/signmsg.parts', 157 'example/data/data.parts', 158 'example/compressed_data/compressed_data.parts'] 159 sizing_parts = ['example/util/util_static.parts', 160 'example/signmsg/signmsg_shared.parts', 161 'example/verifysig/verifysig_shared.parts', 162 'example/verifysig/verifysig11_shared.parts'] 163 example_static_parts = ['example/util/util_static.parts', 164 'example/signmsg/signmsg_static.parts', 165 'example/verifysig/verifysig_static.parts'] 166 tools_parts = ['tools/revokegrp/revokegrp.parts', 167 'tools/revokekey/revokekey.parts', 168 'tools/revokesig/revokesig.parts', 169 'tools/extractkeys/extractkeys.parts', 170 'tools/extractgrps/extractgrps.parts'] 171 testbot_test_parts = ['test/testbot/testbot.parts', 172 'test/testbot/signmsg/signmsg_testbot.parts', 173 'test/testbot/verifysig/verifysig_testbot.parts', 174 'test/testbot/integration/integration_testbot.parts', 175 'test/testbot/ssh_remote/ssh_remote_testbot.parts', 176 'test/testbot/revokegrp/revokegrp_testbot.parts', 177 'test/testbot/revokekey/revokekey_testbot.parts', 178 'test/testbot/revokesig/revokesig_testbot.parts', 179 'test/testbot/extractkeys/extractkeys_testbot.parts', 180 'test/testbot/extractgrps/extractgrps_testbot.parts', 181 'tools/reports/reports.parts'] 182 tss_test_parts = ['test/tss/tss.parts'] 183 package_parts = ['ext/gtest/gtest.parts', 184 'ext/ipp/ippcp.parts', 185 'package.parts'] 186 memory_profiler_parts = ['tools/memory_profiler/memory_profiler.parts'] 187 internal_tools_parts = ['ext/argtable3/argtable3.parts', 188 'tools/ikgfwrapper/ikgfwrapper.parts'] 189 epid_data = ['test/epid_data/epid_data.parts'] 190 perf_benchmark_parts = ['ext/google_benchmark/google_benchmark.parts', 191 'test/performance/performance.parts'] 192 memory_benchmark_parts = ['test/dynamic_memory/dynamic_memory.parts'] 193 ######## End Part groups ############################################### 194 ######## Commandline option setup ####################################### 195 product_variants = [ 196 'production', 197 'internal-test', 198 'package-epid-sdk', 199 'internal-tools', 200 'benchmark', 201 'tiny', 202 'internal-test-tiny' 203 ] 204 205 default_variant = 'production' 206 207 208 def is_production(): 209 return GetOption("product-variant") == 'production' 210 211 212 def is_internal_test(): 213 return GetOption("product-variant") == 'internal-test' 214 215 216 def is_internal_tools(): 217 return GetOption("product-variant") == 'internal-tools' 218 219 220 def is_package(): 221 return GetOption("product-variant") == 'package-epid-sdk' 222 223 224 def is_benchmark(): 225 return GetOption("product-variant") == 'benchmark' 226 227 def is_tiny(): 228 return GetOption("product-variant") == 'tiny' 229 230 def is_internal_test_tiny(): 231 return GetOption("product-variant") == 'internal-test-tiny' 232 233 234 def use_commercial_ipp(): 235 return GetOption("use-commercial-ipp") 236 237 238 def use_tss(): 239 return GetOption("use-tss") 240 241 242 def config_has_instrumentation(): 243 return any(DefaultEnvironment().isConfigBasedOn(config_name) 244 for config_name in ['instr_release']) 245 246 247 def variant_dirname(): 248 s = GetOption("product-variant") 249 if s == 'production': 250 return 'epid-sdk' 251 elif s == 'package-epid-sdk': 252 return 'epid-sdk' 253 elif s == 'tiny': 254 return 'epid-sdk' 255 else: 256 return s 257 258 259 AddOption("--product-variant", "--prod-var", nargs=1, 260 help=("Select product variant to build. Possible " 261 "options are: {0}. The default is {1} if no option " 262 "is specified").format(", ".join(product_variants), 263 default_variant), 264 action='store', dest='product-variant', type='choice', 265 choices=product_variants, default=default_variant) 266 267 AddOption("--use-commercial-ipp", 268 help=("Link with commercial IPP. The IPPCRYPTOROOT environment " 269 "variable must be set."), 270 action='store_true', dest='use-commercial-ipp', 271 default=False) 272 273 AddOption("--use-tss", 274 help=("Link with TPM TSS. The TSSROOT environment variable " 275 "must be set."), 276 action='store_true', dest='use-tss', 277 default=False) 278 279 AddOption("--ipp-shared", 280 help=("Build /ext/ipp as shared library."), 281 action='store_true', dest='ipp-shared', 282 default=False) 283 284 AddOption("--enable-sanitizers", 285 help=("Build with sanitizers (https://github.com/google/sanitizers)."), 286 action='store_true', dest='sanitizers', 287 default=False) 288 289 AddOption("--sanitizers-recover", 290 help=("Configure sanititzers to recover and continue execution " 291 "on error found. Only applicable when sanitizers are enabled." 292 "See --enable-sanitizers option."), 293 action='store_true', dest='sanitizers-recover', 294 default=False) 295 296 297 SetOptionDefault("PRODUCT_VARIANT", variant_dirname()) 298 299 ######## End Commandline option setup ################################### 300 301 302 # fix for parts 0.10.8 until we get better logic to extract ${CC} 303 SetOptionDefault('PARTS_USE_SHORT_TOOL_NAMES', 1) 304 305 306 def enable_sanitizers(recover): 307 """ 308 Configures compiler to enable sanitizers. 309 Adds sanitizer options to default scons environment such 310 that it affects all parts. 311 Args: 312 recover: Enable sanitizers recovery from errors found when True. 313 """ 314 env = DefaultEnvironment() 315 error_msg = None 316 try: 317 major = int(env.subst('$GCC_VERSION').partition('.')[0]) 318 except ValueError: 319 major = 0 320 321 if major >= 6 and env['TARGET_OS'] == 'posix': 322 if 'INTELC_VERSION' not in env: 323 ccflags = ['-fsanitize=address,undefined', '-fno-sanitize=alignment', 324 '-fno-sanitize=shift', '-fno-omit-frame-pointer'] 325 if recover: 326 ccflags = ccflags + ['-fsanitize-recover=all', '-fsanitize-recover=address'] 327 else: 328 ccflags = ccflags + ['-fno-sanitize-recover'] 329 # Extends default flags with sanitizer options 330 SetOptionDefault('CCFLAGS', ccflags) 331 SetOptionDefault('LIBS', ['asan', 'ubsan']) 332 else: 333 error_msg = """ 334 Build with sanitizers is not supported for Intel(R) C++ Compiler. 335 Try scons --toolchain=gcc_6 --target=posix 336 """ 337 else: 338 # User experience with sanitizers in GCC 4.8 is not great. Use at least GCC 6.x. 339 error_msg = """ 340 Build with sanitizers is only supported for GCC version greater than 341 6.x targeting posix OS. Current GCC version is "{0}" and OS target is "{1}". 342 Try scons --toolchain=gcc_6 --target=posix 343 """.format(env.get('GCC_VERSION', 'unknown'), env.get('TARGET_OS', 'unknown')) 344 if error_msg is not None: 345 env.PrintError(error_msg) 346 347 348 def set_default_production_options(): 349 SetOptionDefault('CONFIG', 'release') 350 351 SetOptionDefault('TARGET_VARIANT', '${TARGET_OS}-${TARGET_ARCH}') 352 353 SetOptionDefault('INSTALL_ROOT', 354 '#_install/${PRODUCT_VARIANT}') 355 356 SetOptionDefault('INSTALL_TOOLS_BIN', 357 '$INSTALL_ROOT/tools') 358 359 SetOptionDefault('INSTALL_SAMPLE_BIN', 360 '$INSTALL_ROOT/example') 361 362 SetOptionDefault('INSTALL_EPID_INCLUDE', 363 '$INSTALL_ROOT/include/epid') 364 365 SetOptionDefault('INSTALL_IPP_INCLUDE', 366 '$INSTALL_ROOT/include/ext/ipp/include') 367 368 SetOptionDefault('INSTALL_TEST_BIN', 369 '$INSTALL_ROOT/test') 370 371 SetOptionDefault('INSTALL_LIB', 372 '$INSTALL_ROOT/lib/${TARGET_VARIANT}') 373 374 SetOptionDefault('INSTALL_SAMPLE_DATA', 375 '$INSTALL_ROOT/example') 376 377 SetOptionDefault('INSTALL_TOOLS_DATA', 378 '$INSTALL_ROOT/tools') 379 380 SetOptionDefault('PACKAGE_DIR', 381 '#_package') 382 383 SetOptionDefault('PACKAGE_ROOT', 384 '#_package/${PRODUCT_VARIANT}') 385 386 SetOptionDefault('ROOT', 387 '#') 388 389 SetOptionDefault('PACKAGE_NAME', 390 '{PRODUCT_VARIANT}') 391 392 393 if GetOption("sanitizers"): 394 enable_sanitizers(GetOption("sanitizers-recover")) 395 396 if is_production(): 397 set_default_production_options() 398 ipp_mode = ['install_lib'] 399 if use_commercial_ipp(): 400 ipp_mode.append('use_commercial_ipp') 401 sdk_mode = ['install_lib'] 402 if use_tss(): 403 sdk_mode.append('use_tss') 404 if GetOption('ipp-shared'): 405 ipp_mode.append('build_ipp_shared') 406 include_parts(ipp_parts, mode=ipp_mode, 407 INSTALL_INCLUDE='${INSTALL_IPP_INCLUDE}') 408 include_parts(utest_parts + common_parts + 409 member_parts + verifier_parts, 410 mode=sdk_mode, 411 INSTALL_INCLUDE='${INSTALL_EPID_INCLUDE}') 412 include_parts(util_parts + example_parts, 413 INSTALL_INCLUDE='${INSTALL_EPID_INCLUDE}', 414 INSTALL_BIN='${INSTALL_SAMPLE_BIN}', 415 INSTALL_DATA='${INSTALL_SAMPLE_DATA}') 416 include_parts(tools_parts, 417 INSTALL_BIN='${INSTALL_TOOLS_BIN}', 418 INSTALL_DATA='${INSTALL_TOOLS_DATA}') 419 Default('all') 420 Default('utest::') 421 if not use_tss(): 422 Default('run_utest::') 423 424 if is_internal_test(): 425 set_default_production_options() 426 sdk_mode = [] 427 if use_tss(): 428 sdk_mode.append('use_tss') 429 include_parts(tss_test_parts) 430 include_parts(ipp_parts) 431 include_parts(utest_parts + common_parts + 432 member_parts + verifier_parts, 433 mode=sdk_mode) 434 include_parts(util_parts + example_parts, 435 INSTALL_BIN='${INSTALL_SAMPLE_BIN}', 436 INSTALL_DATA='${INSTALL_SAMPLE_DATA}') 437 include_parts(sizing_parts, 438 INSTALL_BIN='${INSTALL_SAMPLE_BIN}') 439 include_parts(tools_parts, INSTALL_BIN='${INSTALL_TOOLS_BIN}') 440 include_parts(testbot_test_parts) 441 Default('all') 442 443 if is_internal_tools(): 444 set_default_production_options() 445 include_parts(ipp_parts + utest_parts + common_parts + verifier_parts + member_parts + util_parts) 446 include_parts(internal_tools_parts + memory_profiler_parts, 447 INSTALL_BIN='${INSTALL_TOOLS_BIN}') 448 Default('ikgfwrapper', 'memory_profiler') 449 Default('run_utest::memory_profiler::') 450 451 if is_benchmark(): 452 set_default_production_options() 453 MODE = [] 454 if config_has_instrumentation(): 455 MODE.append('use_memory_profiler') 456 ipp_mode = [] 457 if use_commercial_ipp(): 458 ipp_mode.append('use_commercial_ipp') 459 460 # install ipp static and ipp shared builds into separate locations 461 if GetOption('ipp-shared'): 462 ipp_mode.append('build_ipp_shared') 463 SetOptionDefault('INSTALL_TEST_BIN', 464 '$INSTALL_ROOT/test_ipp_shared') 465 SetOptionDefault('INSTALL_LIB', 466 '$INSTALL_ROOT/lib_ipp_shared') 467 else: 468 SetOptionDefault('INSTALL_LIB', 469 '$INSTALL_ROOT/lib') 470 471 # do not allow file links to keep previous builds intact 472 SetOptionDefault('CCOPY_LOGIC', 'copy') 473 474 include_parts(ipp_parts, config_independent=True, mode=MODE + ipp_mode, 475 INSTALL_BIN='${INSTALL_TEST_BIN}') 476 include_parts(example_static_parts + utest_parts + perf_benchmark_parts + 477 common_parts + verifier_parts + 478 sizing_parts + epid_data, 479 config_independent=True, 480 mode=MODE, 481 INSTALL_BIN='${INSTALL_TEST_BIN}') 482 483 member_mode = ['install_lib'] 484 member_cfg = ('embedded' if not DefaultEnvironment().isConfigBasedOn( 485 'debug') and not config_has_instrumentation() else DefaultEnvironment().subst('$CONFIG')) 486 Part(parts_file='epid/common/tinycommon.parts', CONFIG=member_cfg) 487 Part(parts_file='epid/member/tinymember.parts', CONFIG=member_cfg, 488 config_independent=True, mode=MODE + member_mode, INSTALL_BIN='${INSTALL_TEST_BIN}') 489 490 if config_has_instrumentation(): 491 include_parts(memory_benchmark_parts + memory_profiler_parts, 492 config_independent=True, 493 mode=MODE, 494 INSTALL_BIN='${INSTALL_TEST_BIN}') 495 496 Default('build::') 497 498 if is_package(): 499 set_default_production_options() 500 include_parts(package_parts, 501 mode=['install_package'], 502 INSTALL_TOP_LEVEL='${PACKAGE_ROOT}') 503 Default('package') 504 505 if is_tiny(): 506 set_default_production_options() 507 ### Member 508 Part(parts_file='ext/gtest/gtest.parts') 509 member_mode = ['install_lib'] 510 member_cfg = ('embedded' 511 if not DefaultEnvironment().isConfigBasedOn('debug') 512 else DefaultEnvironment().subst('$CONFIG')) 513 Part(parts_file='epid/common/tinycommon.parts', CONFIG=member_cfg) 514 Part(parts_file='epid/member/tinymember.parts', CONFIG=member_cfg, 515 config_independent=True, mode=member_mode) 516 Default('member::') 517 Default('run_utest::member::') 518 ### Verifier, samples and tools 519 verifier_mode = ['install_lib'] 520 ipp_mode = ['install_lib'] 521 if use_commercial_ipp(): 522 ipp_mode.append('use_commercial_ipp') 523 if GetOption('ipp-shared'): 524 ipp_mode.append('build_ipp_shared') 525 include_parts(ipp_parts, mode=ipp_mode, 526 INSTALL_INCLUDE='${INSTALL_IPP_INCLUDE}') 527 Part(parts_file='epid/common-testhelper/common-testhelper.parts', 528 config_independent=True) 529 include_parts(common_parts + verifier_parts, 530 mode=verifier_mode, 531 INSTALL_INCLUDE='${INSTALL_EPID_INCLUDE}') 532 include_parts(util_parts + example_parts, 533 INSTALL_INCLUDE='${INSTALL_EPID_INCLUDE}', 534 INSTALL_BIN='${INSTALL_SAMPLE_BIN}', 535 INSTALL_DATA='${INSTALL_SAMPLE_DATA}') 536 include_parts(tools_parts, 537 INSTALL_BIN='${INSTALL_TOOLS_BIN}', 538 INSTALL_DATA='${INSTALL_TOOLS_DATA}') 539 Default('all') 540 Default('utest::') 541 542 if is_internal_test_tiny(): 543 set_default_production_options() 544 sdk_mode = [] 545 ### Member 546 Part(parts_file='ext/gtest/gtest.parts') 547 member_cfg = ('embedded' 548 if not DefaultEnvironment().isConfigBasedOn('debug') 549 else DefaultEnvironment().subst('$CONFIG')) 550 Part(parts_file='epid/common/tinycommon.parts', CONFIG=member_cfg) 551 Part(parts_file='epid/member/tinymember.parts', CONFIG=member_cfg, 552 config_independent=True, mode=sdk_mode) 553 ### Verifier, samples and tools 554 include_parts(ipp_parts) 555 Part(parts_file='epid/common-testhelper/common-testhelper.parts', 556 config_independent=True) 557 include_parts(common_parts + verifier_parts, 558 mode=sdk_mode) 559 include_parts(util_parts + example_parts, 560 INSTALL_BIN='${INSTALL_SAMPLE_BIN}', 561 INSTALL_DATA='${INSTALL_SAMPLE_DATA}') 562 include_parts(tools_parts, INSTALL_BIN='${INSTALL_TOOLS_BIN}') 563 include_parts(testbot_test_parts) 564 Default('build::') 565 566 log_versions(DefaultEnvironment(), not is_package()) 567
