1 #!/usr/bin/env python2.7 2 3 # Copyright 2015 gRPC authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 from __future__ import print_function 18 import json 19 import pipes 20 import shutil 21 import sys 22 import os 23 import yaml 24 25 run_tests_root = os.path.abspath(os.path.join( 26 os.path.dirname(sys.argv[0]), 27 '../../../tools/run_tests')) 28 sys.path.append(run_tests_root) 29 30 import performance.scenario_config as scenario_config 31 32 configs_from_yaml = yaml.load(open(os.path.join(os.path.dirname(sys.argv[0]), '../../../build.yaml')))['configs'].keys() 33 34 def mutate_scenario(scenario_json, is_tsan): 35 # tweak parameters to get fast test times 36 scenario_json = dict(scenario_json) 37 scenario_json['warmup_seconds'] = 0 38 scenario_json['benchmark_seconds'] = 1 39 outstanding_rpcs_divisor = 1 40 if is_tsan and ( 41 scenario_json['client_config']['client_type'] == 'SYNC_CLIENT' or 42 scenario_json['server_config']['server_type'] == 'SYNC_SERVER'): 43 outstanding_rpcs_divisor = 10 44 scenario_json['client_config']['outstanding_rpcs_per_channel'] = max(1, 45 int(scenario_json['client_config']['outstanding_rpcs_per_channel'] / outstanding_rpcs_divisor)) 46 return scenario_json 47 48 def _scenario_json_string(scenario_json, is_tsan): 49 scenarios_json = {'scenarios': [scenario_config.remove_nonproto_fields(mutate_scenario(scenario_json, is_tsan))]} 50 return json.dumps(scenarios_json) 51 52 def threads_required(scenario_json, where, is_tsan): 53 scenario_json = mutate_scenario(scenario_json, is_tsan) 54 if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): 55 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) 56 return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels'] 57 58 def guess_cpu(scenario_json, is_tsan): 59 client = threads_required(scenario_json, 'client', is_tsan) 60 server = threads_required(scenario_json, 'server', is_tsan) 61 # make an arbitrary guess if set to auto-detect 62 # about the size of the jenkins instances we have for unit tests 63 if client == 0 or server == 0: return 'capacity' 64 return (scenario_json['num_clients'] * client + 65 scenario_json['num_servers'] * server) 66 67 def maybe_exclude_gcov(scenario_json): 68 if scenario_json['client_config']['client_channels'] > 100: 69 return ['gcov'] 70 return [] 71 72 def generate_yaml(): 73 return { 74 'tests': [ 75 { 76 'name': 'json_run_localhost', 77 'shortname': 'json_run_localhost:%s' % scenario_json['name'], 78 'args': ['--scenarios_json', _scenario_json_string(scenario_json, False)], 79 'ci_platforms': ['linux'], 80 'platforms': ['linux'], 81 'flaky': False, 82 'language': 'c++', 83 'boringssl': True, 84 'defaults': 'boringssl', 85 'cpu_cost': guess_cpu(scenario_json, False), 86 'exclude_configs': ['tsan', 'asan'] + maybe_exclude_gcov(scenario_json), 87 'timeout_seconds': 2*60, 88 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), 89 'auto_timeout_scaling': False 90 } 91 for scenario_json in scenario_config.CXXLanguage().scenarios() 92 if 'scalable' in scenario_json.get('CATEGORIES', []) 93 ] + [ 94 { 95 'name': 'qps_json_driver', 96 'shortname': 'qps_json_driver:inproc_%s' % scenario_json['name'], 97 'args': ['--run_inproc', '--scenarios_json', _scenario_json_string(scenario_json, False)], 98 'ci_platforms': ['linux'], 99 'platforms': ['linux'], 100 'flaky': False, 101 'language': 'c++', 102 'boringssl': True, 103 'defaults': 'boringssl', 104 'cpu_cost': guess_cpu(scenario_json, False), 105 'exclude_configs': ['tsan', 'asan'], 106 'timeout_seconds': 6*60, 107 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []) 108 } 109 for scenario_json in scenario_config.CXXLanguage().scenarios() 110 if 'inproc' in scenario_json.get('CATEGORIES', []) 111 ] + [ 112 { 113 'name': 'json_run_localhost', 114 'shortname': 'json_run_localhost:%s_low_thread_count' % scenario_json['name'], 115 'args': ['--scenarios_json', _scenario_json_string(scenario_json, True)], 116 'ci_platforms': ['linux'], 117 'platforms': ['linux'], 118 'flaky': False, 119 'language': 'c++', 120 'boringssl': True, 121 'defaults': 'boringssl', 122 'cpu_cost': guess_cpu(scenario_json, True), 123 'exclude_configs': sorted(c for c in configs_from_yaml if c not in ('tsan', 'asan')), 124 'timeout_seconds': 10*60, 125 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), 126 'auto_timeout_scaling': False 127 } 128 for scenario_json in scenario_config.CXXLanguage().scenarios() 129 if 'scalable' in scenario_json.get('CATEGORIES', []) 130 ] 131 } 132 133 134 print(yaml.dump(generate_yaml()))
