1 #!/usr/bin/env python2.7 2 3 # Copyright 2017 gRPC authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import collections 18 import ctypes 19 import math 20 import sys 21 import yaml 22 import json 23 24 with open('src/core/lib/debug/stats_data.yaml') as f: 25 attrs = yaml.load(f.read()) 26 27 REQUIRED_FIELDS = ['name', 'doc'] 28 29 30 def make_type(name, fields): 31 return (collections.namedtuple(name, ' '.join( 32 list(set(REQUIRED_FIELDS + fields)))), []) 33 34 35 def c_str(s, encoding='ascii'): 36 if isinstance(s, unicode): 37 s = s.encode(encoding) 38 result = '' 39 for c in s: 40 if not (32 <= ord(c) < 127) or c in ('\\', '"'): 41 result += '\\%03o' % ord(c) 42 else: 43 result += c 44 return '"' + result + '"' 45 46 47 types = ( 48 make_type('Counter', []), 49 make_type('Histogram', ['max', 'buckets']), 50 ) 51 52 inst_map = dict((t[0].__name__, t[1]) for t in types) 53 54 stats = [] 55 56 for attr in attrs: 57 found = False 58 for t, lst in types: 59 t_name = t.__name__.lower() 60 if t_name in attr: 61 name = attr[t_name] 62 del attr[t_name] 63 lst.append(t(name=name, **attr)) 64 found = True 65 break 66 assert found, "Bad decl: %s" % attr 67 68 69 def dbl2u64(d): 70 return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value 71 72 73 def shift_works_until(mapped_bounds, shift_bits): 74 for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])): 75 a, b = ab 76 if (a >> shift_bits) == (b >> shift_bits): 77 return i 78 return len(mapped_bounds) 79 80 81 def find_ideal_shift(mapped_bounds, max_size): 82 best = None 83 for shift_bits in reversed(range(0, 64)): 84 n = shift_works_until(mapped_bounds, shift_bits) 85 if n == 0: continue 86 table_size = mapped_bounds[n - 1] >> shift_bits 87 if table_size > max_size: continue 88 if table_size > 65535: continue 89 if best is None: 90 best = (shift_bits, n, table_size) 91 elif best[1] < n: 92 best = (shift_bits, n, table_size) 93 print best 94 return best 95 96 97 def gen_map_table(mapped_bounds, shift_data): 98 tbl = [] 99 cur = 0 100 print mapped_bounds 101 mapped_bounds = [x >> shift_data[0] for x in mapped_bounds] 102 print mapped_bounds 103 for i in range(0, mapped_bounds[shift_data[1] - 1]): 104 while i > mapped_bounds[cur]: 105 cur += 1 106 tbl.append(cur) 107 return tbl 108 109 110 static_tables = [] 111 112 113 def decl_static_table(values, type): 114 global static_tables 115 v = (type, values) 116 for i, vp in enumerate(static_tables): 117 if v == vp: return i 118 print "ADD TABLE: %s %r" % (type, values) 119 r = len(static_tables) 120 static_tables.append(v) 121 return r 122 123 124 def type_for_uint_table(table): 125 mv = max(table) 126 if mv < 2**8: 127 return 'uint8_t' 128 elif mv < 2**16: 129 return 'uint16_t' 130 elif mv < 2**32: 131 return 'uint32_t' 132 else: 133 return 'uint64_t' 134 135 136 def gen_bucket_code(histogram): 137 bounds = [0, 1] 138 done_trivial = False 139 done_unmapped = False 140 first_nontrivial = None 141 first_unmapped = None 142 while len(bounds) < histogram.buckets + 1: 143 if len(bounds) == histogram.buckets: 144 nextb = int(histogram.max) 145 else: 146 mul = math.pow( 147 float(histogram.max) / bounds[-1], 148 1.0 / (histogram.buckets + 1 - len(bounds))) 149 nextb = int(math.ceil(bounds[-1] * mul)) 150 if nextb <= bounds[-1] + 1: 151 nextb = bounds[-1] + 1 152 elif not done_trivial: 153 done_trivial = True 154 first_nontrivial = len(bounds) 155 bounds.append(nextb) 156 bounds_idx = decl_static_table(bounds, 'int') 157 if done_trivial: 158 first_nontrivial_code = dbl2u64(first_nontrivial) 159 code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds] 160 shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 161 256 * histogram.buckets) 162 #print first_nontrivial, shift_data, bounds 163 #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] 164 code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max 165 map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) 166 if first_nontrivial is None: 167 code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % 168 histogram.name.upper()) 169 else: 170 code += 'if (value < %d) {\n' % first_nontrivial 171 code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % 172 histogram.name.upper()) 173 code += 'return;\n' 174 code += '}' 175 first_nontrivial_code = dbl2u64(first_nontrivial) 176 if shift_data is not None: 177 map_table_idx = decl_static_table(map_table, 178 type_for_uint_table(map_table)) 179 code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n' 180 code += '_val.dbl = value;\n' 181 code += 'if (_val.uint < %dull) {\n' % ( 182 (map_table[-1] << shift_data[0]) + first_nontrivial_code) 183 code += 'int bucket = ' 184 code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % ( 185 map_table_idx, first_nontrivial_code, shift_data[0], 186 first_nontrivial) 187 code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx 188 code += 'bucket -= (_val.uint < _bkt.uint);\n' 189 code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper( 190 ) 191 code += 'return;\n' 192 code += '}\n' 193 code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper( 194 ) 195 code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % ( 196 bounds_idx, histogram.buckets) 197 return (code, bounds_idx) 198 199 200 # utility: print a big comment block into a set of files 201 def put_banner(files, banner): 202 for f in files: 203 print >> f, '/*' 204 for line in banner: 205 print >> f, ' * %s' % line 206 print >> f, ' */' 207 print >> f 208 209 210 with open('src/core/lib/debug/stats_data.h', 'w') as H: 211 # copy-paste copyright notice from this file 212 with open(sys.argv[0]) as my_source: 213 copyright = [] 214 for line in my_source: 215 if line[0] != '#': break 216 for line in my_source: 217 if line[0] == '#': 218 copyright.append(line) 219 break 220 for line in my_source: 221 if line[0] != '#': 222 break 223 copyright.append(line) 224 put_banner([H], [line[2:].rstrip() for line in copyright]) 225 226 put_banner( 227 [H], 228 ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) 229 230 print >> H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H" 231 print >> H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H" 232 print >> H 233 print >> H, "#include <grpc/support/port_platform.h>" 234 print >> H 235 print >> H, "#include <inttypes.h>" 236 print >> H, "#include \"src/core/lib/iomgr/exec_ctx.h\"" 237 print >> H 238 239 for typename, instances in sorted(inst_map.items()): 240 print >> H, "typedef enum {" 241 for inst in instances: 242 print >> H, " GRPC_STATS_%s_%s," % (typename.upper(), 243 inst.name.upper()) 244 print >> H, " GRPC_STATS_%s_COUNT" % (typename.upper()) 245 print >> H, "} grpc_stats_%ss;" % (typename.lower()) 246 print >> H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % ( 247 typename.lower(), typename.upper()) 248 print >> H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % ( 249 typename.lower(), typename.upper()) 250 251 histo_start = [] 252 histo_buckets = [] 253 histo_bucket_boundaries = [] 254 255 print >> H, "typedef enum {" 256 first_slot = 0 257 for histogram in inst_map['Histogram']: 258 histo_start.append(first_slot) 259 histo_buckets.append(histogram.buckets) 260 print >> H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % ( 261 histogram.name.upper(), first_slot) 262 print >> H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % ( 263 histogram.name.upper(), histogram.buckets) 264 first_slot += histogram.buckets 265 print >> H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot 266 print >> H, "} grpc_stats_histogram_constants;" 267 268 print >> H, "#if defined(GRPC_COLLECT_STATS) || !defined(NDEBUG)" 269 for ctr in inst_map['Counter']: 270 print >> H, ("#define GRPC_STATS_INC_%s() " + 271 "GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % ( 272 ctr.name.upper(), ctr.name.upper()) 273 for histogram in inst_map['Histogram']: 274 print >> H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % ( 275 histogram.name.upper(), histogram.name.lower()) 276 print >> H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower() 277 278 print >> H, "#else" 279 for ctr in inst_map['Counter']: 280 print >> H, ("#define GRPC_STATS_INC_%s() ") % (ctr.name.upper()) 281 for histogram in inst_map['Histogram']: 282 print >> H, "#define GRPC_STATS_INC_%s(value)" % ( 283 histogram.name.upper()) 284 print >> H, "#endif /* defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) */" 285 286 for i, tbl in enumerate(static_tables): 287 print >> H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, 288 len(tbl[1])) 289 290 print >> H, "extern const int grpc_stats_histo_buckets[%d];" % len( 291 inst_map['Histogram']) 292 print >> H, "extern const int grpc_stats_histo_start[%d];" % len( 293 inst_map['Histogram']) 294 print >> H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len( 295 inst_map['Histogram']) 296 print >> H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len( 297 inst_map['Histogram']) 298 299 print >> H 300 print >> H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */" 301 302 with open('src/core/lib/debug/stats_data.cc', 'w') as C: 303 # copy-paste copyright notice from this file 304 with open(sys.argv[0]) as my_source: 305 copyright = [] 306 for line in my_source: 307 if line[0] != '#': break 308 for line in my_source: 309 if line[0] == '#': 310 copyright.append(line) 311 break 312 for line in my_source: 313 if line[0] != '#': 314 break 315 copyright.append(line) 316 put_banner([C], [line[2:].rstrip() for line in copyright]) 317 318 put_banner( 319 [C], 320 ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) 321 322 print >> C, "#include <grpc/support/port_platform.h>" 323 print >> C 324 print >> C, "#include \"src/core/lib/debug/stats.h\"" 325 print >> C, "#include \"src/core/lib/debug/stats_data.h\"" 326 print >> C, "#include \"src/core/lib/gpr/useful.h\"" 327 print >> C, "#include \"src/core/lib/iomgr/exec_ctx.h\"" 328 print >> C 329 330 histo_code = [] 331 for histogram in inst_map['Histogram']: 332 code, bounds_idx = gen_bucket_code(histogram) 333 histo_bucket_boundaries.append(bounds_idx) 334 histo_code.append(code) 335 336 for typename, instances in sorted(inst_map.items()): 337 print >> C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % ( 338 typename.lower(), typename.upper()) 339 for inst in instances: 340 print >> C, " %s," % c_str(inst.name) 341 print >> C, "};" 342 print >> C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % ( 343 typename.lower(), typename.upper()) 344 for inst in instances: 345 print >> C, " %s," % c_str(inst.doc) 346 print >> C, "};" 347 348 for i, tbl in enumerate(static_tables): 349 print >> C, "const %s grpc_stats_table_%d[%d] = {%s};" % ( 350 tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1])) 351 352 for histogram, code in zip(inst_map['Histogram'], histo_code): 353 print >> C, ("void grpc_stats_inc_%s(int value) {%s}") % ( 354 histogram.name.lower(), code) 355 356 print >> C, "const int grpc_stats_histo_buckets[%d] = {%s};" % ( 357 len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets)) 358 print >> C, "const int grpc_stats_histo_start[%d] = {%s};" % ( 359 len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start)) 360 print >> C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % ( 361 len(inst_map['Histogram']), ','.join( 362 'grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) 363 print >> C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % ( 364 len(inst_map['Histogram']), ','.join( 365 'grpc_stats_inc_%s' % histogram.name.lower() 366 for histogram in inst_map['Histogram'])) 367 368 # patch qps_test bigquery schema 369 RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] 370 371 with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: 372 qps_schema = json.loads(f.read()) 373 374 375 def FindNamed(js, name): 376 for el in js: 377 if el['name'] == name: 378 return el 379 380 381 def RemoveCoreFields(js): 382 new_fields = [] 383 for field in js['fields']: 384 if not field['name'].startswith('core_'): 385 new_fields.append(field) 386 js['fields'] = new_fields 387 388 389 RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) 390 RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) 391 392 393 def AddCoreFields(js): 394 for counter in inst_map['Counter']: 395 js['fields'].append({ 396 'name': 'core_%s' % counter.name, 397 'type': 'INTEGER', 398 'mode': 'NULLABLE' 399 }) 400 for histogram in inst_map['Histogram']: 401 js['fields'].append({ 402 'name': 'core_%s' % histogram.name, 403 'type': 'STRING', 404 'mode': 'NULLABLE' 405 }) 406 js['fields'].append({ 407 'name': 'core_%s_bkts' % histogram.name, 408 'type': 'STRING', 409 'mode': 'NULLABLE' 410 }) 411 for pctl in RECORD_EXPLICIT_PERCENTILES: 412 js['fields'].append({ 413 'name': 'core_%s_%dp' % (histogram.name, pctl), 414 'type': 'FLOAT', 415 'mode': 'NULLABLE' 416 }) 417 418 419 AddCoreFields(FindNamed(qps_schema, 'clientStats')) 420 AddCoreFields(FindNamed(qps_schema, 'serverStats')) 421 422 with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: 423 f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) 424 425 # and generate a helper script to massage scenario results into the format we'd 426 # like to query 427 with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: 428 with open(sys.argv[0]) as my_source: 429 for line in my_source: 430 if line[0] != '#': break 431 for line in my_source: 432 if line[0] == '#': 433 print >> P, line.rstrip() 434 break 435 for line in my_source: 436 if line[0] != '#': 437 break 438 print >> P, line.rstrip() 439 440 print >> P 441 print >> P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' 442 print >> P 443 444 print >> P, 'import massage_qps_stats_helpers' 445 446 print >> P, 'def massage_qps_stats(scenario_result):' 447 print >> P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' 448 print >> P, ' if "coreStats" in stats:' 449 print >> P, ' # Get rid of the "coreStats" element and replace it by statistics' 450 print >> P, ' # that correspond to columns in the bigquery schema.' 451 print >> P, ' core_stats = stats["coreStats"]' 452 print >> P, ' del stats["coreStats"]' 453 for counter in inst_map['Counter']: 454 print >> P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % ( 455 counter.name, counter.name) 456 for i, histogram in enumerate(inst_map['Histogram']): 457 print >> P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name 458 print >> P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name 459 print >> P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name 460 for pctl in RECORD_EXPLICIT_PERCENTILES: 461 print >> P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( 462 histogram.name, pctl, pctl) 463 464 with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: 465 columns = [] 466 for counter in inst_map['Counter']: 467 columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) 468 print >> S, ',\n'.join('%s:%s' % x for x in columns) 469
