1 #!/usr/bin/python 2 # 3 # Copyright (C) 2018 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import argparse 18 import itertools 19 import json 20 import sqlite3 21 22 class SqliteWriter(object): 23 def __init__(self): 24 self.sample_count = 0 25 self.dso_map = {} 26 self.pid_map = {} 27 self.tid_map = {} 28 self.symbol_map = {} 29 30 def open(self, out): 31 self._conn = sqlite3.connect(out) 32 self._c = self._conn.cursor() 33 # Ensure tables exist 34 # The sample replicates pid and tid. 35 try: 36 self._c.execute('''CREATE TABLE pids (id integer PRIMARY KEY AUTOINCREMENT, 37 name text)''') 38 self._c.execute('''CREATE TABLE tids (id integer PRIMARY KEY AUTOINCREMENT, 39 name text)''') 40 self._c.execute('''CREATE TABLE syms (id integer PRIMARY KEY AUTOINCREMENT, 41 name text)''') 42 self._c.execute('''CREATE TABLE dsos (id integer PRIMARY KEY AUTOINCREMENT, 43 name text)''') 44 self._c.execute('''CREATE TABLE samples (id integer PRIMARY KEY AUTOINCREMENT, 45 pid_id int not null, 46 tid_id int not null) 47 ''') 48 self._c.execute('''CREATE TABLE stacks (sample_id int not null, 49 depth int not null, 50 dso_id int not null, 51 sym_id int not null, 52 offset int not null, 53 primary key (sample_id, depth)) 54 ''') 55 except sqlite3.OperationalError: 56 pass # ignore 57 58 def close(self): 59 self._conn.commit() 60 self._conn.close() 61 62 def insert_into_tmp_or_get(self, name, table_dict, table_dict_tmp): 63 if name in table_dict: 64 return table_dict[name] 65 if name in table_dict_tmp: 66 return table_dict_tmp[name] 67 index = len(table_dict) + len(table_dict_tmp) 68 table_dict_tmp[name] = index 69 return index 70 71 def prepare(self): 72 self.dso_tmp_map = {} 73 self.pid_tmp_map = {} 74 self.tid_tmp_map = {} 75 self.symbol_tmp_map = {} 76 self.samples_tmp_list = [] 77 self.stacks_tmp_list = [] 78 79 def write_sqlite_index_table(self, table_dict, table_name): 80 for key, value in table_dict.iteritems(): 81 self._c.execute("insert into {tn} values (?,?)".format(tn=table_name), (value,key)) 82 83 def flush(self): 84 self.write_sqlite_index_table(self.pid_tmp_map, 'pids') 85 self.write_sqlite_index_table(self.tid_tmp_map, 'tids') 86 self.write_sqlite_index_table(self.dso_tmp_map, 'dsos') 87 self.write_sqlite_index_table(self.symbol_tmp_map, 'syms') 88 89 for sample in self.samples_tmp_list: 90 self._c.execute("insert into samples values (?,?,?)", sample) 91 for stack in self.stacks_tmp_list: 92 self._c.execute("insert into stacks values (?,?,?,?,?)", stack) 93 94 self.pid_map.update(self.pid_tmp_map) 95 self.tid_map.update(self.tid_tmp_map) 96 self.dso_map.update(self.dso_tmp_map) 97 self.symbol_map.update(self.symbol_tmp_map) 98 99 self.dso_tmp_map = {} 100 self.pid_tmp_map = {} 101 self.tid_tmp_map = {} 102 self.symbol_tmp_map = {} 103 self.samples_tmp_list = [] 104 self.stacks_tmp_list = [] 105 106 def add_sample(self, sample, tid_name_map): 107 sample_id = self.sample_count 108 self.sample_count = self.sample_count + 1 109 110 def get_name(pid, name_map): 111 if pid in name_map: 112 return name_map[pid] 113 pid_str = str(pid) 114 if pid_str in name_map: 115 return name_map[pid_str] 116 if pid == 0: 117 return "[kernel]" 118 return "[unknown]" 119 120 pid_name = get_name(sample[0], tid_name_map) 121 pid_id = self.insert_into_tmp_or_get(pid_name, self.pid_map, self.pid_tmp_map) 122 tid_name = get_name(sample[1], tid_name_map) 123 tid_id = self.insert_into_tmp_or_get(tid_name, self.tid_map, self.tid_tmp_map) 124 125 self.samples_tmp_list.append((sample_id, pid_id, tid_id)) 126 127 stack_depth = 0 128 for entry in sample[2]: 129 sym_id = self.insert_into_tmp_or_get(entry[0], self.symbol_map, self.symbol_tmp_map) 130 dso = entry[2] 131 if dso is None: 132 dso = "None" 133 dso_id = self.insert_into_tmp_or_get(dso, self.dso_map, self.dso_tmp_map) 134 135 self.stacks_tmp_list.append((sample_id, stack_depth, dso_id, sym_id, entry[1])) 136 137 stack_depth = stack_depth + 1 138 139 140 if __name__ == "__main__": 141 parser = argparse.ArgumentParser(description='''Process a set of perfprofd JSON files produced 142 by perf_proto_stack.py into SQLite database''') 143 144 parser.add_argument('file', help='JSON files to parse and combine', metavar='file', nargs='+') 145 146 parser.add_argument('--sqlite-out', help='SQLite database output', type=str, 147 default='sqlite.db') 148 149 args = parser.parse_args() 150 if args is not None: 151 sql_out = SqliteWriter() 152 sql_out.open(args.sqlite_out) 153 sql_out.prepare() 154 155 for f in args.file: 156 print 'Processing %s' % (f) 157 fp = open(f, 'r') 158 data = json.load(fp) 159 fp.close() 160 161 for sample in data['samples']: 162 sql_out.add_sample(sample, data['names']) 163 164 sql_out.flush() 165 166 sql_out.close() 167
