1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "src/trace_processor/trace_processor_impl.h" 18 19 #include <inttypes.h> 20 #include <algorithm> 21 #include <functional> 22 23 #include "perfetto/base/logging.h" 24 #include "perfetto/base/string_utils.h" 25 #include "perfetto/base/time.h" 26 #include "perfetto/protozero/scattered_heap_buffer.h" 27 #include "src/trace_processor/android_logs_table.h" 28 #include "src/trace_processor/args_table.h" 29 #include "src/trace_processor/args_tracker.h" 30 #include "src/trace_processor/clock_tracker.h" 31 #include "src/trace_processor/counter_definitions_table.h" 32 #include "src/trace_processor/counter_values_table.h" 33 #include "src/trace_processor/event_tracker.h" 34 #include "src/trace_processor/fuchsia_trace_parser.h" 35 #include "src/trace_processor/fuchsia_trace_tokenizer.h" 36 #include "src/trace_processor/heap_profile_tracker.h" 37 #include "src/trace_processor/instants_table.h" 38 #include "src/trace_processor/metrics/metrics.h" 39 #include "src/trace_processor/process_table.h" 40 #include "src/trace_processor/process_tracker.h" 41 #include "src/trace_processor/proto_trace_parser.h" 42 #include "src/trace_processor/proto_trace_tokenizer.h" 43 #include "src/trace_processor/raw_table.h" 44 #include "src/trace_processor/sched_slice_table.h" 45 #include "src/trace_processor/slice_table.h" 46 #include "src/trace_processor/slice_tracker.h" 47 #include "src/trace_processor/span_join_operator_table.h" 48 #include "src/trace_processor/sql_stats_table.h" 49 #include "src/trace_processor/sqlite3_str_split.h" 50 #include "src/trace_processor/stats_table.h" 51 #include "src/trace_processor/string_table.h" 52 #include "src/trace_processor/syscall_tracker.h" 53 #include "src/trace_processor/table.h" 54 #include "src/trace_processor/thread_table.h" 55 #include "src/trace_processor/trace_blob_view.h" 56 #include "src/trace_processor/trace_sorter.h" 57 #include "src/trace_processor/window_operator_table.h" 58 59 #include "perfetto/metrics/android/mem_metric.pbzero.h" 60 #include "perfetto/metrics/metrics.pbzero.h" 61 62 // JSON parsing is only supported in the standalone build. 63 #if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) 64 #include "src/trace_processor/json_trace_parser.h" 65 #include "src/trace_processor/json_trace_tokenizer.h" 66 #endif 67 68 // In Android tree builds, we don't have the percentile module. 69 // Just don't include it. 70 #if !PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD) 71 // defined in sqlite_src/ext/misc/percentile.c 72 extern "C" int sqlite3_percentile_init(sqlite3* db, 73 char** error, 74 const sqlite3_api_routines* api); 75 #endif 76 77 namespace perfetto { 78 namespace trace_processor { 79 namespace { 80 81 std::string RemoveWhitespace(const std::string& input) { 82 std::string str(input); 83 str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end()); 84 return str; 85 } 86 87 void InitializeSqlite(sqlite3* db) { 88 char* error = nullptr; 89 sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error); 90 if (error) { 91 PERFETTO_FATAL("Error setting pragma temp_store: %s", error); 92 } 93 sqlite3_str_split_init(db); 94 // In Android tree builds, we don't have the percentile module. 95 // Just don't include it. 96 #if !PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD) 97 sqlite3_percentile_init(db, &error, nullptr); 98 if (error) { 99 PERFETTO_ELOG("Error initializing: %s", error); 100 sqlite3_free(error); 101 } 102 #endif 103 } 104 105 void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) { 106 char* error = nullptr; 107 sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error); 108 if (error) { 109 PERFETTO_ELOG("Error deleting from bounds table: %s", error); 110 sqlite3_free(error); 111 return; 112 } 113 114 char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64 115 ", %" PRId64 ")", 116 bounds.first, bounds.second); 117 118 sqlite3_exec(db, insert_sql, 0, 0, &error); 119 sqlite3_free(insert_sql); 120 if (error) { 121 PERFETTO_ELOG("Error inserting bounds table: %s", error); 122 sqlite3_free(error); 123 } 124 } 125 126 void CreateBuiltinTables(sqlite3* db) { 127 char* error = nullptr; 128 sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error); 129 if (error) { 130 PERFETTO_ELOG("Error initializing: %s", error); 131 sqlite3_free(error); 132 } 133 sqlite3_exec(db, 134 "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0, 135 0, &error); 136 if (error) { 137 PERFETTO_ELOG("Error initializing: %s", error); 138 sqlite3_free(error); 139 } 140 141 // Initialize the bounds table with some data so even before parsing any data, 142 // we still have a valid table. 143 BuildBoundsTable(db, std::make_pair(0, 0)); 144 } 145 146 void CreateBuiltinViews(sqlite3* db) { 147 char* error = nullptr; 148 sqlite3_exec(db, 149 "CREATE VIEW counters AS " 150 "SELECT * FROM counter_values " 151 "INNER JOIN counter_definitions USING(counter_id);", 152 0, 0, &error); 153 if (error) { 154 PERFETTO_ELOG("Error initializing: %s", error); 155 sqlite3_free(error); 156 } 157 158 sqlite3_exec(db, 159 "CREATE VIEW slice AS " 160 "SELECT " 161 " *, " 162 " CASE ref_type " 163 " WHEN 'utid' THEN ref " 164 " ELSE NULL " 165 " END AS utid " 166 "FROM internal_slice;", 167 0, 0, &error); 168 if (error) { 169 PERFETTO_ELOG("Error initializing: %s", error); 170 sqlite3_free(error); 171 } 172 173 // Legacy view for "slice" table with a deprecated table name. 174 // TODO(eseckler): Remove this view when all users have switched to "slice". 175 sqlite3_exec(db, 176 "CREATE VIEW slices AS " 177 "SELECT * FROM slice;", 178 0, 0, &error); 179 if (error) { 180 PERFETTO_ELOG("Error initializing: %s", error); 181 sqlite3_free(error); 182 } 183 } 184 185 void CreateMetricsFunctions(TraceProcessorImpl* tp, sqlite3* db) { 186 auto ret = sqlite3_create_function_v2(db, "RUN_METRIC", -1, SQLITE_UTF8, tp, 187 metrics::RunMetric, nullptr, nullptr, 188 sqlite_utils::kSqliteStatic); 189 if (ret) { 190 PERFETTO_ELOG("Error initializing RUN_METRIC"); 191 } 192 } 193 194 // Fuchsia traces have a magic number as documented here: 195 // https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0 196 constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010; 197 198 } // namespace 199 200 TraceType GuessTraceType(const uint8_t* data, size_t size) { 201 if (size == 0) 202 return kUnknownTraceType; 203 std::string start(reinterpret_cast<const char*>(data), 204 std::min<size_t>(size, 20)); 205 std::string start_minus_white_space = RemoveWhitespace(start); 206 if (base::StartsWith(start_minus_white_space, "{\"traceEvents\":[")) 207 return kJsonTraceType; 208 if (base::StartsWith(start_minus_white_space, "[{")) 209 return kJsonTraceType; 210 if (size >= 8) { 211 uint64_t first_word = *reinterpret_cast<const uint64_t*>(data); 212 if (first_word == kFuchsiaMagicNumber) 213 return kFuchsiaTraceType; 214 } 215 return ProtoTraceTokenizer::GuessProtoTraceType(data, size); 216 } 217 218 TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) : cfg_(cfg) { 219 sqlite3* db = nullptr; 220 PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK); 221 InitializeSqlite(db); 222 CreateBuiltinTables(db); 223 CreateBuiltinViews(db); 224 CreateMetricsFunctions(this, db); 225 db_.reset(std::move(db)); 226 227 context_.storage.reset(new TraceStorage()); 228 context_.args_tracker.reset(new ArgsTracker(&context_)); 229 context_.slice_tracker.reset(new SliceTracker(&context_)); 230 context_.event_tracker.reset(new EventTracker(&context_)); 231 context_.process_tracker.reset(new ProcessTracker(&context_)); 232 context_.syscall_tracker.reset(new SyscallTracker(&context_)); 233 context_.clock_tracker.reset(new ClockTracker(&context_)); 234 context_.heap_profile_tracker.reset(new HeapProfileTracker(&context_)); 235 236 ArgsTable::RegisterTable(*db_, context_.storage.get()); 237 ProcessTable::RegisterTable(*db_, context_.storage.get()); 238 SchedSliceTable::RegisterTable(*db_, context_.storage.get()); 239 SliceTable::RegisterTable(*db_, context_.storage.get()); 240 SqlStatsTable::RegisterTable(*db_, context_.storage.get()); 241 StringTable::RegisterTable(*db_, context_.storage.get()); 242 ThreadTable::RegisterTable(*db_, context_.storage.get()); 243 CounterDefinitionsTable::RegisterTable(*db_, context_.storage.get()); 244 CounterValuesTable::RegisterTable(*db_, context_.storage.get()); 245 SpanJoinOperatorTable::RegisterTable(*db_, context_.storage.get()); 246 WindowOperatorTable::RegisterTable(*db_, context_.storage.get()); 247 InstantsTable::RegisterTable(*db_, context_.storage.get()); 248 StatsTable::RegisterTable(*db_, context_.storage.get()); 249 AndroidLogsTable::RegisterTable(*db_, context_.storage.get()); 250 RawTable::RegisterTable(*db_, context_.storage.get()); 251 } 252 253 TraceProcessorImpl::~TraceProcessorImpl() { 254 for (auto* it : iterators_) 255 it->Reset(); 256 } 257 258 bool TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data, size_t size) { 259 if (size == 0) 260 return true; 261 if (unrecoverable_parse_error_) 262 return false; 263 264 // If this is the first Parse() call, guess the trace type and create the 265 // appropriate parser. 266 if (!context_.chunk_reader) { 267 TraceType trace_type; 268 { 269 auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats( 270 stats::guess_trace_type_duration_ns); 271 trace_type = GuessTraceType(data.get(), size); 272 } 273 int64_t window_size_ns = static_cast<int64_t>(cfg_.window_size_ns); 274 switch (trace_type) { 275 case kJsonTraceType: 276 PERFETTO_DLOG("Legacy JSON trace detected"); 277 #if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD) 278 context_.chunk_reader.reset(new JsonTraceTokenizer(&context_)); 279 // JSON traces have no guarantees about the order of events in them. 280 window_size_ns = std::numeric_limits<int64_t>::max(); 281 context_.sorter.reset(new TraceSorter(&context_, window_size_ns)); 282 context_.parser.reset(new JsonTraceParser(&context_)); 283 #else 284 PERFETTO_FATAL("JSON traces only supported in standalone mode."); 285 #endif 286 break; 287 case kProtoWithTrackEventsTraceType: 288 case kProtoTraceType: 289 if (trace_type == kProtoWithTrackEventsTraceType) { 290 // TrackEvents can be ordered arbitrarily due to out-of-order absolute 291 // timestamps and cross-packet-sequence events (e.g. async events). 292 window_size_ns = std::numeric_limits<int64_t>::max(); 293 } 294 context_.chunk_reader.reset(new ProtoTraceTokenizer(&context_)); 295 context_.sorter.reset(new TraceSorter(&context_, window_size_ns)); 296 context_.parser.reset(new ProtoTraceParser(&context_)); 297 break; 298 case kFuchsiaTraceType: 299 context_.chunk_reader.reset(new FuchsiaTraceTokenizer(&context_)); 300 context_.sorter.reset(new TraceSorter(&context_, window_size_ns)); 301 context_.parser.reset(new FuchsiaTraceParser(&context_)); 302 break; 303 case kUnknownTraceType: 304 return false; 305 } 306 } 307 308 auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats( 309 stats::parse_trace_duration_ns); 310 bool res = context_.chunk_reader->Parse(std::move(data), size); 311 unrecoverable_parse_error_ |= !res; 312 return res; 313 } 314 315 void TraceProcessorImpl::NotifyEndOfFile() { 316 if (unrecoverable_parse_error_ || !context_.chunk_reader) 317 return; 318 319 context_.sorter->ExtractEventsForced(); 320 context_.event_tracker->FlushPendingEvents(); 321 BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs()); 322 } 323 324 TraceProcessor::Iterator TraceProcessorImpl::ExecuteQuery( 325 const std::string& sql, 326 int64_t time_queued) { 327 sqlite3_stmt* raw_stmt; 328 int err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()), 329 &raw_stmt, nullptr); 330 base::Optional<std::string> error; 331 uint32_t col_count = 0; 332 if (err != SQLITE_OK) { 333 error = sqlite3_errmsg(*db_); 334 } else { 335 col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt)); 336 } 337 338 base::TimeNanos t_start = base::GetWallTimeNs(); 339 uint32_t sql_stats_row = 340 context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued, 341 t_start.count()); 342 343 std::unique_ptr<IteratorImpl> impl(new IteratorImpl( 344 this, *db_, ScopedStmt(raw_stmt), col_count, error, sql_stats_row)); 345 iterators_.emplace_back(impl.get()); 346 return TraceProcessor::Iterator(std::move(impl)); 347 } 348 349 void TraceProcessorImpl::InterruptQuery() { 350 if (!db_) 351 return; 352 query_interrupted_.store(true); 353 sqlite3_interrupt(db_.get()); 354 } 355 356 int TraceProcessorImpl::ComputeMetric( 357 const std::vector<std::string>& metric_names, 358 std::vector<uint8_t>* metrics_proto) { 359 return metrics::ComputeMetrics(this, metric_names, metrics_proto); 360 } 361 362 TraceProcessor::IteratorImpl::IteratorImpl(TraceProcessorImpl* trace_processor, 363 sqlite3* db, 364 ScopedStmt stmt, 365 uint32_t column_count, 366 base::Optional<std::string> error, 367 uint32_t sql_stats_row) 368 : trace_processor_(trace_processor), 369 db_(db), 370 stmt_(std::move(stmt)), 371 column_count_(column_count), 372 error_(error), 373 sql_stats_row_(sql_stats_row) {} 374 375 TraceProcessor::IteratorImpl::~IteratorImpl() { 376 if (trace_processor_) { 377 auto* its = &trace_processor_->iterators_; 378 auto it = std::find(its->begin(), its->end(), this); 379 PERFETTO_CHECK(it != its->end()); 380 its->erase(it); 381 382 base::TimeNanos t_end = base::GetWallTimeNs(); 383 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats(); 384 sql_stats->RecordQueryEnd(sql_stats_row_, t_end.count()); 385 } 386 } 387 388 void TraceProcessor::IteratorImpl::Reset() { 389 *this = IteratorImpl(nullptr, nullptr, ScopedStmt(), 0, base::nullopt, 0); 390 } 391 392 void TraceProcessor::IteratorImpl::RecordFirstNextInSqlStats() { 393 base::TimeNanos t_first_next = base::GetWallTimeNs(); 394 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats(); 395 sql_stats->RecordQueryFirstNext(sql_stats_row_, t_first_next.count()); 396 } 397 398 } // namespace trace_processor 399 } // namespace perfetto 400
