Home | History | Annotate | Download | only in filesystem 
      1 /*
      2  * Copyright (C) 2018 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "src/traced/probes/filesystem/inode_file_data_source.h"
     18 
     19 #include <dirent.h>
     20 #include <sys/stat.h>
     21 #include <sys/types.h>
     22 #include <unistd.h>
     23 #include <queue>
     24 #include <unordered_map>
     25 
     26 #include "perfetto/base/logging.h"
     27 #include "perfetto/base/scoped_file.h"
     28 #include "perfetto/tracing/core/trace_packet.h"
     29 #include "perfetto/tracing/core/trace_writer.h"
     30 
     31 #include "perfetto/trace/trace_packet.pbzero.h"
     32 #include "src/traced/probes/filesystem/file_scanner.h"
     33 
     34 namespace perfetto {
     35 namespace {
     36 constexpr uint32_t kScanIntervalMs = 10000;  // 10s
     37 constexpr uint32_t kScanDelayMs = 10000;     // 10s
     38 constexpr uint32_t kScanBatchSize = 15000;
     39 
     40 uint32_t OrDefault(uint32_t value, uint32_t def) {
     41   return value ? value : def;
     42 }
     43 
     44 std::string DbgFmt(const std::vector<std::string>& values) {
     45   if (values.empty())
     46     return "";
     47 
     48   std::string result;
     49   for (auto it = values.cbegin(); it != values.cend() - 1; ++it)
     50     result += *it + ",";
     51 
     52   result += values.back();
     53   return result;
     54 }
     55 
     56 std::map<std::string, std::vector<std::string>> BuildMountpointMapping(
     57     const DataSourceConfig& source_config) {
     58   std::map<std::string, std::vector<std::string>> m;
     59   for (const auto& map_entry :
     60        source_config.inode_file_config().mount_point_mapping())
     61     m.emplace(map_entry.mountpoint(), map_entry.scan_roots());
     62 
     63   return m;
     64 }
     65 
     66 class StaticMapDelegate : public FileScanner::Delegate {
     67  public:
     68   StaticMapDelegate(
     69       std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>* map)
     70       : map_(map) {}
     71   ~StaticMapDelegate() {}
     72 
     73  private:
     74   bool OnInodeFound(BlockDeviceID block_device_id,
     75                     Inode inode_number,
     76                     const std::string& path,
     77                     protos::pbzero::InodeFileMap_Entry_Type type) {
     78     std::unordered_map<Inode, InodeMapValue>& inode_map =
     79         (*map_)[block_device_id];
     80     inode_map[inode_number].SetType(type);
     81     inode_map[inode_number].AddPath(path);
     82     return true;
     83   }
     84   void OnInodeScanDone() {}
     85   std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>* map_;
     86 };
     87 
     88 }  // namespace
     89 
     90 // static
     91 constexpr int InodeFileDataSource::kTypeId;
     92 
     93 void CreateStaticDeviceToInodeMap(
     94     const std::string& root_directory,
     95     std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>*
     96         static_file_map) {
     97   StaticMapDelegate delegate(static_file_map);
     98   FileScanner scanner({root_directory}, &delegate);
     99   scanner.Scan();
    100 }
    101 
    102 void InodeFileDataSource::FillInodeEntry(InodeFileMap* destination,
    103                                          Inode inode_number,
    104                                          const InodeMapValue& inode_map_value) {
    105   auto* entry = destination->add_entries();
    106   entry->set_inode_number(inode_number);
    107   entry->set_type(inode_map_value.type());
    108   for (const auto& path : inode_map_value.paths())
    109     entry->add_paths(path.c_str());
    110 }
    111 
    112 InodeFileDataSource::InodeFileDataSource(
    113     DataSourceConfig source_config,
    114     base::TaskRunner* task_runner,
    115     TracingSessionID session_id,
    116     std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>*
    117         static_file_map,
    118     LRUInodeCache* cache,
    119     std::unique_ptr<TraceWriter> writer)
    120     : ProbesDataSource(session_id, kTypeId),
    121       source_config_(std::move(source_config)),
    122       scan_mount_points_(
    123           source_config_.inode_file_config().scan_mount_points().cbegin(),
    124           source_config_.inode_file_config().scan_mount_points().cend()),
    125       mount_point_mapping_(BuildMountpointMapping(source_config_)),
    126       task_runner_(task_runner),
    127       static_file_map_(static_file_map),
    128       cache_(cache),
    129       writer_(std::move(writer)),
    130       weak_factory_(this) {}
    131 
    132 InodeFileDataSource::~InodeFileDataSource() = default;
    133 
    134 void InodeFileDataSource::Start() {
    135   // Nothing special to do, this data source is only reacting to on-demand
    136   // events such as OnInodes().
    137 }
    138 
    139 void InodeFileDataSource::AddInodesFromStaticMap(
    140     BlockDeviceID block_device_id,
    141     std::set<Inode>* inode_numbers) {
    142   // Check if block device id exists in static file map
    143   auto static_map_entry = static_file_map_->find(block_device_id);
    144   if (static_map_entry == static_file_map_->end())
    145     return;
    146 
    147   uint64_t system_found_count = 0;
    148   for (auto it = inode_numbers->begin(); it != inode_numbers->end();) {
    149     Inode inode_number = *it;
    150     // Check if inode number exists in static file map for given block device id
    151     auto inode_it = static_map_entry->second.find(inode_number);
    152     if (inode_it == static_map_entry->second.end()) {
    153       ++it;
    154       continue;
    155     }
    156     system_found_count++;
    157     it = inode_numbers->erase(it);
    158     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    159                    inode_it->second);
    160   }
    161   PERFETTO_DLOG("%" PRIu64 " inodes found in static file map",
    162                 system_found_count);
    163 }
    164 
    165 void InodeFileDataSource::AddInodesFromLRUCache(
    166     BlockDeviceID block_device_id,
    167     std::set<Inode>* inode_numbers) {
    168   uint64_t cache_found_count = 0;
    169   for (auto it = inode_numbers->begin(); it != inode_numbers->end();) {
    170     Inode inode_number = *it;
    171     auto value = cache_->Get(std::make_pair(block_device_id, inode_number));
    172     if (value == nullptr) {
    173       ++it;
    174       continue;
    175     }
    176     cache_found_count++;
    177     it = inode_numbers->erase(it);
    178     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    179                    *value);
    180   }
    181   if (cache_found_count > 0)
    182     PERFETTO_DLOG("%" PRIu64 " inodes found in cache", cache_found_count);
    183 }
    184 
    185 void InodeFileDataSource::Flush(FlushRequestID,
    186                                 std::function<void()> callback) {
    187   ResetTracePacket();
    188   writer_->Flush(callback);
    189 }
    190 
    191 void InodeFileDataSource::OnInodes(
    192     const std::vector<std::pair<Inode, BlockDeviceID>>& inodes) {
    193   if (mount_points_.empty()) {
    194     mount_points_ = ParseMounts();
    195   }
    196   // Group inodes from FtraceMetadata by block device
    197   std::map<BlockDeviceID, std::set<Inode>> inode_file_maps;
    198   for (const auto& inodes_pair : inodes) {
    199     Inode inode_number = inodes_pair.first;
    200     BlockDeviceID block_device_id = inodes_pair.second;
    201     inode_file_maps[block_device_id].emplace(inode_number);
    202   }
    203   if (inode_file_maps.size() > 1)
    204     PERFETTO_DLOG("Saw %zu block devices.", inode_file_maps.size());
    205 
    206   // Write a TracePacket with an InodeFileMap proto for each block device id
    207   for (auto& inode_file_map_data : inode_file_maps) {
    208     BlockDeviceID block_device_id = inode_file_map_data.first;
    209     std::set<Inode>& inode_numbers = inode_file_map_data.second;
    210     PERFETTO_DLOG("Saw %zu unique inode numbers.", inode_numbers.size());
    211 
    212     // Add entries to InodeFileMap as inodes are found and resolved to their
    213     // paths/type
    214     AddInodesFromStaticMap(block_device_id, &inode_numbers);
    215     AddInodesFromLRUCache(block_device_id, &inode_numbers);
    216 
    217     if (source_config_.inode_file_config().do_not_scan())
    218       inode_numbers.clear();
    219 
    220     // If we defined mount points we want to scan in the config,
    221     // skip inodes on other mount points.
    222     if (!scan_mount_points_.empty()) {
    223       bool process = true;
    224       auto range = mount_points_.equal_range(block_device_id);
    225       for (auto it = range.first; it != range.second; ++it) {
    226         if (scan_mount_points_.count(it->second) == 0) {
    227           process = false;
    228           break;
    229         }
    230       }
    231       if (!process)
    232         continue;
    233     }
    234 
    235     if (!inode_numbers.empty()) {
    236       // Try to piggy back the current scan.
    237       auto it = missing_inodes_.find(block_device_id);
    238       if (it != missing_inodes_.end()) {
    239         it->second.insert(inode_numbers.cbegin(), inode_numbers.cend());
    240       }
    241       next_missing_inodes_[block_device_id].insert(inode_numbers.cbegin(),
    242                                                    inode_numbers.cend());
    243       if (!scan_running_) {
    244         scan_running_ = true;
    245         auto weak_this = GetWeakPtr();
    246         task_runner_->PostDelayedTask(
    247             [weak_this] {
    248               if (!weak_this) {
    249                 PERFETTO_DLOG("Giving up filesystem scan.");
    250                 return;
    251               }
    252               weak_this.get()->FindMissingInodes();
    253             },
    254             GetScanDelayMs());
    255       }
    256     }
    257   }
    258 }
    259 
    260 InodeFileMap* InodeFileDataSource::AddToCurrentTracePacket(
    261     BlockDeviceID block_device_id) {
    262   seen_block_devices_.emplace(block_device_id);
    263   if (!has_current_trace_packet_ ||
    264       current_block_device_id_ != block_device_id) {
    265     if (has_current_trace_packet_)
    266       current_trace_packet_->Finalize();
    267     current_trace_packet_ = writer_->NewTracePacket();
    268     current_file_map_ = current_trace_packet_->set_inode_file_map();
    269     has_current_trace_packet_ = true;
    270 
    271     // Add block device id to InodeFileMap
    272     current_file_map_->set_block_device_id(
    273         static_cast<uint64_t>(block_device_id));
    274     // Add mount points to InodeFileMap
    275     auto range = mount_points_.equal_range(block_device_id);
    276     for (std::multimap<BlockDeviceID, std::string>::iterator it = range.first;
    277          it != range.second; ++it)
    278       current_file_map_->add_mount_points(it->second.c_str());
    279   }
    280   return current_file_map_;
    281 }
    282 
    283 void InodeFileDataSource::RemoveFromNextMissingInodes(
    284     BlockDeviceID block_device_id,
    285     Inode inode_number) {
    286   auto it = next_missing_inodes_.find(block_device_id);
    287   if (it == next_missing_inodes_.end())
    288     return;
    289   it->second.erase(inode_number);
    290 }
    291 
    292 bool InodeFileDataSource::OnInodeFound(
    293     BlockDeviceID block_device_id,
    294     Inode inode_number,
    295     const std::string& path,
    296     protos::pbzero::InodeFileMap_Entry_Type type) {
    297   auto it = missing_inodes_.find(block_device_id);
    298   if (it == missing_inodes_.end())
    299     return true;
    300 
    301   size_t n = it->second.erase(inode_number);
    302   if (n == 0)
    303     return true;
    304 
    305   if (it->second.empty())
    306     missing_inodes_.erase(it);
    307 
    308   RemoveFromNextMissingInodes(block_device_id, inode_number);
    309 
    310   std::pair<BlockDeviceID, Inode> key{block_device_id, inode_number};
    311   auto cur_val = cache_->Get(key);
    312   if (cur_val) {
    313     cur_val->AddPath(path);
    314     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    315                    *cur_val);
    316   } else {
    317     InodeMapValue new_val(InodeMapValue(type, {path}));
    318     cache_->Insert(key, new_val);
    319     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    320                    new_val);
    321   }
    322   PERFETTO_DLOG("Filled %s", path.c_str());
    323   return !missing_inodes_.empty();
    324 }
    325 
    326 void InodeFileDataSource::ResetTracePacket() {
    327   current_block_device_id_ = 0;
    328   current_file_map_ = nullptr;
    329   if (has_current_trace_packet_)
    330     current_trace_packet_->Finalize();
    331   has_current_trace_packet_ = false;
    332 }
    333 
    334 void InodeFileDataSource::OnInodeScanDone() {
    335   // Finalize the accumulated trace packets.
    336   ResetTracePacket();
    337   file_scanner_.reset();
    338   if (!missing_inodes_.empty()) {
    339     // At least write mount point mapping for inodes that are not found.
    340     for (const auto& p : missing_inodes_) {
    341       if (seen_block_devices_.count(p.first) == 0)
    342         AddToCurrentTracePacket(p.first);
    343     }
    344   }
    345 
    346   if (next_missing_inodes_.empty()) {
    347     scan_running_ = false;
    348   } else {
    349     auto weak_this = GetWeakPtr();
    350     PERFETTO_DLOG("Starting another filesystem scan.");
    351     task_runner_->PostDelayedTask(
    352         [weak_this] {
    353           if (!weak_this) {
    354             PERFETTO_DLOG("Giving up filesystem scan.");
    355             return;
    356           }
    357           weak_this->FindMissingInodes();
    358         },
    359         GetScanDelayMs());
    360   }
    361 }
    362 
    363 void InodeFileDataSource::AddRootsForBlockDevice(
    364     BlockDeviceID block_device_id,
    365     std::vector<std::string>* roots) {
    366   auto range = mount_points_.equal_range(block_device_id);
    367   for (auto it = range.first; it != range.second; ++it) {
    368     PERFETTO_DLOG("Trying to replace %s", it->second.c_str());
    369     auto replace_it = mount_point_mapping_.find(it->second);
    370     if (replace_it != mount_point_mapping_.end()) {
    371       roots->insert(roots->end(), replace_it->second.cbegin(),
    372                     replace_it->second.cend());
    373       return;
    374     }
    375   }
    376 
    377   for (auto it = range.first; it != range.second; ++it)
    378     roots->emplace_back(it->second);
    379 }
    380 
    381 void InodeFileDataSource::FindMissingInodes() {
    382   missing_inodes_ = std::move(next_missing_inodes_);
    383   std::vector<std::string> roots;
    384   for (auto& p : missing_inodes_)
    385     AddRootsForBlockDevice(p.first, &roots);
    386 
    387   PERFETTO_DCHECK(file_scanner_.get() == nullptr);
    388   auto weak_this = GetWeakPtr();
    389   PERFETTO_DLOG("Starting scan of %s", DbgFmt(roots).c_str());
    390   file_scanner_ = std::unique_ptr<FileScanner>(new FileScanner(
    391       std::move(roots), this, GetScanIntervalMs(), GetScanBatchSize()));
    392 
    393   file_scanner_->Scan(task_runner_);
    394 }
    395 
    396 uint32_t InodeFileDataSource::GetScanIntervalMs() const {
    397   return OrDefault(source_config_.inode_file_config().scan_interval_ms(),
    398                    kScanIntervalMs);
    399 }
    400 
    401 uint32_t InodeFileDataSource::GetScanDelayMs() const {
    402   return OrDefault(source_config_.inode_file_config().scan_delay_ms(),
    403                    kScanDelayMs);
    404 }
    405 
    406 uint32_t InodeFileDataSource::GetScanBatchSize() const {
    407   return OrDefault(source_config_.inode_file_config().scan_batch_size(),
    408                    kScanBatchSize);
    409 }
    410 
    411 base::WeakPtr<InodeFileDataSource> InodeFileDataSource::GetWeakPtr() const {
    412   return weak_factory_.GetWeakPtr();
    413 }
    414 
    415 }  // namespace perfetto
    416