Home | History | Annotate | Download | only in filesystem 
      1 /*
      2  * Copyright (C) 2018 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "src/traced/probes/filesystem/inode_file_data_source.h"
     18 
     19 #include <dirent.h>
     20 #include <sys/stat.h>
     21 #include <sys/types.h>
     22 #include <unistd.h>
     23 #include <queue>
     24 #include <unordered_map>
     25 
     26 #include "perfetto/base/logging.h"
     27 #include "perfetto/base/scoped_file.h"
     28 #include "perfetto/tracing/core/trace_packet.h"
     29 #include "perfetto/tracing/core/trace_writer.h"
     30 
     31 #include "perfetto/trace/trace_packet.pbzero.h"
     32 #include "src/traced/probes/filesystem/file_scanner.h"
     33 
     34 namespace perfetto {
     35 namespace {
     36 constexpr uint32_t kScanIntervalMs = 10000;  // 10s
     37 constexpr uint32_t kScanDelayMs = 10000;     // 10s
     38 constexpr uint32_t kScanBatchSize = 15000;
     39 
     40 uint32_t OrDefault(uint32_t value, uint32_t def) {
     41   return value ? value : def;
     42 }
     43 
     44 std::string DbgFmt(const std::vector<std::string>& values) {
     45   if (values.empty())
     46     return "";
     47 
     48   std::string result;
     49   for (auto it = values.cbegin(); it != values.cend() - 1; ++it)
     50     result += *it + ",";
     51 
     52   result += values.back();
     53   return result;
     54 }
     55 
     56 std::map<std::string, std::vector<std::string>> BuildMountpointMapping(
     57     const DataSourceConfig& source_config) {
     58   std::map<std::string, std::vector<std::string>> m;
     59   for (const auto& map_entry :
     60        source_config.inode_file_config().mount_point_mapping())
     61     m.emplace(map_entry.mountpoint(), map_entry.scan_roots());
     62 
     63   return m;
     64 }
     65 
     66 class StaticMapDelegate : public FileScanner::Delegate {
     67  public:
     68   StaticMapDelegate(
     69       std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>* map)
     70       : map_(map) {}
     71   ~StaticMapDelegate() {}
     72 
     73  private:
     74   bool OnInodeFound(BlockDeviceID block_device_id,
     75                     Inode inode_number,
     76                     const std::string& path,
     77                     protos::pbzero::InodeFileMap_Entry_Type type) {
     78     std::unordered_map<Inode, InodeMapValue>& inode_map =
     79         (*map_)[block_device_id];
     80     inode_map[inode_number].SetType(type);
     81     inode_map[inode_number].AddPath(path);
     82     return true;
     83   }
     84   void OnInodeScanDone() {}
     85   std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>* map_;
     86 };
     87 
     88 }  // namespace
     89 
     90 void CreateStaticDeviceToInodeMap(
     91     const std::string& root_directory,
     92     std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>*
     93         static_file_map) {
     94   StaticMapDelegate delegate(static_file_map);
     95   FileScanner scanner({root_directory}, &delegate);
     96   scanner.Scan();
     97 }
     98 
     99 void InodeFileDataSource::FillInodeEntry(InodeFileMap* destination,
    100                                          Inode inode_number,
    101                                          const InodeMapValue& inode_map_value) {
    102   auto* entry = destination->add_entries();
    103   entry->set_inode_number(inode_number);
    104   entry->set_type(inode_map_value.type());
    105   for (const auto& path : inode_map_value.paths())
    106     entry->add_paths(path.c_str());
    107 }
    108 
    109 InodeFileDataSource::InodeFileDataSource(
    110     DataSourceConfig source_config,
    111     base::TaskRunner* task_runner,
    112     TracingSessionID id,
    113     std::map<BlockDeviceID, std::unordered_map<Inode, InodeMapValue>>*
    114         static_file_map,
    115     LRUInodeCache* cache,
    116     std::unique_ptr<TraceWriter> writer)
    117     : source_config_(std::move(source_config)),
    118       scan_mount_points_(
    119           source_config_.inode_file_config().scan_mount_points().cbegin(),
    120           source_config_.inode_file_config().scan_mount_points().cend()),
    121       mount_point_mapping_(BuildMountpointMapping(source_config_)),
    122       task_runner_(task_runner),
    123       session_id_(id),
    124       static_file_map_(static_file_map),
    125       cache_(cache),
    126       writer_(std::move(writer)),
    127       weak_factory_(this) {}
    128 
    129 InodeFileDataSource::~InodeFileDataSource() = default;
    130 
    131 void InodeFileDataSource::AddInodesFromStaticMap(
    132     BlockDeviceID block_device_id,
    133     std::set<Inode>* inode_numbers) {
    134   // Check if block device id exists in static file map
    135   auto static_map_entry = static_file_map_->find(block_device_id);
    136   if (static_map_entry == static_file_map_->end())
    137     return;
    138 
    139   uint64_t system_found_count = 0;
    140   for (auto it = inode_numbers->begin(); it != inode_numbers->end();) {
    141     Inode inode_number = *it;
    142     // Check if inode number exists in static file map for given block device id
    143     auto inode_it = static_map_entry->second.find(inode_number);
    144     if (inode_it == static_map_entry->second.end()) {
    145       ++it;
    146       continue;
    147     }
    148     system_found_count++;
    149     it = inode_numbers->erase(it);
    150     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    151                    inode_it->second);
    152   }
    153   PERFETTO_DLOG("%" PRIu64 " inodes found in static file map",
    154                 system_found_count);
    155 }
    156 
    157 void InodeFileDataSource::AddInodesFromLRUCache(
    158     BlockDeviceID block_device_id,
    159     std::set<Inode>* inode_numbers) {
    160   uint64_t cache_found_count = 0;
    161   for (auto it = inode_numbers->begin(); it != inode_numbers->end();) {
    162     Inode inode_number = *it;
    163     auto value = cache_->Get(std::make_pair(block_device_id, inode_number));
    164     if (value == nullptr) {
    165       ++it;
    166       continue;
    167     }
    168     cache_found_count++;
    169     it = inode_numbers->erase(it);
    170     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    171                    *value);
    172   }
    173   if (cache_found_count > 0)
    174     PERFETTO_DLOG("%" PRIu64 " inodes found in cache", cache_found_count);
    175 }
    176 
    177 void InodeFileDataSource::Flush() {
    178   ResetTracePacket();
    179   writer_->Flush();
    180 }
    181 
    182 void InodeFileDataSource::OnInodes(
    183     const std::vector<std::pair<Inode, BlockDeviceID>>& inodes) {
    184   if (mount_points_.empty()) {
    185     mount_points_ = ParseMounts();
    186   }
    187   // Group inodes from FtraceMetadata by block device
    188   std::map<BlockDeviceID, std::set<Inode>> inode_file_maps;
    189   for (const auto& inodes_pair : inodes) {
    190     Inode inode_number = inodes_pair.first;
    191     BlockDeviceID block_device_id = inodes_pair.second;
    192     inode_file_maps[block_device_id].emplace(inode_number);
    193   }
    194   if (inode_file_maps.size() > 1)
    195     PERFETTO_DLOG("Saw %zu block devices.", inode_file_maps.size());
    196 
    197   // Write a TracePacket with an InodeFileMap proto for each block device id
    198   for (auto& inode_file_map_data : inode_file_maps) {
    199     BlockDeviceID block_device_id = inode_file_map_data.first;
    200     std::set<Inode>& inode_numbers = inode_file_map_data.second;
    201     PERFETTO_DLOG("Saw %zu unique inode numbers.", inode_numbers.size());
    202 
    203     // Add entries to InodeFileMap as inodes are found and resolved to their
    204     // paths/type
    205     AddInodesFromStaticMap(block_device_id, &inode_numbers);
    206     AddInodesFromLRUCache(block_device_id, &inode_numbers);
    207 
    208     if (source_config_.inode_file_config().do_not_scan())
    209       inode_numbers.clear();
    210 
    211     // If we defined mount points we want to scan in the config,
    212     // skip inodes on other mount points.
    213     if (!scan_mount_points_.empty()) {
    214       bool process = true;
    215       auto range = mount_points_.equal_range(block_device_id);
    216       for (auto it = range.first; it != range.second; ++it) {
    217         if (scan_mount_points_.count(it->second) == 0) {
    218           process = false;
    219           break;
    220         }
    221       }
    222       if (!process)
    223         continue;
    224     }
    225 
    226     if (!inode_numbers.empty()) {
    227       // Try to piggy back the current scan.
    228       auto it = missing_inodes_.find(block_device_id);
    229       if (it != missing_inodes_.end()) {
    230         it->second.insert(inode_numbers.cbegin(), inode_numbers.cend());
    231       }
    232       next_missing_inodes_[block_device_id].insert(inode_numbers.cbegin(),
    233                                                    inode_numbers.cend());
    234       if (!scan_running_) {
    235         scan_running_ = true;
    236         auto weak_this = GetWeakPtr();
    237         task_runner_->PostDelayedTask(
    238             [weak_this] {
    239               if (!weak_this) {
    240                 PERFETTO_DLOG("Giving up filesystem scan.");
    241                 return;
    242               }
    243               weak_this.get()->FindMissingInodes();
    244             },
    245             GetScanDelayMs());
    246       }
    247     }
    248   }
    249 }
    250 
    251 InodeFileMap* InodeFileDataSource::AddToCurrentTracePacket(
    252     BlockDeviceID block_device_id) {
    253   seen_block_devices_.emplace(block_device_id);
    254   if (!has_current_trace_packet_ ||
    255       current_block_device_id_ != block_device_id) {
    256     if (has_current_trace_packet_)
    257       current_trace_packet_->Finalize();
    258     current_trace_packet_ = writer_->NewTracePacket();
    259     current_file_map_ = current_trace_packet_->set_inode_file_map();
    260     has_current_trace_packet_ = true;
    261 
    262     // Add block device id to InodeFileMap
    263     current_file_map_->set_block_device_id(
    264         static_cast<uint64_t>(block_device_id));
    265     // Add mount points to InodeFileMap
    266     auto range = mount_points_.equal_range(block_device_id);
    267     for (std::multimap<BlockDeviceID, std::string>::iterator it = range.first;
    268          it != range.second; ++it)
    269       current_file_map_->add_mount_points(it->second.c_str());
    270   }
    271   return current_file_map_;
    272 }
    273 
    274 void InodeFileDataSource::RemoveFromNextMissingInodes(
    275     BlockDeviceID block_device_id,
    276     Inode inode_number) {
    277   auto it = next_missing_inodes_.find(block_device_id);
    278   if (it == next_missing_inodes_.end())
    279     return;
    280   it->second.erase(inode_number);
    281 }
    282 
    283 bool InodeFileDataSource::OnInodeFound(
    284     BlockDeviceID block_device_id,
    285     Inode inode_number,
    286     const std::string& path,
    287     protos::pbzero::InodeFileMap_Entry_Type type) {
    288   auto it = missing_inodes_.find(block_device_id);
    289   if (it == missing_inodes_.end())
    290     return true;
    291 
    292   size_t n = it->second.erase(inode_number);
    293   if (n == 0)
    294     return true;
    295 
    296   if (it->second.empty())
    297     missing_inodes_.erase(it);
    298 
    299   RemoveFromNextMissingInodes(block_device_id, inode_number);
    300 
    301   std::pair<BlockDeviceID, Inode> key{block_device_id, inode_number};
    302   auto cur_val = cache_->Get(key);
    303   if (cur_val) {
    304     cur_val->AddPath(path);
    305     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    306                    *cur_val);
    307   } else {
    308     InodeMapValue new_val(InodeMapValue(type, {path}));
    309     cache_->Insert(key, new_val);
    310     FillInodeEntry(AddToCurrentTracePacket(block_device_id), inode_number,
    311                    new_val);
    312   }
    313   PERFETTO_DLOG("Filled %s", path.c_str());
    314   return !missing_inodes_.empty();
    315 }
    316 
    317 void InodeFileDataSource::ResetTracePacket() {
    318   current_block_device_id_ = 0;
    319   current_file_map_ = nullptr;
    320   if (has_current_trace_packet_)
    321     current_trace_packet_->Finalize();
    322   has_current_trace_packet_ = false;
    323 }
    324 
    325 void InodeFileDataSource::OnInodeScanDone() {
    326   // Finalize the accumulated trace packets.
    327   ResetTracePacket();
    328   file_scanner_.reset();
    329   if (!missing_inodes_.empty()) {
    330     // At least write mount point mapping for inodes that are not found.
    331     for (const auto& p : missing_inodes_) {
    332       if (seen_block_devices_.count(p.first) == 0)
    333         AddToCurrentTracePacket(p.first);
    334     }
    335   }
    336 
    337   if (next_missing_inodes_.empty()) {
    338     scan_running_ = false;
    339   } else {
    340     auto weak_this = GetWeakPtr();
    341     PERFETTO_DLOG("Starting another filesystem scan.");
    342     task_runner_->PostDelayedTask(
    343         [weak_this] {
    344           if (!weak_this) {
    345             PERFETTO_DLOG("Giving up filesystem scan.");
    346             return;
    347           }
    348           weak_this->FindMissingInodes();
    349         },
    350         GetScanDelayMs());
    351   }
    352 }
    353 
    354 void InodeFileDataSource::AddRootsForBlockDevice(
    355     BlockDeviceID block_device_id,
    356     std::vector<std::string>* roots) {
    357   auto range = mount_points_.equal_range(block_device_id);
    358   for (auto it = range.first; it != range.second; ++it) {
    359     PERFETTO_DLOG("Trying to replace %s", it->second.c_str());
    360     auto replace_it = mount_point_mapping_.find(it->second);
    361     if (replace_it != mount_point_mapping_.end()) {
    362       roots->insert(roots->end(), replace_it->second.cbegin(),
    363                     replace_it->second.cend());
    364       return;
    365     }
    366   }
    367 
    368   for (auto it = range.first; it != range.second; ++it)
    369     roots->emplace_back(it->second);
    370 }
    371 
    372 void InodeFileDataSource::FindMissingInodes() {
    373   missing_inodes_ = std::move(next_missing_inodes_);
    374   std::vector<std::string> roots;
    375   for (auto& p : missing_inodes_)
    376     AddRootsForBlockDevice(p.first, &roots);
    377 
    378   PERFETTO_DCHECK(file_scanner_.get() == nullptr);
    379   auto weak_this = GetWeakPtr();
    380   PERFETTO_DLOG("Starting scan of %s", DbgFmt(roots).c_str());
    381   file_scanner_ = std::unique_ptr<FileScanner>(new FileScanner(
    382       std::move(roots), this, GetScanIntervalMs(), GetScanBatchSize()));
    383 
    384   file_scanner_->Scan(task_runner_);
    385 }
    386 
    387 uint32_t InodeFileDataSource::GetScanIntervalMs() const {
    388   return OrDefault(source_config_.inode_file_config().scan_interval_ms(),
    389                    kScanIntervalMs);
    390 }
    391 
    392 uint32_t InodeFileDataSource::GetScanDelayMs() const {
    393   return OrDefault(source_config_.inode_file_config().scan_delay_ms(),
    394                    kScanDelayMs);
    395 }
    396 
    397 uint32_t InodeFileDataSource::GetScanBatchSize() const {
    398   return OrDefault(source_config_.inode_file_config().scan_batch_size(),
    399                    kScanBatchSize);
    400 }
    401 
    402 base::WeakPtr<InodeFileDataSource> InodeFileDataSource::GetWeakPtr() const {
    403   return weak_factory_.GetWeakPtr();
    404 }
    405 
    406 }  // namespace perfetto
    407