1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import os 7 import sys 8 import unittest 9 10 from caching_file_system import CachingFileSystem 11 from extensions_paths import SERVER2 12 from file_system import StatInfo 13 from local_file_system import LocalFileSystem 14 from mock_file_system import MockFileSystem 15 from object_store_creator import ObjectStoreCreator 16 from test_file_system import TestFileSystem 17 from test_object_store import TestObjectStore 18 19 20 def _CreateLocalFs(): 21 return LocalFileSystem.Create(SERVER2, 'test_data', 'file_system/') 22 23 24 class CachingFileSystemTest(unittest.TestCase): 25 def setUp(self): 26 # Use this to make sure that every time _CreateCachingFileSystem is called 27 # the underlying object store data is the same, within each test. 28 self._object_store_dbs = {} 29 30 def _CreateCachingFileSystem(self, fs, start_empty=False): 31 def store_type_constructor(namespace, start_empty=False): 32 '''Returns an ObjectStore backed onto test-lifetime-persistent objects 33 in |_object_store_dbs|. 34 ''' 35 if namespace not in self._object_store_dbs: 36 self._object_store_dbs[namespace] = {} 37 db = self._object_store_dbs[namespace] 38 if start_empty: 39 db.clear() 40 return TestObjectStore(namespace, init=db) 41 object_store_creator = ObjectStoreCreator(start_empty=start_empty, 42 store_type=store_type_constructor) 43 return CachingFileSystem(fs, object_store_creator) 44 45 def testReadFiles(self): 46 file_system = self._CreateCachingFileSystem( 47 _CreateLocalFs(), start_empty=False) 48 expected = { 49 './test1.txt': 'test1\n', 50 './test2.txt': 'test2\n', 51 './test3.txt': 'test3\n', 52 } 53 self.assertEqual( 54 expected, 55 file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get()) 56 57 def testListDir(self): 58 file_system = self._CreateCachingFileSystem( 59 _CreateLocalFs(), start_empty=False) 60 expected = ['dir/'] + ['file%d.html' % i for i in range(7)] 61 file_system._read_object_store.Set( 62 'list/', 63 (expected, file_system.Stat('list/').version)) 64 self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get())) 65 66 expected.remove('file0.html') 67 file_system._read_object_store.Set( 68 'list/', 69 (expected, file_system.Stat('list/').version)) 70 self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get())) 71 72 def testCaching(self): 73 test_fs = TestFileSystem({ 74 'bob': { 75 'bob0': 'bob/bob0 contents', 76 'bob1': 'bob/bob1 contents', 77 'bob2': 'bob/bob2 contents', 78 'bob3': 'bob/bob3 contents', 79 } 80 }) 81 mock_fs = MockFileSystem(test_fs) 82 def create_empty_caching_fs(): 83 return self._CreateCachingFileSystem(mock_fs, start_empty=True) 84 85 file_system = create_empty_caching_fs() 86 87 # The stat/read should happen before resolving the Future, and resolving 88 # the future shouldn't do any additional work. 89 get_future = file_system.ReadSingle('bob/bob0') 90 self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 91 self.assertEqual('bob/bob0 contents', get_future.Get()) 92 self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1)) 93 94 # Resource has been cached, so test resource is not re-fetched. 95 self.assertEqual('bob/bob0 contents', 96 file_system.ReadSingle('bob/bob0').Get()) 97 self.assertTrue(*mock_fs.CheckAndReset()) 98 99 # Test if the Stat version is the same the resource is not re-fetched. 100 file_system = create_empty_caching_fs() 101 self.assertEqual('bob/bob0 contents', 102 file_system.ReadSingle('bob/bob0').Get()) 103 self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 104 105 # Test if there is a newer version, the resource is re-fetched. 106 file_system = create_empty_caching_fs() 107 test_fs.IncrementStat(); 108 future = file_system.ReadSingle('bob/bob0') 109 self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1)) 110 self.assertEqual('bob/bob0 contents', future.Get()) 111 self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1)) 112 113 # Test directory and subdirectory stats are cached. 114 file_system = create_empty_caching_fs() 115 file_system._stat_object_store.Del('bob/bob0') 116 file_system._read_object_store.Del('bob/bob0') 117 file_system._stat_object_store.Del('bob/bob1') 118 test_fs.IncrementStat(); 119 futures = (file_system.ReadSingle('bob/bob1'), 120 file_system.ReadSingle('bob/bob0')) 121 self.assertTrue(*mock_fs.CheckAndReset(read_count=2)) 122 self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'), 123 tuple(future.Get() for future in futures)) 124 self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2, stat_count=1)) 125 self.assertEqual('bob/bob1 contents', 126 file_system.ReadSingle('bob/bob1').Get()) 127 self.assertTrue(*mock_fs.CheckAndReset()) 128 129 # Test a more recent parent directory doesn't force a refetch of children. 130 file_system = create_empty_caching_fs() 131 file_system._read_object_store.Del('bob/bob0') 132 file_system._read_object_store.Del('bob/bob1') 133 futures = (file_system.ReadSingle('bob/bob1'), 134 file_system.ReadSingle('bob/bob2'), 135 file_system.ReadSingle('bob/bob3')) 136 self.assertTrue(*mock_fs.CheckAndReset(read_count=3)) 137 self.assertEqual( 138 ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'), 139 tuple(future.Get() for future in futures)) 140 self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3, stat_count=1)) 141 142 test_fs.IncrementStat(path='bob/bob0') 143 file_system = create_empty_caching_fs() 144 self.assertEqual('bob/bob1 contents', 145 file_system.ReadSingle('bob/bob1').Get()) 146 self.assertEqual('bob/bob2 contents', 147 file_system.ReadSingle('bob/bob2').Get()) 148 self.assertEqual('bob/bob3 contents', 149 file_system.ReadSingle('bob/bob3').Get()) 150 self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 151 152 file_system = create_empty_caching_fs() 153 file_system._stat_object_store.Del('bob/bob0') 154 future = file_system.ReadSingle('bob/bob0') 155 self.assertTrue(*mock_fs.CheckAndReset(read_count=1)) 156 self.assertEqual('bob/bob0 contents', future.Get()) 157 self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1)) 158 self.assertEqual('bob/bob0 contents', 159 file_system.ReadSingle('bob/bob0').Get()) 160 self.assertTrue(*mock_fs.CheckAndReset()) 161 162 def testCachedStat(self): 163 test_fs = TestFileSystem({ 164 'bob': { 165 'bob0': 'bob/bob0 contents', 166 'bob1': 'bob/bob1 contents' 167 } 168 }) 169 mock_fs = MockFileSystem(test_fs) 170 171 file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False) 172 173 self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 174 self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 175 self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 176 self.assertTrue(*mock_fs.CheckAndReset()) 177 178 # Caching happens on a directory basis, so reading other files from that 179 # directory won't result in a stat. 180 self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1')) 181 self.assertEqual( 182 StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}), 183 file_system.Stat('bob/')) 184 self.assertTrue(*mock_fs.CheckAndReset()) 185 186 # Even though the stat is bumped, the object store still has it cached so 187 # this won't update. 188 test_fs.IncrementStat() 189 self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0')) 190 self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1')) 191 self.assertEqual( 192 StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}), 193 file_system.Stat('bob/')) 194 self.assertTrue(*mock_fs.CheckAndReset()) 195 196 def testFreshStat(self): 197 test_fs = TestFileSystem({ 198 'bob': { 199 'bob0': 'bob/bob0 contents', 200 'bob1': 'bob/bob1 contents' 201 } 202 }) 203 mock_fs = MockFileSystem(test_fs) 204 205 def run_expecting_stat(stat): 206 def run(): 207 file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True) 208 self.assertEqual( 209 StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}), 210 file_system.Stat('bob/')) 211 self.assertTrue(*mock_fs.CheckAndReset(stat_count=1)) 212 self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0')) 213 self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0')) 214 self.assertTrue(*mock_fs.CheckAndReset()) 215 run() 216 run() 217 218 run_expecting_stat('0') 219 test_fs.IncrementStat() 220 run_expecting_stat('1') 221 222 def testSkipNotFound(self): 223 caching_fs = self._CreateCachingFileSystem(TestFileSystem({ 224 'bob': { 225 'bob0': 'bob/bob0 contents', 226 'bob1': 'bob/bob1 contents' 227 } 228 })) 229 def read_skip_not_found(paths): 230 return caching_fs.Read(paths, skip_not_found=True).Get() 231 self.assertEqual({}, read_skip_not_found(('grub',))) 232 self.assertEqual({}, read_skip_not_found(('bob/bob2',))) 233 self.assertEqual({ 234 'bob/bob0': 'bob/bob0 contents', 235 }, read_skip_not_found(('bob/bob0', 'bob/bob2'))) 236 237 238 if __name__ == '__main__': 239 unittest.main() 240
