1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import os 6 import shutil 7 import sys 8 import tempfile 9 import unittest 10 11 import mock 12 from pyfakefs import fake_filesystem_unittest 13 14 import py_utils 15 from py_utils import cloud_storage 16 from py_utils import lock 17 18 _CLOUD_STORAGE_GLOBAL_LOCK_PATH = os.path.join( 19 os.path.dirname(__file__), 'cloud_storage_global_lock.py') 20 21 def _FakeReadHash(_): 22 return 'hashthis!' 23 24 25 def _FakeCalulateHashMatchesRead(_): 26 return 'hashthis!' 27 28 29 def _FakeCalulateHashNewHash(_): 30 return 'omgnewhash' 31 32 33 class BaseFakeFsUnitTest(fake_filesystem_unittest.TestCase): 34 35 def setUp(self): 36 self.original_environ = os.environ.copy() 37 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '' 38 self.setUpPyfakefs() 39 self.fs.CreateFile( 40 os.path.join(py_utils.GetCatapultDir(), 41 'third_party', 'gsutil', 'gsutil')) 42 43 def CreateFiles(self, file_paths): 44 for f in file_paths: 45 self.fs.CreateFile(f) 46 47 def tearDown(self): 48 self.tearDownPyfakefs() 49 os.environ = self.original_environ 50 51 def _FakeRunCommand(self, cmd): 52 pass 53 54 def _FakeGet(self, bucket, remote_path, local_path): 55 pass 56 57 58 class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest): 59 60 def _AssertRunCommandRaisesError(self, communicate_strs, error): 61 with mock.patch('py_utils.cloud_storage.subprocess.Popen') as popen: 62 p_mock = mock.Mock() 63 popen.return_value = p_mock 64 p_mock.returncode = 1 65 for stderr in communicate_strs: 66 p_mock.communicate.return_value = ('', stderr) 67 self.assertRaises(error, cloud_storage._RunCommand, []) 68 69 def testRunCommandCredentialsError(self): 70 strs = ['You are attempting to access protected data with no configured', 71 'Failure: No handler was ready to authenticate.'] 72 self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError) 73 74 def testRunCommandPermissionError(self): 75 strs = ['status=403', 'status 403', '403 Forbidden'] 76 self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError) 77 78 def testRunCommandNotFoundError(self): 79 strs = ['InvalidUriError', 'No such object', 'No URLs matched', 80 'One or more URLs matched no', 'InvalidUriError'] 81 self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError) 82 83 def testRunCommandServerError(self): 84 strs = ['500 Internal Server Error'] 85 self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError) 86 87 def testRunCommandGenericError(self): 88 strs = ['Random string'] 89 self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError) 90 91 def testInsertCreatesValidCloudUrl(self): 92 orig_run_command = cloud_storage._RunCommand 93 try: 94 cloud_storage._RunCommand = self._FakeRunCommand 95 remote_path = 'test-remote-path.html' 96 local_path = 'test-local-path.html' 97 cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET, 98 remote_path, local_path) 99 self.assertEqual('https://console.developers.google.com/m/cloudstorage' 100 '/b/chromium-telemetry/o/test-remote-path.html', 101 cloud_url) 102 finally: 103 cloud_storage._RunCommand = orig_run_command 104 105 @mock.patch('py_utils.cloud_storage.subprocess') 106 def testExistsReturnsFalse(self, subprocess_mock): 107 p_mock = mock.Mock() 108 subprocess_mock.Popen.return_value = p_mock 109 p_mock.communicate.return_value = ( 110 '', 111 'CommandException: One or more URLs matched no objects.\n') 112 p_mock.returncode_result = 1 113 self.assertFalse(cloud_storage.Exists('fake bucket', 114 'fake remote path')) 115 116 @unittest.skipIf(sys.platform.startswith('win'), 117 'https://github.com/catapult-project/catapult/issues/1861') 118 def testGetFilesInDirectoryIfChanged(self): 119 self.CreateFiles([ 120 'real_dir_path/dir1/1file1.sha1', 121 'real_dir_path/dir1/1file2.txt', 122 'real_dir_path/dir1/1file3.sha1', 123 'real_dir_path/dir2/2file.txt', 124 'real_dir_path/dir3/3file1.sha1']) 125 126 def IncrementFilesUpdated(*_): 127 IncrementFilesUpdated.files_updated += 1 128 IncrementFilesUpdated.files_updated = 0 129 orig_get_if_changed = cloud_storage.GetIfChanged 130 cloud_storage.GetIfChanged = IncrementFilesUpdated 131 try: 132 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 133 os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET) 134 self.assertEqual(0, IncrementFilesUpdated.files_updated) 135 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 136 'fake_dir_path', cloud_storage.PUBLIC_BUCKET) 137 self.assertEqual(0, IncrementFilesUpdated.files_updated) 138 cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path', 139 cloud_storage.PUBLIC_BUCKET) 140 self.assertEqual(3, IncrementFilesUpdated.files_updated) 141 finally: 142 cloud_storage.GetIfChanged = orig_get_if_changed 143 144 def testCopy(self): 145 orig_run_command = cloud_storage._RunCommand 146 147 def AssertCorrectRunCommandArgs(args): 148 self.assertEqual(expected_args, args) 149 cloud_storage._RunCommand = AssertCorrectRunCommandArgs 150 expected_args = ['cp', 'gs://bucket1/remote_path1', 151 'gs://bucket2/remote_path2'] 152 try: 153 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 154 finally: 155 cloud_storage._RunCommand = orig_run_command 156 157 @mock.patch('py_utils.cloud_storage._FileLock') 158 def testDisableCloudStorageIo(self, unused_lock_mock): 159 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1' 160 dir_path = 'real_dir_path' 161 self.fs.CreateDirectory(dir_path) 162 file_path = os.path.join(dir_path, 'file1') 163 file_path_sha = file_path + '.sha1' 164 165 def CleanTimeStampFile(): 166 os.remove(file_path + '.fetchts') 167 168 self.CreateFiles([file_path, file_path_sha]) 169 with open(file_path_sha, 'w') as f: 170 f.write('hash1234') 171 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 172 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 173 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 174 cloud_storage.Get('bucket', 'foo', file_path) 175 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 176 cloud_storage.GetIfChanged(file_path, 'foo') 177 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 178 cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234') 179 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 180 cloud_storage.Insert('bucket', 'foo', file_path) 181 182 CleanTimeStampFile() 183 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 184 cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket') 185 186 187 class GetIfChangedTests(BaseFakeFsUnitTest): 188 189 def setUp(self): 190 super(GetIfChangedTests, self).setUp() 191 self._orig_read_hash = cloud_storage.ReadHash 192 self._orig_calculate_hash = cloud_storage.CalculateHash 193 194 def tearDown(self): 195 super(GetIfChangedTests, self).tearDown() 196 cloud_storage.CalculateHash = self._orig_calculate_hash 197 cloud_storage.ReadHash = self._orig_read_hash 198 199 @mock.patch('py_utils.cloud_storage._FileLock') 200 @mock.patch('py_utils.cloud_storage._GetLocked') 201 def testHashPathDoesNotExists(self, unused_get_locked, unused_lock_mock): 202 cloud_storage.ReadHash = _FakeReadHash 203 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 204 file_path = 'test-file-path.wpr' 205 206 cloud_storage._GetLocked = self._FakeGet 207 # hash_path doesn't exist. 208 self.assertFalse(cloud_storage.GetIfChanged(file_path, 209 cloud_storage.PUBLIC_BUCKET)) 210 211 @mock.patch('py_utils.cloud_storage._FileLock') 212 @mock.patch('py_utils.cloud_storage._GetLocked') 213 def testHashPathExistsButFilePathDoesNot( 214 self, unused_get_locked, unused_lock_mock): 215 cloud_storage.ReadHash = _FakeReadHash 216 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 217 file_path = 'test-file-path.wpr' 218 hash_path = file_path + '.sha1' 219 220 # hash_path exists, but file_path doesn't. 221 self.CreateFiles([hash_path]) 222 self.assertTrue(cloud_storage.GetIfChanged(file_path, 223 cloud_storage.PUBLIC_BUCKET)) 224 225 @mock.patch('py_utils.cloud_storage._FileLock') 226 @mock.patch('py_utils.cloud_storage._GetLocked') 227 def testHashPathAndFileHashExistWithSameHash( 228 self, unused_get_locked, unused_lock_mock): 229 cloud_storage.ReadHash = _FakeReadHash 230 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 231 file_path = 'test-file-path.wpr' 232 233 # hash_path and file_path exist, and have same hash. 234 self.CreateFiles([file_path]) 235 self.assertFalse(cloud_storage.GetIfChanged(file_path, 236 cloud_storage.PUBLIC_BUCKET)) 237 238 @mock.patch('py_utils.cloud_storage._FileLock') 239 @mock.patch('py_utils.cloud_storage._GetLocked') 240 def testHashPathAndFileHashExistWithDifferentHash( 241 self, mock_get_locked, unused_get_locked): 242 cloud_storage.ReadHash = _FakeReadHash 243 cloud_storage.CalculateHash = _FakeCalulateHashNewHash 244 file_path = 'test-file-path.wpr' 245 hash_path = file_path + '.sha1' 246 247 def _FakeGetLocked(bucket, expected_hash, file_path): 248 del bucket, expected_hash, file_path # unused 249 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 250 251 mock_get_locked.side_effect = _FakeGetLocked 252 253 self.CreateFiles([file_path, hash_path]) 254 # hash_path and file_path exist, and have different hashes. 255 self.assertTrue(cloud_storage.GetIfChanged(file_path, 256 cloud_storage.PUBLIC_BUCKET)) 257 258 @mock.patch('py_utils.cloud_storage._FileLock') 259 @mock.patch('py_utils.cloud_storage.CalculateHash') 260 @mock.patch('py_utils.cloud_storage._GetLocked') 261 def testNoHashComputationNeededUponSecondCall( 262 self, mock_get_locked, mock_calculate_hash, unused_get_locked): 263 mock_calculate_hash.side_effect = _FakeCalulateHashNewHash 264 cloud_storage.ReadHash = _FakeReadHash 265 file_path = 'test-file-path.wpr' 266 hash_path = file_path + '.sha1' 267 268 def _FakeGetLocked(bucket, expected_hash, file_path): 269 del bucket, expected_hash, file_path # unused 270 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 271 272 mock_get_locked.side_effect = _FakeGetLocked 273 274 self.CreateFiles([file_path, hash_path]) 275 # hash_path and file_path exist, and have different hashes. This first call 276 # will invoke a fetch. 277 self.assertTrue(cloud_storage.GetIfChanged(file_path, 278 cloud_storage.PUBLIC_BUCKET)) 279 280 # The fetch left a .fetchts file on machine. 281 self.assertTrue(os.path.exists(file_path + '.fetchts')) 282 283 # Subsequent invocations of GetIfChanged should not invoke CalculateHash. 284 mock_calculate_hash.assert_not_called() 285 self.assertFalse(cloud_storage.GetIfChanged(file_path, 286 cloud_storage.PUBLIC_BUCKET)) 287 self.assertFalse(cloud_storage.GetIfChanged(file_path, 288 cloud_storage.PUBLIC_BUCKET)) 289 290 @mock.patch('py_utils.cloud_storage._FileLock') 291 @mock.patch('py_utils.cloud_storage.CalculateHash') 292 @mock.patch('py_utils.cloud_storage._GetLocked') 293 def testRefetchingFileUponHashFileChange( 294 self, mock_get_locked, mock_calculate_hash, unused_get_locked): 295 mock_calculate_hash.side_effect = _FakeCalulateHashNewHash 296 cloud_storage.ReadHash = _FakeReadHash 297 file_path = 'test-file-path.wpr' 298 hash_path = file_path + '.sha1' 299 300 def _FakeGetLocked(bucket, expected_hash, file_path): 301 del bucket, expected_hash, file_path # unused 302 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 303 304 mock_get_locked.side_effect = _FakeGetLocked 305 306 self.CreateFiles([file_path, hash_path]) 307 # hash_path and file_path exist, and have different hashes. This first call 308 # will invoke a fetch. 309 self.assertTrue(cloud_storage.GetIfChanged(file_path, 310 cloud_storage.PUBLIC_BUCKET)) 311 312 # The fetch left a .fetchts file on machine. 313 self.assertTrue(os.path.exists(file_path + '.fetchts')) 314 315 with open(file_path + '.fetchts') as f: 316 fetchts = float(f.read()) 317 318 # Updating the .sha1 hash_path file with the new hash after .fetchts 319 # is created. 320 file_obj = self.fs.GetObject(hash_path) 321 file_obj.SetMTime(fetchts + 100) 322 323 cloud_storage.ReadHash = lambda _: 'hashNeW' 324 def _FakeGetLockedNewHash(bucket, expected_hash, file_path): 325 del bucket, expected_hash, file_path # unused 326 cloud_storage.CalculateHash = lambda _: 'hashNeW' 327 328 mock_get_locked.side_effect = _FakeGetLockedNewHash 329 330 # hash_path and file_path exist, and have different hashes. This first call 331 # will invoke a fetch. 332 self.assertTrue(cloud_storage.GetIfChanged(file_path, 333 cloud_storage.PUBLIC_BUCKET)) 334 335 336 class CloudStorageRealFsUnitTest(unittest.TestCase): 337 338 def setUp(self): 339 self.original_environ = os.environ.copy() 340 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '' 341 342 def tearDown(self): 343 os.environ = self.original_environ 344 345 @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005) 346 def testGetPseudoLockUnavailableCausesTimeout(self): 347 with tempfile.NamedTemporaryFile(suffix='.pseudo_lock') as pseudo_lock_fd: 348 with lock.FileLock(pseudo_lock_fd, lock.LOCK_EX | lock.LOCK_NB): 349 with self.assertRaises(py_utils.TimeoutException): 350 file_path = pseudo_lock_fd.name.replace('.pseudo_lock', '') 351 cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET) 352 353 @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005) 354 def testGetGlobalLockUnavailableCausesTimeout(self): 355 with open(_CLOUD_STORAGE_GLOBAL_LOCK_PATH) as global_lock_fd: 356 with lock.FileLock(global_lock_fd, lock.LOCK_EX | lock.LOCK_NB): 357 tmp_dir = tempfile.mkdtemp() 358 try: 359 file_path = os.path.join(tmp_dir, 'foo') 360 with self.assertRaises(py_utils.TimeoutException): 361 cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET) 362 finally: 363 shutil.rmtree(tmp_dir) 364 365 366 class CloudStorageErrorHandlingTest(unittest.TestCase): 367 def runTest(self): 368 self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr( 369 'ServiceException: 401 Anonymous users does not have ' 370 'storage.objects.get access to object chrome-partner-telemetry'), 371 cloud_storage.CredentialsError) 372 self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr( 373 '403 Caller does not have storage.objects.list access to bucket ' 374 'chrome-telemetry'), cloud_storage.PermissionError) 375