1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import os 6 import shutil 7 import sys 8 import tempfile 9 import unittest 10 11 import mock 12 from pyfakefs import fake_filesystem_unittest 13 14 import py_utils 15 from py_utils import cloud_storage 16 from py_utils import lock 17 18 _CLOUD_STORAGE_GLOBAL_LOCK_PATH = os.path.join( 19 os.path.dirname(__file__), 'cloud_storage_global_lock.py') 20 21 def _FakeReadHash(_): 22 return 'hashthis!' 23 24 25 def _FakeCalulateHashMatchesRead(_): 26 return 'hashthis!' 27 28 29 def _FakeCalulateHashNewHash(_): 30 return 'omgnewhash' 31 32 33 class BaseFakeFsUnitTest(fake_filesystem_unittest.TestCase): 34 35 def setUp(self): 36 self.original_environ = os.environ.copy() 37 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '' 38 self.setUpPyfakefs() 39 self.fs.CreateFile( 40 os.path.join(py_utils.GetCatapultDir(), 41 'third_party', 'gsutil', 'gsutil')) 42 43 def CreateFiles(self, file_paths): 44 for f in file_paths: 45 self.fs.CreateFile(f) 46 47 def tearDown(self): 48 self.tearDownPyfakefs() 49 os.environ = self.original_environ 50 51 def _FakeRunCommand(self, cmd): 52 pass 53 54 def _FakeGet(self, bucket, remote_path, local_path): 55 pass 56 57 58 class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest): 59 60 def _AssertRunCommandRaisesError(self, communicate_strs, error): 61 with mock.patch('py_utils.cloud_storage.subprocess.Popen') as popen: 62 p_mock = mock.Mock() 63 popen.return_value = p_mock 64 p_mock.returncode = 1 65 for stderr in communicate_strs: 66 p_mock.communicate.return_value = ('', stderr) 67 self.assertRaises(error, cloud_storage._RunCommand, []) 68 69 def testRunCommandCredentialsError(self): 70 strs = ['You are attempting to access protected data with no configured', 71 'Failure: No handler was ready to authenticate.'] 72 self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError) 73 74 def testRunCommandPermissionError(self): 75 strs = ['status=403', 'status 403', '403 Forbidden'] 76 self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError) 77 78 def testRunCommandNotFoundError(self): 79 strs = ['InvalidUriError', 'No such object', 'No URLs matched', 80 'One or more URLs matched no', 'InvalidUriError'] 81 self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError) 82 83 def testRunCommandServerError(self): 84 strs = ['500 Internal Server Error'] 85 self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError) 86 87 def testRunCommandGenericError(self): 88 strs = ['Random string'] 89 self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError) 90 91 def testInsertCreatesValidCloudUrl(self): 92 orig_run_command = cloud_storage._RunCommand 93 try: 94 cloud_storage._RunCommand = self._FakeRunCommand 95 remote_path = 'test-remote-path.html' 96 local_path = 'test-local-path.html' 97 cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET, 98 remote_path, local_path) 99 self.assertEqual('https://console.developers.google.com/m/cloudstorage' 100 '/b/chromium-telemetry/o/test-remote-path.html', 101 cloud_url) 102 finally: 103 cloud_storage._RunCommand = orig_run_command 104 105 @mock.patch('py_utils.cloud_storage.subprocess') 106 def testExistsReturnsFalse(self, subprocess_mock): 107 p_mock = mock.Mock() 108 subprocess_mock.Popen.return_value = p_mock 109 p_mock.communicate.return_value = ( 110 '', 111 'CommandException: One or more URLs matched no objects.\n') 112 p_mock.returncode_result = 1 113 self.assertFalse(cloud_storage.Exists('fake bucket', 114 'fake remote path')) 115 116 @unittest.skipIf(sys.platform.startswith('win'), 117 'https://github.com/catapult-project/catapult/issues/1861') 118 def testGetFilesInDirectoryIfChanged(self): 119 self.CreateFiles([ 120 'real_dir_path/dir1/1file1.sha1', 121 'real_dir_path/dir1/1file2.txt', 122 'real_dir_path/dir1/1file3.sha1', 123 'real_dir_path/dir2/2file.txt', 124 'real_dir_path/dir3/3file1.sha1']) 125 126 def IncrementFilesUpdated(*_): 127 IncrementFilesUpdated.files_updated += 1 128 IncrementFilesUpdated.files_updated = 0 129 orig_get_if_changed = cloud_storage.GetIfChanged 130 cloud_storage.GetIfChanged = IncrementFilesUpdated 131 try: 132 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 133 os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET) 134 self.assertEqual(0, IncrementFilesUpdated.files_updated) 135 self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged, 136 'fake_dir_path', cloud_storage.PUBLIC_BUCKET) 137 self.assertEqual(0, IncrementFilesUpdated.files_updated) 138 cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path', 139 cloud_storage.PUBLIC_BUCKET) 140 self.assertEqual(3, IncrementFilesUpdated.files_updated) 141 finally: 142 cloud_storage.GetIfChanged = orig_get_if_changed 143 144 def testCopy(self): 145 orig_run_command = cloud_storage._RunCommand 146 147 def AssertCorrectRunCommandArgs(args): 148 self.assertEqual(expected_args, args) 149 cloud_storage._RunCommand = AssertCorrectRunCommandArgs 150 expected_args = ['cp', 'gs://bucket1/remote_path1', 151 'gs://bucket2/remote_path2'] 152 try: 153 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 154 finally: 155 cloud_storage._RunCommand = orig_run_command 156 157 @mock.patch('py_utils.cloud_storage.subprocess.Popen') 158 def testSwarmingUsesExistingEnv(self, mock_popen): 159 os.environ['SWARMING_HEADLESS'] = '1' 160 161 mock_gsutil = mock_popen() 162 mock_gsutil.communicate = mock.MagicMock(return_value=('a', 'b')) 163 mock_gsutil.returncode = None 164 165 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 166 167 mock_popen.assert_called_with( 168 mock.ANY, stderr=-1, env=os.environ, stdout=-1) 169 170 @mock.patch('py_utils.cloud_storage._FileLock') 171 def testDisableCloudStorageIo(self, unused_lock_mock): 172 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1' 173 dir_path = 'real_dir_path' 174 self.fs.CreateDirectory(dir_path) 175 file_path = os.path.join(dir_path, 'file1') 176 file_path_sha = file_path + '.sha1' 177 178 def CleanTimeStampFile(): 179 os.remove(file_path + '.fetchts') 180 181 self.CreateFiles([file_path, file_path_sha]) 182 with open(file_path_sha, 'w') as f: 183 f.write('hash1234') 184 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 185 cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') 186 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 187 cloud_storage.Get('bucket', 'foo', file_path) 188 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 189 cloud_storage.GetIfChanged(file_path, 'foo') 190 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 191 cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234') 192 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 193 cloud_storage.Insert('bucket', 'foo', file_path) 194 195 CleanTimeStampFile() 196 with self.assertRaises(cloud_storage.CloudStorageIODisabled): 197 cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket') 198 199 200 class GetIfChangedTests(BaseFakeFsUnitTest): 201 202 def setUp(self): 203 super(GetIfChangedTests, self).setUp() 204 self._orig_read_hash = cloud_storage.ReadHash 205 self._orig_calculate_hash = cloud_storage.CalculateHash 206 207 def tearDown(self): 208 super(GetIfChangedTests, self).tearDown() 209 cloud_storage.CalculateHash = self._orig_calculate_hash 210 cloud_storage.ReadHash = self._orig_read_hash 211 212 @mock.patch('py_utils.cloud_storage._FileLock') 213 @mock.patch('py_utils.cloud_storage._GetLocked') 214 def testHashPathDoesNotExists(self, unused_get_locked, unused_lock_mock): 215 cloud_storage.ReadHash = _FakeReadHash 216 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 217 file_path = 'test-file-path.wpr' 218 219 cloud_storage._GetLocked = self._FakeGet 220 # hash_path doesn't exist. 221 self.assertFalse(cloud_storage.GetIfChanged(file_path, 222 cloud_storage.PUBLIC_BUCKET)) 223 224 @mock.patch('py_utils.cloud_storage._FileLock') 225 @mock.patch('py_utils.cloud_storage._GetLocked') 226 def testHashPathExistsButFilePathDoesNot( 227 self, unused_get_locked, unused_lock_mock): 228 cloud_storage.ReadHash = _FakeReadHash 229 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 230 file_path = 'test-file-path.wpr' 231 hash_path = file_path + '.sha1' 232 233 # hash_path exists, but file_path doesn't. 234 self.CreateFiles([hash_path]) 235 self.assertTrue(cloud_storage.GetIfChanged(file_path, 236 cloud_storage.PUBLIC_BUCKET)) 237 238 @mock.patch('py_utils.cloud_storage._FileLock') 239 @mock.patch('py_utils.cloud_storage._GetLocked') 240 def testHashPathAndFileHashExistWithSameHash( 241 self, unused_get_locked, unused_lock_mock): 242 cloud_storage.ReadHash = _FakeReadHash 243 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 244 file_path = 'test-file-path.wpr' 245 246 # hash_path and file_path exist, and have same hash. 247 self.CreateFiles([file_path]) 248 self.assertFalse(cloud_storage.GetIfChanged(file_path, 249 cloud_storage.PUBLIC_BUCKET)) 250 251 @mock.patch('py_utils.cloud_storage._FileLock') 252 @mock.patch('py_utils.cloud_storage._GetLocked') 253 def testHashPathAndFileHashExistWithDifferentHash( 254 self, mock_get_locked, unused_get_locked): 255 cloud_storage.ReadHash = _FakeReadHash 256 cloud_storage.CalculateHash = _FakeCalulateHashNewHash 257 file_path = 'test-file-path.wpr' 258 hash_path = file_path + '.sha1' 259 260 def _FakeGetLocked(bucket, expected_hash, file_path): 261 del bucket, expected_hash, file_path # unused 262 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 263 264 mock_get_locked.side_effect = _FakeGetLocked 265 266 self.CreateFiles([file_path, hash_path]) 267 # hash_path and file_path exist, and have different hashes. 268 self.assertTrue(cloud_storage.GetIfChanged(file_path, 269 cloud_storage.PUBLIC_BUCKET)) 270 271 @mock.patch('py_utils.cloud_storage._FileLock') 272 @mock.patch('py_utils.cloud_storage.CalculateHash') 273 @mock.patch('py_utils.cloud_storage._GetLocked') 274 def testNoHashComputationNeededUponSecondCall( 275 self, mock_get_locked, mock_calculate_hash, unused_get_locked): 276 mock_calculate_hash.side_effect = _FakeCalulateHashNewHash 277 cloud_storage.ReadHash = _FakeReadHash 278 file_path = 'test-file-path.wpr' 279 hash_path = file_path + '.sha1' 280 281 def _FakeGetLocked(bucket, expected_hash, file_path): 282 del bucket, expected_hash, file_path # unused 283 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 284 285 mock_get_locked.side_effect = _FakeGetLocked 286 287 self.CreateFiles([file_path, hash_path]) 288 # hash_path and file_path exist, and have different hashes. This first call 289 # will invoke a fetch. 290 self.assertTrue(cloud_storage.GetIfChanged(file_path, 291 cloud_storage.PUBLIC_BUCKET)) 292 293 # The fetch left a .fetchts file on machine. 294 self.assertTrue(os.path.exists(file_path + '.fetchts')) 295 296 # Subsequent invocations of GetIfChanged should not invoke CalculateHash. 297 mock_calculate_hash.assert_not_called() 298 self.assertFalse(cloud_storage.GetIfChanged(file_path, 299 cloud_storage.PUBLIC_BUCKET)) 300 self.assertFalse(cloud_storage.GetIfChanged(file_path, 301 cloud_storage.PUBLIC_BUCKET)) 302 303 @mock.patch('py_utils.cloud_storage._FileLock') 304 @mock.patch('py_utils.cloud_storage.CalculateHash') 305 @mock.patch('py_utils.cloud_storage._GetLocked') 306 def testRefetchingFileUponHashFileChange( 307 self, mock_get_locked, mock_calculate_hash, unused_get_locked): 308 mock_calculate_hash.side_effect = _FakeCalulateHashNewHash 309 cloud_storage.ReadHash = _FakeReadHash 310 file_path = 'test-file-path.wpr' 311 hash_path = file_path + '.sha1' 312 313 def _FakeGetLocked(bucket, expected_hash, file_path): 314 del bucket, expected_hash, file_path # unused 315 cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead 316 317 mock_get_locked.side_effect = _FakeGetLocked 318 319 self.CreateFiles([file_path, hash_path]) 320 # hash_path and file_path exist, and have different hashes. This first call 321 # will invoke a fetch. 322 self.assertTrue(cloud_storage.GetIfChanged(file_path, 323 cloud_storage.PUBLIC_BUCKET)) 324 325 # The fetch left a .fetchts file on machine. 326 self.assertTrue(os.path.exists(file_path + '.fetchts')) 327 328 with open(file_path + '.fetchts') as f: 329 fetchts = float(f.read()) 330 331 # Updating the .sha1 hash_path file with the new hash after .fetchts 332 # is created. 333 file_obj = self.fs.GetObject(hash_path) 334 file_obj.SetMTime(fetchts + 100) 335 336 cloud_storage.ReadHash = lambda _: 'hashNeW' 337 def _FakeGetLockedNewHash(bucket, expected_hash, file_path): 338 del bucket, expected_hash, file_path # unused 339 cloud_storage.CalculateHash = lambda _: 'hashNeW' 340 341 mock_get_locked.side_effect = _FakeGetLockedNewHash 342 343 # hash_path and file_path exist, and have different hashes. This first call 344 # will invoke a fetch. 345 self.assertTrue(cloud_storage.GetIfChanged(file_path, 346 cloud_storage.PUBLIC_BUCKET)) 347 348 349 class CloudStorageRealFsUnitTest(unittest.TestCase): 350 351 def setUp(self): 352 self.original_environ = os.environ.copy() 353 os.environ['DISABLE_CLOUD_STORAGE_IO'] = '' 354 355 def tearDown(self): 356 os.environ = self.original_environ 357 358 @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005) 359 def testGetPseudoLockUnavailableCausesTimeout(self): 360 with tempfile.NamedTemporaryFile(suffix='.pseudo_lock') as pseudo_lock_fd: 361 with lock.FileLock(pseudo_lock_fd, lock.LOCK_EX | lock.LOCK_NB): 362 with self.assertRaises(py_utils.TimeoutException): 363 file_path = pseudo_lock_fd.name.replace('.pseudo_lock', '') 364 cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET) 365 366 @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005) 367 def testGetGlobalLockUnavailableCausesTimeout(self): 368 with open(_CLOUD_STORAGE_GLOBAL_LOCK_PATH) as global_lock_fd: 369 with lock.FileLock(global_lock_fd, lock.LOCK_EX | lock.LOCK_NB): 370 tmp_dir = tempfile.mkdtemp() 371 try: 372 file_path = os.path.join(tmp_dir, 'foo') 373 with self.assertRaises(py_utils.TimeoutException): 374 cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET) 375 finally: 376 shutil.rmtree(tmp_dir) 377 378 379 class CloudStorageErrorHandlingTest(unittest.TestCase): 380 def runTest(self): 381 self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr( 382 'ServiceException: 401 Anonymous users does not have ' 383 'storage.objects.get access to object chrome-partner-telemetry'), 384 cloud_storage.CredentialsError) 385 self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr( 386 '403 Caller does not have storage.objects.list access to bucket ' 387 'chrome-telemetry'), cloud_storage.PermissionError) 388