Home | History | Annotate | Download | only in py_utils 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import os
      6 import shutil
      7 import sys
      8 import tempfile
      9 import unittest
     10 
     11 import mock
     12 from pyfakefs import fake_filesystem_unittest
     13 
     14 import py_utils
     15 from py_utils import cloud_storage
     16 from py_utils import lock
     17 
     18 _CLOUD_STORAGE_GLOBAL_LOCK_PATH = os.path.join(
     19     os.path.dirname(__file__), 'cloud_storage_global_lock.py')
     20 
     21 def _FakeReadHash(_):
     22   return 'hashthis!'
     23 
     24 
     25 def _FakeCalulateHashMatchesRead(_):
     26   return 'hashthis!'
     27 
     28 
     29 def _FakeCalulateHashNewHash(_):
     30   return 'omgnewhash'
     31 
     32 
     33 class BaseFakeFsUnitTest(fake_filesystem_unittest.TestCase):
     34 
     35   def setUp(self):
     36     self.original_environ = os.environ.copy()
     37     os.environ['DISABLE_CLOUD_STORAGE_IO'] = ''
     38     self.setUpPyfakefs()
     39     self.fs.CreateFile(
     40         os.path.join(py_utils.GetCatapultDir(),
     41                      'third_party', 'gsutil', 'gsutil'))
     42 
     43   def CreateFiles(self, file_paths):
     44     for f in file_paths:
     45       self.fs.CreateFile(f)
     46 
     47   def tearDown(self):
     48     self.tearDownPyfakefs()
     49     os.environ = self.original_environ
     50 
     51   def _FakeRunCommand(self, cmd):
     52     pass
     53 
     54   def _FakeGet(self, bucket, remote_path, local_path):
     55     pass
     56 
     57 
     58 class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest):
     59 
     60   def _AssertRunCommandRaisesError(self, communicate_strs, error):
     61     with mock.patch('py_utils.cloud_storage.subprocess.Popen') as popen:
     62       p_mock = mock.Mock()
     63       popen.return_value = p_mock
     64       p_mock.returncode = 1
     65       for stderr in communicate_strs:
     66         p_mock.communicate.return_value = ('', stderr)
     67         self.assertRaises(error, cloud_storage._RunCommand, [])
     68 
     69   def testRunCommandCredentialsError(self):
     70     strs = ['You are attempting to access protected data with no configured',
     71             'Failure: No handler was ready to authenticate.']
     72     self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError)
     73 
     74   def testRunCommandPermissionError(self):
     75     strs = ['status=403', 'status 403', '403 Forbidden']
     76     self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError)
     77 
     78   def testRunCommandNotFoundError(self):
     79     strs = ['InvalidUriError', 'No such object', 'No URLs matched',
     80             'One or more URLs matched no', 'InvalidUriError']
     81     self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError)
     82 
     83   def testRunCommandServerError(self):
     84     strs = ['500 Internal Server Error']
     85     self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError)
     86 
     87   def testRunCommandGenericError(self):
     88     strs = ['Random string']
     89     self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError)
     90 
     91   def testInsertCreatesValidCloudUrl(self):
     92     orig_run_command = cloud_storage._RunCommand
     93     try:
     94       cloud_storage._RunCommand = self._FakeRunCommand
     95       remote_path = 'test-remote-path.html'
     96       local_path = 'test-local-path.html'
     97       cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET,
     98                                        remote_path, local_path)
     99       self.assertEqual('https://console.developers.google.com/m/cloudstorage'
    100                        '/b/chromium-telemetry/o/test-remote-path.html',
    101                        cloud_url)
    102     finally:
    103       cloud_storage._RunCommand = orig_run_command
    104 
    105   @mock.patch('py_utils.cloud_storage.subprocess')
    106   def testExistsReturnsFalse(self, subprocess_mock):
    107     p_mock = mock.Mock()
    108     subprocess_mock.Popen.return_value = p_mock
    109     p_mock.communicate.return_value = (
    110         '',
    111         'CommandException: One or more URLs matched no objects.\n')
    112     p_mock.returncode_result = 1
    113     self.assertFalse(cloud_storage.Exists('fake bucket',
    114                                           'fake remote path'))
    115 
    116   @unittest.skipIf(sys.platform.startswith('win'),
    117                    'https://github.com/catapult-project/catapult/issues/1861')
    118   def testGetFilesInDirectoryIfChanged(self):
    119     self.CreateFiles([
    120         'real_dir_path/dir1/1file1.sha1',
    121         'real_dir_path/dir1/1file2.txt',
    122         'real_dir_path/dir1/1file3.sha1',
    123         'real_dir_path/dir2/2file.txt',
    124         'real_dir_path/dir3/3file1.sha1'])
    125 
    126     def IncrementFilesUpdated(*_):
    127       IncrementFilesUpdated.files_updated += 1
    128     IncrementFilesUpdated.files_updated = 0
    129     orig_get_if_changed = cloud_storage.GetIfChanged
    130     cloud_storage.GetIfChanged = IncrementFilesUpdated
    131     try:
    132       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
    133                         os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET)
    134       self.assertEqual(0, IncrementFilesUpdated.files_updated)
    135       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
    136                         'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
    137       self.assertEqual(0, IncrementFilesUpdated.files_updated)
    138       cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path',
    139                                                  cloud_storage.PUBLIC_BUCKET)
    140       self.assertEqual(3, IncrementFilesUpdated.files_updated)
    141     finally:
    142       cloud_storage.GetIfChanged = orig_get_if_changed
    143 
    144   def testCopy(self):
    145     orig_run_command = cloud_storage._RunCommand
    146 
    147     def AssertCorrectRunCommandArgs(args):
    148       self.assertEqual(expected_args, args)
    149     cloud_storage._RunCommand = AssertCorrectRunCommandArgs
    150     expected_args = ['cp', 'gs://bucket1/remote_path1',
    151                      'gs://bucket2/remote_path2']
    152     try:
    153       cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
    154     finally:
    155       cloud_storage._RunCommand = orig_run_command
    156 
    157   @mock.patch('py_utils.cloud_storage.subprocess.Popen')
    158   def testSwarmingUsesExistingEnv(self, mock_popen):
    159     os.environ['SWARMING_HEADLESS'] = '1'
    160 
    161     mock_gsutil = mock_popen()
    162     mock_gsutil.communicate = mock.MagicMock(return_value=('a', 'b'))
    163     mock_gsutil.returncode = None
    164 
    165     cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
    166 
    167     mock_popen.assert_called_with(
    168         mock.ANY, stderr=-1, env=os.environ, stdout=-1)
    169 
    170   @mock.patch('py_utils.cloud_storage._FileLock')
    171   def testDisableCloudStorageIo(self, unused_lock_mock):
    172     os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
    173     dir_path = 'real_dir_path'
    174     self.fs.CreateDirectory(dir_path)
    175     file_path = os.path.join(dir_path, 'file1')
    176     file_path_sha = file_path + '.sha1'
    177 
    178     def CleanTimeStampFile():
    179       os.remove(file_path + '.fetchts')
    180 
    181     self.CreateFiles([file_path, file_path_sha])
    182     with open(file_path_sha, 'w') as f:
    183       f.write('hash1234')
    184     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    185       cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
    186     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    187       cloud_storage.Get('bucket', 'foo', file_path)
    188     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    189       cloud_storage.GetIfChanged(file_path, 'foo')
    190     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    191       cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234')
    192     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    193       cloud_storage.Insert('bucket', 'foo', file_path)
    194 
    195     CleanTimeStampFile()
    196     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    197       cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
    198 
    199 
    200 class GetIfChangedTests(BaseFakeFsUnitTest):
    201 
    202   def setUp(self):
    203     super(GetIfChangedTests, self).setUp()
    204     self._orig_read_hash = cloud_storage.ReadHash
    205     self._orig_calculate_hash = cloud_storage.CalculateHash
    206 
    207   def tearDown(self):
    208     super(GetIfChangedTests, self).tearDown()
    209     cloud_storage.CalculateHash = self._orig_calculate_hash
    210     cloud_storage.ReadHash = self._orig_read_hash
    211 
    212   @mock.patch('py_utils.cloud_storage._FileLock')
    213   @mock.patch('py_utils.cloud_storage._GetLocked')
    214   def testHashPathDoesNotExists(self, unused_get_locked, unused_lock_mock):
    215     cloud_storage.ReadHash = _FakeReadHash
    216     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    217     file_path = 'test-file-path.wpr'
    218 
    219     cloud_storage._GetLocked = self._FakeGet
    220     # hash_path doesn't exist.
    221     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    222                                                 cloud_storage.PUBLIC_BUCKET))
    223 
    224   @mock.patch('py_utils.cloud_storage._FileLock')
    225   @mock.patch('py_utils.cloud_storage._GetLocked')
    226   def testHashPathExistsButFilePathDoesNot(
    227       self, unused_get_locked, unused_lock_mock):
    228     cloud_storage.ReadHash = _FakeReadHash
    229     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    230     file_path = 'test-file-path.wpr'
    231     hash_path = file_path + '.sha1'
    232 
    233     # hash_path exists, but file_path doesn't.
    234     self.CreateFiles([hash_path])
    235     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    236                                                cloud_storage.PUBLIC_BUCKET))
    237 
    238   @mock.patch('py_utils.cloud_storage._FileLock')
    239   @mock.patch('py_utils.cloud_storage._GetLocked')
    240   def testHashPathAndFileHashExistWithSameHash(
    241       self, unused_get_locked, unused_lock_mock):
    242     cloud_storage.ReadHash = _FakeReadHash
    243     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    244     file_path = 'test-file-path.wpr'
    245 
    246     # hash_path and file_path exist, and have same hash.
    247     self.CreateFiles([file_path])
    248     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    249                                                 cloud_storage.PUBLIC_BUCKET))
    250 
    251   @mock.patch('py_utils.cloud_storage._FileLock')
    252   @mock.patch('py_utils.cloud_storage._GetLocked')
    253   def testHashPathAndFileHashExistWithDifferentHash(
    254       self, mock_get_locked, unused_get_locked):
    255     cloud_storage.ReadHash = _FakeReadHash
    256     cloud_storage.CalculateHash = _FakeCalulateHashNewHash
    257     file_path = 'test-file-path.wpr'
    258     hash_path = file_path + '.sha1'
    259 
    260     def _FakeGetLocked(bucket, expected_hash, file_path):
    261       del bucket, expected_hash, file_path  # unused
    262       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    263 
    264     mock_get_locked.side_effect = _FakeGetLocked
    265 
    266     self.CreateFiles([file_path, hash_path])
    267     # hash_path and file_path exist, and have different hashes.
    268     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    269                                                cloud_storage.PUBLIC_BUCKET))
    270 
    271   @mock.patch('py_utils.cloud_storage._FileLock')
    272   @mock.patch('py_utils.cloud_storage.CalculateHash')
    273   @mock.patch('py_utils.cloud_storage._GetLocked')
    274   def testNoHashComputationNeededUponSecondCall(
    275       self, mock_get_locked, mock_calculate_hash, unused_get_locked):
    276     mock_calculate_hash.side_effect = _FakeCalulateHashNewHash
    277     cloud_storage.ReadHash = _FakeReadHash
    278     file_path = 'test-file-path.wpr'
    279     hash_path = file_path + '.sha1'
    280 
    281     def _FakeGetLocked(bucket, expected_hash, file_path):
    282       del bucket, expected_hash, file_path  # unused
    283       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    284 
    285     mock_get_locked.side_effect = _FakeGetLocked
    286 
    287     self.CreateFiles([file_path, hash_path])
    288     # hash_path and file_path exist, and have different hashes. This first call
    289     # will invoke a fetch.
    290     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    291                                                cloud_storage.PUBLIC_BUCKET))
    292 
    293     # The fetch left a .fetchts file on machine.
    294     self.assertTrue(os.path.exists(file_path + '.fetchts'))
    295 
    296     # Subsequent invocations of GetIfChanged should not invoke CalculateHash.
    297     mock_calculate_hash.assert_not_called()
    298     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    299                                                 cloud_storage.PUBLIC_BUCKET))
    300     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    301                                                 cloud_storage.PUBLIC_BUCKET))
    302 
    303   @mock.patch('py_utils.cloud_storage._FileLock')
    304   @mock.patch('py_utils.cloud_storage.CalculateHash')
    305   @mock.patch('py_utils.cloud_storage._GetLocked')
    306   def testRefetchingFileUponHashFileChange(
    307       self, mock_get_locked, mock_calculate_hash, unused_get_locked):
    308     mock_calculate_hash.side_effect = _FakeCalulateHashNewHash
    309     cloud_storage.ReadHash = _FakeReadHash
    310     file_path = 'test-file-path.wpr'
    311     hash_path = file_path + '.sha1'
    312 
    313     def _FakeGetLocked(bucket, expected_hash, file_path):
    314       del bucket, expected_hash, file_path  # unused
    315       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    316 
    317     mock_get_locked.side_effect = _FakeGetLocked
    318 
    319     self.CreateFiles([file_path, hash_path])
    320     # hash_path and file_path exist, and have different hashes. This first call
    321     # will invoke a fetch.
    322     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    323                                                cloud_storage.PUBLIC_BUCKET))
    324 
    325     # The fetch left a .fetchts file on machine.
    326     self.assertTrue(os.path.exists(file_path + '.fetchts'))
    327 
    328     with open(file_path + '.fetchts') as f:
    329       fetchts = float(f.read())
    330 
    331     # Updating the .sha1 hash_path file with the new hash after .fetchts
    332     # is created.
    333     file_obj = self.fs.GetObject(hash_path)
    334     file_obj.SetMTime(fetchts + 100)
    335 
    336     cloud_storage.ReadHash = lambda _: 'hashNeW'
    337     def _FakeGetLockedNewHash(bucket, expected_hash, file_path):
    338       del bucket, expected_hash, file_path  # unused
    339       cloud_storage.CalculateHash = lambda _: 'hashNeW'
    340 
    341     mock_get_locked.side_effect = _FakeGetLockedNewHash
    342 
    343     # hash_path and file_path exist, and have different hashes. This first call
    344     # will invoke a fetch.
    345     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    346                                                cloud_storage.PUBLIC_BUCKET))
    347 
    348 
    349 class CloudStorageRealFsUnitTest(unittest.TestCase):
    350 
    351   def setUp(self):
    352     self.original_environ = os.environ.copy()
    353     os.environ['DISABLE_CLOUD_STORAGE_IO'] = ''
    354 
    355   def tearDown(self):
    356     os.environ = self.original_environ
    357 
    358   @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005)
    359   def testGetPseudoLockUnavailableCausesTimeout(self):
    360     with tempfile.NamedTemporaryFile(suffix='.pseudo_lock') as pseudo_lock_fd:
    361       with lock.FileLock(pseudo_lock_fd, lock.LOCK_EX | lock.LOCK_NB):
    362         with self.assertRaises(py_utils.TimeoutException):
    363           file_path = pseudo_lock_fd.name.replace('.pseudo_lock', '')
    364           cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET)
    365 
    366   @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005)
    367   def testGetGlobalLockUnavailableCausesTimeout(self):
    368     with open(_CLOUD_STORAGE_GLOBAL_LOCK_PATH) as global_lock_fd:
    369       with lock.FileLock(global_lock_fd, lock.LOCK_EX | lock.LOCK_NB):
    370         tmp_dir = tempfile.mkdtemp()
    371         try:
    372           file_path = os.path.join(tmp_dir, 'foo')
    373           with self.assertRaises(py_utils.TimeoutException):
    374             cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET)
    375         finally:
    376           shutil.rmtree(tmp_dir)
    377 
    378 
    379 class CloudStorageErrorHandlingTest(unittest.TestCase):
    380   def runTest(self):
    381     self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr(
    382         'ServiceException: 401 Anonymous users does not have '
    383         'storage.objects.get access to object chrome-partner-telemetry'),
    384                           cloud_storage.CredentialsError)
    385     self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr(
    386         '403 Caller does not have storage.objects.list access to bucket '
    387         'chrome-telemetry'), cloud_storage.PermissionError)
    388