Home | History | Annotate | Download | only in py_utils 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import os
      6 import shutil
      7 import sys
      8 import tempfile
      9 import unittest
     10 
     11 import mock
     12 from pyfakefs import fake_filesystem_unittest
     13 
     14 import py_utils
     15 from py_utils import cloud_storage
     16 from py_utils import lock
     17 
     18 _CLOUD_STORAGE_GLOBAL_LOCK_PATH = os.path.join(
     19     os.path.dirname(__file__), 'cloud_storage_global_lock.py')
     20 
     21 def _FakeReadHash(_):
     22   return 'hashthis!'
     23 
     24 
     25 def _FakeCalulateHashMatchesRead(_):
     26   return 'hashthis!'
     27 
     28 
     29 def _FakeCalulateHashNewHash(_):
     30   return 'omgnewhash'
     31 
     32 
     33 class BaseFakeFsUnitTest(fake_filesystem_unittest.TestCase):
     34 
     35   def setUp(self):
     36     self.original_environ = os.environ.copy()
     37     os.environ['DISABLE_CLOUD_STORAGE_IO'] = ''
     38     self.setUpPyfakefs()
     39     self.fs.CreateFile(
     40         os.path.join(py_utils.GetCatapultDir(),
     41                      'third_party', 'gsutil', 'gsutil'))
     42 
     43   def CreateFiles(self, file_paths):
     44     for f in file_paths:
     45       self.fs.CreateFile(f)
     46 
     47   def tearDown(self):
     48     self.tearDownPyfakefs()
     49     os.environ = self.original_environ
     50 
     51   def _FakeRunCommand(self, cmd):
     52     pass
     53 
     54   def _FakeGet(self, bucket, remote_path, local_path):
     55     pass
     56 
     57 
     58 class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest):
     59 
     60   def _AssertRunCommandRaisesError(self, communicate_strs, error):
     61     with mock.patch('py_utils.cloud_storage.subprocess.Popen') as popen:
     62       p_mock = mock.Mock()
     63       popen.return_value = p_mock
     64       p_mock.returncode = 1
     65       for stderr in communicate_strs:
     66         p_mock.communicate.return_value = ('', stderr)
     67         self.assertRaises(error, cloud_storage._RunCommand, [])
     68 
     69   def testRunCommandCredentialsError(self):
     70     strs = ['You are attempting to access protected data with no configured',
     71             'Failure: No handler was ready to authenticate.']
     72     self._AssertRunCommandRaisesError(strs, cloud_storage.CredentialsError)
     73 
     74   def testRunCommandPermissionError(self):
     75     strs = ['status=403', 'status 403', '403 Forbidden']
     76     self._AssertRunCommandRaisesError(strs, cloud_storage.PermissionError)
     77 
     78   def testRunCommandNotFoundError(self):
     79     strs = ['InvalidUriError', 'No such object', 'No URLs matched',
     80             'One or more URLs matched no', 'InvalidUriError']
     81     self._AssertRunCommandRaisesError(strs, cloud_storage.NotFoundError)
     82 
     83   def testRunCommandServerError(self):
     84     strs = ['500 Internal Server Error']
     85     self._AssertRunCommandRaisesError(strs, cloud_storage.ServerError)
     86 
     87   def testRunCommandGenericError(self):
     88     strs = ['Random string']
     89     self._AssertRunCommandRaisesError(strs, cloud_storage.CloudStorageError)
     90 
     91   def testInsertCreatesValidCloudUrl(self):
     92     orig_run_command = cloud_storage._RunCommand
     93     try:
     94       cloud_storage._RunCommand = self._FakeRunCommand
     95       remote_path = 'test-remote-path.html'
     96       local_path = 'test-local-path.html'
     97       cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET,
     98                                        remote_path, local_path)
     99       self.assertEqual('https://console.developers.google.com/m/cloudstorage'
    100                        '/b/chromium-telemetry/o/test-remote-path.html',
    101                        cloud_url)
    102     finally:
    103       cloud_storage._RunCommand = orig_run_command
    104 
    105   @mock.patch('py_utils.cloud_storage.subprocess')
    106   def testExistsReturnsFalse(self, subprocess_mock):
    107     p_mock = mock.Mock()
    108     subprocess_mock.Popen.return_value = p_mock
    109     p_mock.communicate.return_value = (
    110         '',
    111         'CommandException: One or more URLs matched no objects.\n')
    112     p_mock.returncode_result = 1
    113     self.assertFalse(cloud_storage.Exists('fake bucket',
    114                                           'fake remote path'))
    115 
    116   @unittest.skipIf(sys.platform.startswith('win'),
    117                    'https://github.com/catapult-project/catapult/issues/1861')
    118   def testGetFilesInDirectoryIfChanged(self):
    119     self.CreateFiles([
    120         'real_dir_path/dir1/1file1.sha1',
    121         'real_dir_path/dir1/1file2.txt',
    122         'real_dir_path/dir1/1file3.sha1',
    123         'real_dir_path/dir2/2file.txt',
    124         'real_dir_path/dir3/3file1.sha1'])
    125 
    126     def IncrementFilesUpdated(*_):
    127       IncrementFilesUpdated.files_updated += 1
    128     IncrementFilesUpdated.files_updated = 0
    129     orig_get_if_changed = cloud_storage.GetIfChanged
    130     cloud_storage.GetIfChanged = IncrementFilesUpdated
    131     try:
    132       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
    133                         os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET)
    134       self.assertEqual(0, IncrementFilesUpdated.files_updated)
    135       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
    136                         'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
    137       self.assertEqual(0, IncrementFilesUpdated.files_updated)
    138       cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path',
    139                                                  cloud_storage.PUBLIC_BUCKET)
    140       self.assertEqual(3, IncrementFilesUpdated.files_updated)
    141     finally:
    142       cloud_storage.GetIfChanged = orig_get_if_changed
    143 
    144   def testCopy(self):
    145     orig_run_command = cloud_storage._RunCommand
    146 
    147     def AssertCorrectRunCommandArgs(args):
    148       self.assertEqual(expected_args, args)
    149     cloud_storage._RunCommand = AssertCorrectRunCommandArgs
    150     expected_args = ['cp', 'gs://bucket1/remote_path1',
    151                      'gs://bucket2/remote_path2']
    152     try:
    153       cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
    154     finally:
    155       cloud_storage._RunCommand = orig_run_command
    156 
    157   @mock.patch('py_utils.cloud_storage._FileLock')
    158   def testDisableCloudStorageIo(self, unused_lock_mock):
    159     os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
    160     dir_path = 'real_dir_path'
    161     self.fs.CreateDirectory(dir_path)
    162     file_path = os.path.join(dir_path, 'file1')
    163     file_path_sha = file_path + '.sha1'
    164 
    165     def CleanTimeStampFile():
    166       os.remove(file_path + '.fetchts')
    167 
    168     self.CreateFiles([file_path, file_path_sha])
    169     with open(file_path_sha, 'w') as f:
    170       f.write('hash1234')
    171     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    172       cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
    173     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    174       cloud_storage.Get('bucket', 'foo', file_path)
    175     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    176       cloud_storage.GetIfChanged(file_path, 'foo')
    177     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    178       cloud_storage.GetIfHashChanged('bar', file_path, 'bucket', 'hash1234')
    179     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    180       cloud_storage.Insert('bucket', 'foo', file_path)
    181 
    182     CleanTimeStampFile()
    183     with self.assertRaises(cloud_storage.CloudStorageIODisabled):
    184       cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
    185 
    186 
    187 class GetIfChangedTests(BaseFakeFsUnitTest):
    188 
    189   def setUp(self):
    190     super(GetIfChangedTests, self).setUp()
    191     self._orig_read_hash = cloud_storage.ReadHash
    192     self._orig_calculate_hash = cloud_storage.CalculateHash
    193 
    194   def tearDown(self):
    195     super(GetIfChangedTests, self).tearDown()
    196     cloud_storage.CalculateHash = self._orig_calculate_hash
    197     cloud_storage.ReadHash = self._orig_read_hash
    198 
    199   @mock.patch('py_utils.cloud_storage._FileLock')
    200   @mock.patch('py_utils.cloud_storage._GetLocked')
    201   def testHashPathDoesNotExists(self, unused_get_locked, unused_lock_mock):
    202     cloud_storage.ReadHash = _FakeReadHash
    203     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    204     file_path = 'test-file-path.wpr'
    205 
    206     cloud_storage._GetLocked = self._FakeGet
    207     # hash_path doesn't exist.
    208     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    209                                                 cloud_storage.PUBLIC_BUCKET))
    210 
    211   @mock.patch('py_utils.cloud_storage._FileLock')
    212   @mock.patch('py_utils.cloud_storage._GetLocked')
    213   def testHashPathExistsButFilePathDoesNot(
    214       self, unused_get_locked, unused_lock_mock):
    215     cloud_storage.ReadHash = _FakeReadHash
    216     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    217     file_path = 'test-file-path.wpr'
    218     hash_path = file_path + '.sha1'
    219 
    220     # hash_path exists, but file_path doesn't.
    221     self.CreateFiles([hash_path])
    222     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    223                                                cloud_storage.PUBLIC_BUCKET))
    224 
    225   @mock.patch('py_utils.cloud_storage._FileLock')
    226   @mock.patch('py_utils.cloud_storage._GetLocked')
    227   def testHashPathAndFileHashExistWithSameHash(
    228       self, unused_get_locked, unused_lock_mock):
    229     cloud_storage.ReadHash = _FakeReadHash
    230     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    231     file_path = 'test-file-path.wpr'
    232 
    233     # hash_path and file_path exist, and have same hash.
    234     self.CreateFiles([file_path])
    235     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    236                                                 cloud_storage.PUBLIC_BUCKET))
    237 
    238   @mock.patch('py_utils.cloud_storage._FileLock')
    239   @mock.patch('py_utils.cloud_storage._GetLocked')
    240   def testHashPathAndFileHashExistWithDifferentHash(
    241       self, mock_get_locked, unused_get_locked):
    242     cloud_storage.ReadHash = _FakeReadHash
    243     cloud_storage.CalculateHash = _FakeCalulateHashNewHash
    244     file_path = 'test-file-path.wpr'
    245     hash_path = file_path + '.sha1'
    246 
    247     def _FakeGetLocked(bucket, expected_hash, file_path):
    248       del bucket, expected_hash, file_path  # unused
    249       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    250 
    251     mock_get_locked.side_effect = _FakeGetLocked
    252 
    253     self.CreateFiles([file_path, hash_path])
    254     # hash_path and file_path exist, and have different hashes.
    255     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    256                                                cloud_storage.PUBLIC_BUCKET))
    257 
    258   @mock.patch('py_utils.cloud_storage._FileLock')
    259   @mock.patch('py_utils.cloud_storage.CalculateHash')
    260   @mock.patch('py_utils.cloud_storage._GetLocked')
    261   def testNoHashComputationNeededUponSecondCall(
    262       self, mock_get_locked, mock_calculate_hash, unused_get_locked):
    263     mock_calculate_hash.side_effect = _FakeCalulateHashNewHash
    264     cloud_storage.ReadHash = _FakeReadHash
    265     file_path = 'test-file-path.wpr'
    266     hash_path = file_path + '.sha1'
    267 
    268     def _FakeGetLocked(bucket, expected_hash, file_path):
    269       del bucket, expected_hash, file_path  # unused
    270       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    271 
    272     mock_get_locked.side_effect = _FakeGetLocked
    273 
    274     self.CreateFiles([file_path, hash_path])
    275     # hash_path and file_path exist, and have different hashes. This first call
    276     # will invoke a fetch.
    277     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    278                                                cloud_storage.PUBLIC_BUCKET))
    279 
    280     # The fetch left a .fetchts file on machine.
    281     self.assertTrue(os.path.exists(file_path + '.fetchts'))
    282 
    283     # Subsequent invocations of GetIfChanged should not invoke CalculateHash.
    284     mock_calculate_hash.assert_not_called()
    285     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    286                                                 cloud_storage.PUBLIC_BUCKET))
    287     self.assertFalse(cloud_storage.GetIfChanged(file_path,
    288                                                 cloud_storage.PUBLIC_BUCKET))
    289 
    290   @mock.patch('py_utils.cloud_storage._FileLock')
    291   @mock.patch('py_utils.cloud_storage.CalculateHash')
    292   @mock.patch('py_utils.cloud_storage._GetLocked')
    293   def testRefetchingFileUponHashFileChange(
    294       self, mock_get_locked, mock_calculate_hash, unused_get_locked):
    295     mock_calculate_hash.side_effect = _FakeCalulateHashNewHash
    296     cloud_storage.ReadHash = _FakeReadHash
    297     file_path = 'test-file-path.wpr'
    298     hash_path = file_path + '.sha1'
    299 
    300     def _FakeGetLocked(bucket, expected_hash, file_path):
    301       del bucket, expected_hash, file_path  # unused
    302       cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
    303 
    304     mock_get_locked.side_effect = _FakeGetLocked
    305 
    306     self.CreateFiles([file_path, hash_path])
    307     # hash_path and file_path exist, and have different hashes. This first call
    308     # will invoke a fetch.
    309     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    310                                                cloud_storage.PUBLIC_BUCKET))
    311 
    312     # The fetch left a .fetchts file on machine.
    313     self.assertTrue(os.path.exists(file_path + '.fetchts'))
    314 
    315     with open(file_path + '.fetchts') as f:
    316       fetchts = float(f.read())
    317 
    318     # Updating the .sha1 hash_path file with the new hash after .fetchts
    319     # is created.
    320     file_obj = self.fs.GetObject(hash_path)
    321     file_obj.SetMTime(fetchts + 100)
    322 
    323     cloud_storage.ReadHash = lambda _: 'hashNeW'
    324     def _FakeGetLockedNewHash(bucket, expected_hash, file_path):
    325       del bucket, expected_hash, file_path  # unused
    326       cloud_storage.CalculateHash = lambda _: 'hashNeW'
    327 
    328     mock_get_locked.side_effect = _FakeGetLockedNewHash
    329 
    330     # hash_path and file_path exist, and have different hashes. This first call
    331     # will invoke a fetch.
    332     self.assertTrue(cloud_storage.GetIfChanged(file_path,
    333                                                cloud_storage.PUBLIC_BUCKET))
    334 
    335 
    336 class CloudStorageRealFsUnitTest(unittest.TestCase):
    337 
    338   def setUp(self):
    339     self.original_environ = os.environ.copy()
    340     os.environ['DISABLE_CLOUD_STORAGE_IO'] = ''
    341 
    342   def tearDown(self):
    343     os.environ = self.original_environ
    344 
    345   @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005)
    346   def testGetPseudoLockUnavailableCausesTimeout(self):
    347     with tempfile.NamedTemporaryFile(suffix='.pseudo_lock') as pseudo_lock_fd:
    348       with lock.FileLock(pseudo_lock_fd, lock.LOCK_EX | lock.LOCK_NB):
    349         with self.assertRaises(py_utils.TimeoutException):
    350           file_path = pseudo_lock_fd.name.replace('.pseudo_lock', '')
    351           cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET)
    352 
    353   @mock.patch('py_utils.cloud_storage.LOCK_ACQUISITION_TIMEOUT', .005)
    354   def testGetGlobalLockUnavailableCausesTimeout(self):
    355     with open(_CLOUD_STORAGE_GLOBAL_LOCK_PATH) as global_lock_fd:
    356       with lock.FileLock(global_lock_fd, lock.LOCK_EX | lock.LOCK_NB):
    357         tmp_dir = tempfile.mkdtemp()
    358         try:
    359           file_path = os.path.join(tmp_dir, 'foo')
    360           with self.assertRaises(py_utils.TimeoutException):
    361             cloud_storage.GetIfChanged(file_path, cloud_storage.PUBLIC_BUCKET)
    362         finally:
    363           shutil.rmtree(tmp_dir)
    364 
    365 
    366 class CloudStorageErrorHandlingTest(unittest.TestCase):
    367   def runTest(self):
    368     self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr(
    369         'ServiceException: 401 Anonymous users does not have '
    370         'storage.objects.get access to object chrome-partner-telemetry'),
    371                           cloud_storage.CredentialsError)
    372     self.assertIsInstance(cloud_storage.GetErrorObjectForCloudStorageStderr(
    373         '403 Caller does not have storage.objects.list access to bucket '
    374         'chrome-telemetry'), cloud_storage.PermissionError)
    375