1 """Tests for job_directories.""" 2 3 from __future__ import absolute_import 4 from __future__ import division 5 from __future__ import print_function 6 7 import contextlib 8 import datetime 9 import mox 10 import os 11 import shutil 12 import tempfile 13 import unittest 14 15 import common 16 from autotest_lib.site_utils import job_directories 17 from autotest_lib.client.common_lib import time_utils 18 19 20 class SwarmingJobDirectoryTestCase(unittest.TestCase): 21 """Tests SwarmingJobDirectory.""" 22 23 def test_get_job_directories_legacy(self): 24 with _change_to_tempdir(): 25 os.makedirs("swarming-3e4391423c3a4311/b") 26 os.mkdir("not-a-swarming-dir") 27 results = job_directories.SwarmingJobDirectory.get_job_directories() 28 self.assertEqual(set(results), {"swarming-3e4391423c3a4311"}) 29 30 def test_get_job_directories(self): 31 with _change_to_tempdir(): 32 os.makedirs("swarming-3e4391423c3a4310/1") 33 os.makedirs("swarming-3e4391423c3a4310/0") 34 os.makedirs("swarming-3e4391423c3a4310/a") 35 os.mkdir("not-a-swarming-dir") 36 results = job_directories.SwarmingJobDirectory.get_job_directories() 37 self.assertEqual(set(results), 38 {"swarming-3e4391423c3a4310/1", 39 "swarming-3e4391423c3a4310/a"}) 40 41 42 class GetJobIDOrTaskID(unittest.TestCase): 43 """Tests get_job_id_or_task_id.""" 44 45 def test_legacy_swarming_path(self): 46 self.assertEqual( 47 "3e4391423c3a4311", 48 job_directories.get_job_id_or_task_id( 49 "/autotest/results/swarming-3e4391423c3a4311"), 50 ) 51 self.assertEqual( 52 "3e4391423c3a4311", 53 job_directories.get_job_id_or_task_id( 54 "swarming-3e4391423c3a4311"), 55 ) 56 57 def test_swarming_path(self): 58 self.assertEqual( 59 "3e4391423c3a4311", 60 job_directories.get_job_id_or_task_id( 61 "/autotest/results/swarming-3e4391423c3a4310/1"), 62 ) 63 self.assertEqual( 64 "3e4391423c3a431f", 65 job_directories.get_job_id_or_task_id( 66 "swarming-3e4391423c3a4310/f"), 67 ) 68 69 70 71 class JobDirectorySubclassTests(mox.MoxTestBase): 72 """Test specific to RegularJobDirectory and SpecialJobDirectory. 73 74 This provides coverage for the implementation in both 75 RegularJobDirectory and SpecialJobDirectory. 76 77 """ 78 79 def setUp(self): 80 super(JobDirectorySubclassTests, self).setUp() 81 self.mox.StubOutWithMock(job_directories, '_AFE') 82 83 84 def test_regular_job_fields(self): 85 """Test the constructor for `RegularJobDirectory`. 86 87 Construct a regular job, and assert that the `dirname` 88 and `_id` attributes are set as expected. 89 90 """ 91 resultsdir = '118-fubar' 92 job = job_directories.RegularJobDirectory(resultsdir) 93 self.assertEqual(job.dirname, resultsdir) 94 self.assertEqual(job._id, '118') 95 96 97 def test_special_job_fields(self): 98 """Test the constructor for `SpecialJobDirectory`. 99 100 Construct a special job, and assert that the `dirname` 101 and `_id` attributes are set as expected. 102 103 """ 104 destdir = 'hosts/host1' 105 resultsdir = destdir + '/118-reset' 106 job = job_directories.SpecialJobDirectory(resultsdir) 107 self.assertEqual(job.dirname, resultsdir) 108 self.assertEqual(job._id, '118') 109 110 111 def _check_finished_job(self, jobtime, hqetimes, expected): 112 """Mock and test behavior of a finished job. 113 114 Initialize the mocks for a call to 115 `get_timestamp_if_finished()`, then simulate one call. 116 Assert that the returned timestamp matches the passed 117 in expected value. 118 119 @param jobtime Time used to construct a _MockJob object. 120 @param hqetimes List of times used to construct 121 _MockHostQueueEntry objects. 122 @param expected Expected time to be returned by 123 get_timestamp_if_finished 124 125 """ 126 job = job_directories.RegularJobDirectory('118-fubar') 127 job_directories._AFE.get_jobs( 128 id=job._id, finished=True).AndReturn( 129 [_MockJob(jobtime)]) 130 job_directories._AFE.get_host_queue_entries( 131 finished_on__isnull=False, 132 job_id=job._id).AndReturn( 133 [_MockHostQueueEntry(t) for t in hqetimes]) 134 self.mox.ReplayAll() 135 self.assertEqual(expected, job.get_timestamp_if_finished()) 136 self.mox.VerifyAll() 137 138 139 def test_finished_regular_job(self): 140 """Test getting the timestamp for a finished regular job. 141 142 Tests the return value for 143 `RegularJobDirectory.get_timestamp_if_finished()` when 144 the AFE indicates the job is finished. 145 146 """ 147 created_timestamp = make_timestamp(1, True) 148 hqe_timestamp = make_timestamp(0, True) 149 self._check_finished_job(created_timestamp, 150 [hqe_timestamp], 151 hqe_timestamp) 152 153 154 def test_finished_regular_job_multiple_hqes(self): 155 """Test getting the timestamp for a regular job with multiple hqes. 156 157 Tests the return value for 158 `RegularJobDirectory.get_timestamp_if_finished()` when 159 the AFE indicates the job is finished and the job has multiple host 160 queue entries. 161 162 Tests that the returned timestamp is the latest timestamp in 163 the list of HQEs, regardless of the returned order. 164 165 """ 166 created_timestamp = make_timestamp(2, True) 167 older_hqe_timestamp = make_timestamp(1, True) 168 newer_hqe_timestamp = make_timestamp(0, True) 169 hqe_list = [older_hqe_timestamp, 170 newer_hqe_timestamp] 171 self._check_finished_job(created_timestamp, 172 hqe_list, 173 newer_hqe_timestamp) 174 self.mox.ResetAll() 175 hqe_list.reverse() 176 self._check_finished_job(created_timestamp, 177 hqe_list, 178 newer_hqe_timestamp) 179 180 181 def test_finished_regular_job_null_finished_times(self): 182 """Test getting the timestamp for an aborted regular job. 183 184 Tests the return value for 185 `RegularJobDirectory.get_timestamp_if_finished()` when 186 the AFE indicates the job is finished and the job has aborted host 187 queue entries. 188 189 """ 190 timestamp = make_timestamp(0, True) 191 self._check_finished_job(timestamp, [], timestamp) 192 193 194 def test_unfinished_regular_job(self): 195 """Test getting the timestamp for an unfinished regular job. 196 197 Tests the return value for 198 `RegularJobDirectory.get_timestamp_if_finished()` when 199 the AFE indicates the job is not finished. 200 201 """ 202 job = job_directories.RegularJobDirectory('118-fubar') 203 job_directories._AFE.get_jobs( 204 id=job._id, finished=True).AndReturn([]) 205 self.mox.ReplayAll() 206 self.assertIsNone(job.get_timestamp_if_finished()) 207 self.mox.VerifyAll() 208 209 210 def test_finished_special_job(self): 211 """Test getting the timestamp for a finished special job. 212 213 Tests the return value for 214 `SpecialJobDirectory.get_timestamp_if_finished()` when 215 the AFE indicates the job is finished. 216 217 """ 218 job = job_directories.SpecialJobDirectory( 219 'hosts/host1/118-reset') 220 timestamp = make_timestamp(0, True) 221 job_directories._AFE.get_special_tasks( 222 id=job._id, is_complete=True).AndReturn( 223 [_MockSpecialTask(timestamp)]) 224 self.mox.ReplayAll() 225 self.assertEqual(timestamp, 226 job.get_timestamp_if_finished()) 227 self.mox.VerifyAll() 228 229 230 def test_unfinished_special_job(self): 231 """Test getting the timestamp for an unfinished special job. 232 233 Tests the return value for 234 `SpecialJobDirectory.get_timestamp_if_finished()` when 235 the AFE indicates the job is not finished. 236 237 """ 238 job = job_directories.SpecialJobDirectory( 239 'hosts/host1/118-reset') 240 job_directories._AFE.get_special_tasks( 241 id=job._id, is_complete=True).AndReturn([]) 242 self.mox.ReplayAll() 243 self.assertIsNone(job.get_timestamp_if_finished()) 244 self.mox.VerifyAll() 245 246 247 class JobExpirationTests(unittest.TestCase): 248 """Tests to exercise `job_directories.is_job_expired()`.""" 249 250 def test_expired(self): 251 """Test detection of an expired job.""" 252 timestamp = make_timestamp(_TEST_EXPIRATION_AGE, True) 253 self.assertTrue( 254 job_directories.is_job_expired( 255 _TEST_EXPIRATION_AGE, timestamp)) 256 257 258 def test_alive(self): 259 """Test detection of a job that's not expired.""" 260 # N.B. This test may fail if its run time exceeds more than 261 # about _MARGIN_SECS seconds. 262 timestamp = make_timestamp(_TEST_EXPIRATION_AGE, False) 263 self.assertFalse( 264 job_directories.is_job_expired( 265 _TEST_EXPIRATION_AGE, timestamp)) 266 267 268 # When constructing sample time values for testing expiration, 269 # allow this many seconds between the expiration time and the 270 # current time. 271 _MARGIN_SECS = 10.0 272 # Test value to use for `days_old`, if nothing else is required. 273 _TEST_EXPIRATION_AGE = 7 274 275 276 class _MockJob(object): 277 """Class to mock the return value of `AFE.get_jobs()`.""" 278 def __init__(self, created): 279 self.created_on = created 280 281 282 class _MockHostQueueEntry(object): 283 """Class to mock the return value of `AFE.get_host_queue_entries()`.""" 284 def __init__(self, finished): 285 self.finished_on = finished 286 287 288 class _MockSpecialTask(object): 289 """Class to mock the return value of `AFE.get_special_tasks()`.""" 290 def __init__(self, finished): 291 self.time_finished = finished 292 293 294 @contextlib.contextmanager 295 def _change_to_tempdir(): 296 old_dir = os.getcwd() 297 tempdir = tempfile.mkdtemp('job_directories_unittest') 298 try: 299 os.chdir(tempdir) 300 yield 301 finally: 302 os.chdir(old_dir) 303 shutil.rmtree(tempdir) 304 305 306 def make_timestamp(age_limit, is_expired): 307 """Create a timestamp for use by `job_directories.is_job_expired()`. 308 309 The timestamp will meet the syntactic requirements for 310 timestamps used as input to `is_job_expired()`. If 311 `is_expired` is true, the timestamp will be older than 312 `age_limit` days before the current time; otherwise, the 313 date will be younger. 314 315 @param age_limit The number of days before expiration of the 316 target timestamp. 317 @param is_expired Whether the timestamp should be expired 318 relative to `age_limit`. 319 320 """ 321 seconds = -_MARGIN_SECS 322 if is_expired: 323 seconds = -seconds 324 delta = datetime.timedelta(days=age_limit, seconds=seconds) 325 reference_time = datetime.datetime.now() - delta 326 return reference_time.strftime(time_utils.TIME_FMT) 327 328 329 if __name__ == '__main__': 330 unittest.main() 331
