1 /* 2 * Copyright (c) 2016 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.job; 18 19 import com.android.vts.entity.DeviceInfoEntity; 20 import com.android.vts.entity.TestAcknowledgmentEntity; 21 import com.android.vts.entity.TestCaseRunEntity; 22 import com.android.vts.entity.TestCaseRunEntity.TestCase; 23 import com.android.vts.entity.TestRunEntity; 24 import com.android.vts.entity.TestStatusEntity; 25 import com.android.vts.entity.TestStatusEntity.TestCaseReference; 26 import com.android.vts.proto.VtsReportMessage.TestCaseResult; 27 import com.android.vts.util.DatastoreHelper; 28 import com.android.vts.util.EmailHelper; 29 import com.android.vts.util.FilterUtil; 30 import com.android.vts.util.TimeUtil; 31 import com.google.appengine.api.datastore.DatastoreFailureException; 32 import com.google.appengine.api.datastore.DatastoreService; 33 import com.google.appengine.api.datastore.DatastoreServiceFactory; 34 import com.google.appengine.api.datastore.DatastoreTimeoutException; 35 import com.google.appengine.api.datastore.Entity; 36 import com.google.appengine.api.datastore.EntityNotFoundException; 37 import com.google.appengine.api.datastore.FetchOptions; 38 import com.google.appengine.api.datastore.Key; 39 import com.google.appengine.api.datastore.KeyFactory; 40 import com.google.appengine.api.datastore.Query; 41 import com.google.appengine.api.datastore.Query.Filter; 42 import com.google.appengine.api.datastore.Query.SortDirection; 43 import com.google.appengine.api.datastore.Transaction; 44 import com.google.appengine.api.taskqueue.Queue; 45 import com.google.appengine.api.taskqueue.QueueFactory; 46 import com.google.appengine.api.taskqueue.TaskOptions; 47 import java.io.IOException; 48 import java.io.UnsupportedEncodingException; 49 import java.util.ArrayList; 50 import java.util.Comparator; 51 import java.util.ConcurrentModificationException; 52 import java.util.HashMap; 53 import java.util.HashSet; 54 import java.util.List; 55 import java.util.Map; 56 import java.util.Set; 57 import java.util.concurrent.TimeUnit; 58 import java.util.logging.Level; 59 import java.util.logging.Logger; 60 import javax.mail.Message; 61 import javax.mail.MessagingException; 62 import javax.servlet.http.HttpServlet; 63 import javax.servlet.http.HttpServletRequest; 64 import javax.servlet.http.HttpServletResponse; 65 import org.apache.commons.lang.StringUtils; 66 67 /** Represents the notifications service which is automatically called on a fixed schedule. */ 68 public class VtsAlertJobServlet extends HttpServlet { 69 private static final String ALERT_JOB_URL = "/task/vts_alert_job"; 70 protected static final Logger logger = Logger.getLogger(VtsAlertJobServlet.class.getName()); 71 protected static final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for 72 73 /** 74 * Process the current test case failures for a test. 75 * 76 * @param status The TestStatusEntity object for the test. 77 * @returns a map from test case name to the test case run ID for which the test case failed. 78 */ 79 private static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) { 80 if (status.failingTestCases == null || status.failingTestCases.size() == 0) { 81 return new HashMap<>(); 82 } 83 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 84 Map<String, TestCase> failingTestcases = new HashMap<>(); 85 Set<Key> gets = new HashSet<>(); 86 for (TestCaseReference testCaseRef : status.failingTestCases) { 87 gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId)); 88 } 89 if (gets.size() == 0) { 90 return failingTestcases; 91 } 92 Map<Key, Entity> testCaseMap = datastore.get(gets); 93 94 for (TestCaseReference testCaseRef : status.failingTestCases) { 95 Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId); 96 if (!testCaseMap.containsKey(key)) { 97 continue; 98 } 99 Entity testCaseRun = testCaseMap.get(key); 100 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 101 if (testCaseRunEntity.testCases.size() <= testCaseRef.offset) { 102 continue; 103 } 104 TestCase testCase = testCaseRunEntity.testCases.get(testCaseRef.offset); 105 failingTestcases.put(testCase.name, testCase); 106 } 107 return failingTestcases; 108 } 109 110 /** 111 * Get the test acknowledgments for a test key. 112 * 113 * @param testKey The key to the test whose acknowledgments to fetch. 114 * @return A list of test acknowledgments. 115 */ 116 private static List<TestAcknowledgmentEntity> getTestCaseAcknowledgments(Key testKey) { 117 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 118 119 List<TestAcknowledgmentEntity> acks = new ArrayList<>(); 120 Filter testFilter = 121 new Query.FilterPredicate( 122 TestAcknowledgmentEntity.TEST_KEY, Query.FilterOperator.EQUAL, testKey); 123 Query q = new Query(TestAcknowledgmentEntity.KIND).setFilter(testFilter); 124 125 for (Entity ackEntity : datastore.prepare(q).asIterable()) { 126 TestAcknowledgmentEntity ack = TestAcknowledgmentEntity.fromEntity(ackEntity); 127 if (ack == null) continue; 128 acks.add(ack); 129 } 130 return acks; 131 } 132 133 /** 134 * Get the test runs for the test in the specified time window. 135 * 136 * If the start and end time delta is greater than one day, the query will be truncated. 137 * 138 * @param testKey The key to the test whose runs to query. 139 * @param startTime The start time for the query. 140 * @param endTime The end time for the query. 141 * @return A list of test runs in the specified time window. 142 */ 143 private static List<TestRunEntity> getTestRuns(Key testKey, long startTime, long endTime) { 144 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 145 Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); 146 long delta = endTime - startTime; 147 delta = Math.min(delta, TimeUnit.DAYS.toMicros(1)); 148 Filter runFilter = 149 FilterUtil.getTimeFilter( 150 testKey, TestRunEntity.KIND, endTime - delta + 1, endTime, testTypeFilter); 151 152 Query q = 153 new Query(TestRunEntity.KIND) 154 .setAncestor(testKey) 155 .setFilter(runFilter) 156 .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING); 157 158 List<TestRunEntity> testRuns = new ArrayList<>(); 159 for (Entity testRunEntity : 160 datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) { 161 TestRunEntity testRun = TestRunEntity.fromEntity(testRunEntity); 162 if (testRun == null) continue; 163 testRuns.add(testRun); 164 } 165 return testRuns; 166 } 167 168 /** 169 * Separate the test cases which are acknowledged by the provided acknowledgments. 170 * 171 * @param testCases The list of test case names. 172 * @param devices The list of devices for a test run. 173 * @param acks The list of acknowledgments for the test. 174 * @return A list of acknowledged test case names that have been removed from the input test 175 * cases. 176 */ 177 public static Set<String> separateAcknowledged( 178 Set<String> testCases, 179 List<DeviceInfoEntity> devices, 180 List<TestAcknowledgmentEntity> acks) { 181 Set<String> acknowledged = new HashSet<>(); 182 for (TestAcknowledgmentEntity ack : acks) { 183 boolean allDevices = ack.devices == null || ack.devices.size() == 0; 184 boolean allBranches = ack.branches == null || ack.branches.size() == 0; 185 boolean isRelevant = allDevices && allBranches; 186 187 // Determine if the acknowledgment is relevant to the devices. 188 if (!isRelevant) { 189 for (DeviceInfoEntity device : devices) { 190 boolean deviceAcknowledged = 191 allDevices || ack.devices.contains(device.buildFlavor); 192 boolean branchAcknowledged = 193 allBranches || ack.branches.contains(device.branch); 194 if (deviceAcknowledged && branchAcknowledged) isRelevant = true; 195 } 196 } 197 198 if (isRelevant) { 199 // Separate the test cases 200 boolean allTestCases = ack.testCaseNames == null || ack.testCaseNames.size() == 0; 201 if (allTestCases) { 202 acknowledged.addAll(testCases); 203 testCases.removeAll(acknowledged); 204 } else { 205 for (String testCase : ack.testCaseNames) { 206 if (testCases.contains(testCase)) { 207 acknowledged.add(testCase); 208 testCases.remove(testCase); 209 } 210 } 211 } 212 } 213 } 214 return acknowledged; 215 } 216 217 /** 218 * Checks whether any new failures have occurred beginning since (and including) startTime. 219 * 220 * @param testRuns The list of test runs for which to update the status. 221 * @param link The string URL linking to the test's status table. 222 * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last 223 * status update. 224 * @param emailAddresses The list of email addresses to send notifications to. 225 * @param messages The email Message queue. 226 * @returns latest TestStatusMessage or null if no update is available. 227 * @throws IOException 228 */ 229 public TestStatusEntity getTestStatus( 230 List<TestRunEntity> testRuns, 231 String link, 232 Map<String, TestCase> failedTestCaseMap, 233 List<TestAcknowledgmentEntity> testAcks, 234 List<String> emailAddresses, 235 List<Message> messages) 236 throws IOException { 237 if (testRuns.size() == 0) return null; 238 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 239 240 TestRunEntity mostRecentRun = null; 241 Map<String, TestCaseResult> mostRecentTestCaseResults = new HashMap<>(); 242 Map<String, TestCase> testCaseBreakageMap = new HashMap<>(); 243 int passingTestcaseCount = 0; 244 List<TestCaseReference> failingTestCases = new ArrayList<>(); 245 Set<String> fixedTestcases = new HashSet<>(); 246 Set<String> newTestcaseFailures = new HashSet<>(); 247 Set<String> continuedTestcaseFailures = new HashSet<>(); 248 Set<String> skippedTestcaseFailures = new HashSet<>(); 249 Set<String> transientTestcaseFailures = new HashSet<>(); 250 251 for (TestRunEntity testRun : testRuns) { 252 if (mostRecentRun == null) { 253 mostRecentRun = testRun; 254 } 255 List<Key> testCaseKeys = new ArrayList<>(); 256 for (long testCaseId : testRun.testCaseIds) { 257 testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); 258 } 259 Map<Key, Entity> entityMap = datastore.get(testCaseKeys); 260 for (Key testCaseKey : testCaseKeys) { 261 if (!entityMap.containsKey(testCaseKey)) { 262 logger.log(Level.WARNING, "Test case entity missing: " + testCaseKey); 263 continue; 264 } 265 Entity testCaseRun = entityMap.get(testCaseKey); 266 TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); 267 if (testCaseRunEntity == null) { 268 logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey()); 269 continue; 270 } 271 for (TestCase testCase : testCaseRunEntity.testCases) { 272 String testCaseName = testCase.name; 273 TestCaseResult result = TestCaseResult.valueOf(testCase.result); 274 275 if (mostRecentRun == testRun) { 276 mostRecentTestCaseResults.put(testCaseName, result); 277 } else { 278 if (!mostRecentTestCaseResults.containsKey(testCaseName)) { 279 // Deprecate notifications for tests that are not present on newer runs 280 continue; 281 } 282 TestCaseResult mostRecentRes = mostRecentTestCaseResults.get(testCaseName); 283 if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) { 284 mostRecentTestCaseResults.put(testCaseName, result); 285 } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) { 286 // Test is passing now, witnessed a transient failure 287 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 288 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 289 transientTestcaseFailures.add(testCaseName); 290 } 291 } 292 } 293 294 // Record test case breakages 295 if (result != TestCaseResult.TEST_CASE_RESULT_PASS 296 && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { 297 testCaseBreakageMap.put(testCaseName, testCase); 298 } 299 } 300 } 301 } 302 303 Set<String> buildIdList = new HashSet<>(); 304 List<DeviceInfoEntity> devices = new ArrayList<>(); 305 Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.key); 306 for (Entity device : datastore.prepare(deviceQuery).asIterable()) { 307 DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); 308 if (deviceEntity == null) { 309 continue; 310 } 311 buildIdList.add(deviceEntity.buildId); 312 devices.add(deviceEntity); 313 } 314 String footer = EmailHelper.getEmailFooter(mostRecentRun, devices, link); 315 String buildId = StringUtils.join(buildIdList, ","); 316 317 for (String testCaseName : mostRecentTestCaseResults.keySet()) { 318 TestCaseResult mostRecentResult = mostRecentTestCaseResults.get(testCaseName); 319 boolean previouslyFailed = failedTestCaseMap.containsKey(testCaseName); 320 if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) { 321 // persist previous status 322 if (previouslyFailed) { 323 skippedTestcaseFailures.add(testCaseName); 324 failingTestCases.add( 325 new TestCaseReference(failedTestCaseMap.get(testCaseName))); 326 } else { 327 ++passingTestcaseCount; 328 } 329 } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) { 330 ++passingTestcaseCount; 331 if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) { 332 fixedTestcases.add(testCaseName); 333 } 334 } else { 335 if (!previouslyFailed) { 336 newTestcaseFailures.add(testCaseName); 337 failingTestCases.add( 338 new TestCaseReference(testCaseBreakageMap.get(testCaseName))); 339 } else { 340 continuedTestcaseFailures.add(testCaseName); 341 failingTestCases.add( 342 new TestCaseReference(failedTestCaseMap.get(testCaseName))); 343 } 344 } 345 } 346 347 Set<String> acknowledgedFailures = 348 separateAcknowledged(newTestcaseFailures, devices, testAcks); 349 acknowledgedFailures.addAll( 350 separateAcknowledged(transientTestcaseFailures, devices, testAcks)); 351 acknowledgedFailures.addAll( 352 separateAcknowledged(continuedTestcaseFailures, devices, testAcks)); 353 354 String summary = new String(); 355 if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) { 356 summary += "The following test cases failed in the latest test run:<br>"; 357 358 // Add new test case failures to top of summary in bold font. 359 List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures); 360 sortedNewTestcaseFailures.sort(Comparator.naturalOrder()); 361 for (String testcaseName : sortedNewTestcaseFailures) { 362 summary += "- " + "<b>" + testcaseName + "</b><br>"; 363 } 364 365 // Add continued test case failures to summary. 366 List<String> sortedContinuedTestcaseFailures = 367 new ArrayList<>(continuedTestcaseFailures); 368 sortedContinuedTestcaseFailures.sort(Comparator.naturalOrder()); 369 for (String testcaseName : sortedContinuedTestcaseFailures) { 370 summary += "- " + testcaseName + "<br>"; 371 } 372 } 373 if (fixedTestcases.size() > 0) { 374 // Add fixed test cases to summary. 375 summary += "<br><br>The following test cases were fixed in the latest test run:<br>"; 376 List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases); 377 sortedFixedTestcases.sort(Comparator.naturalOrder()); 378 for (String testcaseName : sortedFixedTestcases) { 379 summary += "- <i>" + testcaseName + "</i><br>"; 380 } 381 } 382 if (transientTestcaseFailures.size() > 0) { 383 // Add transient test case failures to summary. 384 summary += "<br><br>The following transient test case failures occured:<br>"; 385 List<String> sortedTransientTestcaseFailures = 386 new ArrayList<>(transientTestcaseFailures); 387 sortedTransientTestcaseFailures.sort(Comparator.naturalOrder()); 388 for (String testcaseName : sortedTransientTestcaseFailures) { 389 summary += "- " + testcaseName + "<br>"; 390 } 391 } 392 if (skippedTestcaseFailures.size() > 0) { 393 // Add skipped test case failures to summary. 394 summary += "<br><br>The following test cases have not been run since failing:<br>"; 395 List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures); 396 sortedSkippedTestcaseFailures.sort(Comparator.naturalOrder()); 397 for (String testcaseName : sortedSkippedTestcaseFailures) { 398 summary += "- " + testcaseName + "<br>"; 399 } 400 } 401 if (acknowledgedFailures.size() > 0) { 402 // Add acknowledged test case failures to summary. 403 List<String> sortedAcknowledgedFailures = new ArrayList<>(acknowledgedFailures); 404 sortedAcknowledgedFailures.sort(Comparator.naturalOrder()); 405 if (acknowledgedFailures.size() > 0) { 406 summary += 407 "<br><br>The following acknowledged test case failures continued to fail:<br>"; 408 for (String testcaseName : sortedAcknowledgedFailures) { 409 summary += "- " + testcaseName + "<br>"; 410 } 411 } 412 } 413 414 String testName = mostRecentRun.key.getParent().getName(); 415 String uploadDateString = TimeUtil.getDateString(mostRecentRun.startTimestamp); 416 String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString; 417 if (newTestcaseFailures.size() > 0) { 418 String body = 419 "Hello,<br><br>New test case failure(s) in " 420 + testName 421 + " for device build ID(s): " 422 + buildId 423 + ".<br><br>" 424 + summary 425 + footer; 426 try { 427 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 428 } catch (MessagingException | UnsupportedEncodingException e) { 429 logger.log(Level.WARNING, "Error composing email : ", e); 430 } 431 } else if (continuedTestcaseFailures.size() > 0) { 432 String body = 433 "Hello,<br><br>Continuous test case failure(s) in " 434 + testName 435 + " for device build ID(s): " 436 + buildId 437 + ".<br><br>" 438 + summary 439 + footer; 440 try { 441 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 442 } catch (MessagingException | UnsupportedEncodingException e) { 443 logger.log(Level.WARNING, "Error composing email : ", e); 444 } 445 } else if (transientTestcaseFailures.size() > 0) { 446 String body = 447 "Hello,<br><br>Transient test case failure(s) in " 448 + testName 449 + " but tests all " 450 + "are passing in the latest device build(s): " 451 + buildId 452 + ".<br><br>" 453 + summary 454 + footer; 455 try { 456 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 457 } catch (MessagingException | UnsupportedEncodingException e) { 458 logger.log(Level.WARNING, "Error composing email : ", e); 459 } 460 } else if (fixedTestcases.size() > 0) { 461 String body = 462 "Hello,<br><br>All test cases passed in " 463 + testName 464 + " for device build ID(s): " 465 + buildId 466 + "!<br><br>" 467 + summary 468 + footer; 469 try { 470 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); 471 } catch (MessagingException | UnsupportedEncodingException e) { 472 logger.log(Level.WARNING, "Error composing email : ", e); 473 } 474 } 475 return new TestStatusEntity( 476 testName, 477 mostRecentRun.startTimestamp, 478 passingTestcaseCount, 479 failingTestCases.size(), 480 failingTestCases); 481 } 482 483 /** 484 * Add a task to process test run data 485 * 486 * @param testRunKey The key of the test run whose data process. 487 */ 488 public static void addTask(Key testRunKey) { 489 Queue queue = QueueFactory.getDefaultQueue(); 490 String keyString = KeyFactory.keyToString(testRunKey); 491 queue.add( 492 TaskOptions.Builder.withUrl(ALERT_JOB_URL) 493 .param("runKey", keyString) 494 .method(TaskOptions.Method.POST)); 495 } 496 497 @Override 498 public void doPost(HttpServletRequest request, HttpServletResponse response) 499 throws IOException { 500 DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); 501 String runKeyString = request.getParameter("runKey"); 502 503 Key testRunKey; 504 try { 505 testRunKey = KeyFactory.stringToKey(runKeyString); 506 } catch (IllegalArgumentException e) { 507 logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); 508 return; 509 } 510 String testName = testRunKey.getParent().getName(); 511 512 TestStatusEntity status = null; 513 Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName); 514 try { 515 status = TestStatusEntity.fromEntity(datastore.get(statusKey)); 516 } catch (EntityNotFoundException e) { 517 // no existing status 518 } 519 if (status == null) { 520 status = new TestStatusEntity(testName); 521 } 522 if (status.timestamp >= testRunKey.getId()) { 523 // Another job has already updated the status first 524 return; 525 } 526 List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); 527 528 StringBuffer fullUrl = request.getRequestURL(); 529 String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); 530 String link = 531 baseUrl + "/show_tree?testName=" + testName + "&endTime=" + testRunKey.getId(); 532 533 List<Message> messageQueue = new ArrayList<>(); 534 Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status); 535 List<TestAcknowledgmentEntity> testAcks = 536 getTestCaseAcknowledgments(testRunKey.getParent()); 537 List<TestRunEntity> testRuns = 538 getTestRuns(testRunKey.getParent(), status.timestamp, testRunKey.getId()); 539 if (testRuns.size() == 0) return; 540 541 TestStatusEntity newStatus = 542 getTestStatus(testRuns, link, failedTestcaseMap, testAcks, emails, messageQueue); 543 if (newStatus == null) { 544 // No changes to status 545 return; 546 } 547 548 int retries = 0; 549 while (true) { 550 Transaction txn = datastore.beginTransaction(); 551 try { 552 try { 553 status = TestStatusEntity.fromEntity(datastore.get(statusKey)); 554 } catch (EntityNotFoundException e) { 555 // no status left 556 } 557 if (status == null || status.timestamp >= newStatus.timestamp) { 558 txn.rollback(); 559 } else { // This update is most recent. 560 datastore.put(newStatus.toEntity()); 561 txn.commit(); 562 EmailHelper.sendAll(messageQueue); 563 } 564 break; 565 } catch (ConcurrentModificationException 566 | DatastoreFailureException 567 | DatastoreTimeoutException e) { 568 logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey); 569 if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) { 570 logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey); 571 throw e; 572 } 573 } finally { 574 if (txn.isActive()) { 575 txn.rollback(); 576 } 577 } 578 } 579 } 580 } 581
