1 package org.testng.internal; 2 3 import java.lang.annotation.Annotation; 4 import java.lang.reflect.InvocationTargetException; 5 import java.lang.reflect.Method; 6 import java.util.Collection; 7 import java.util.Collections; 8 import java.util.Iterator; 9 import java.util.List; 10 import java.util.Map; 11 import java.util.Set; 12 13 import org.testng.IClass; 14 import org.testng.IClassListener; 15 import org.testng.IConfigurable; 16 import org.testng.IConfigurationListener; 17 import org.testng.IConfigurationListener2; 18 import org.testng.IHookable; 19 import org.testng.IInvokedMethod; 20 import org.testng.IInvokedMethodListener; 21 import org.testng.IRetryAnalyzer; 22 import org.testng.ITestClass; 23 import org.testng.ITestContext; 24 import org.testng.ITestListener; 25 import org.testng.ITestNGMethod; 26 import org.testng.ITestResult; 27 import org.testng.Reporter; 28 import org.testng.SkipException; 29 import org.testng.SuiteRunState; 30 import org.testng.TestException; 31 import org.testng.TestNGException; 32 import org.testng.annotations.IConfigurationAnnotation; 33 import org.testng.annotations.NoInjection; 34 import org.testng.collections.Lists; 35 import org.testng.collections.Maps; 36 import org.testng.collections.Sets; 37 import org.testng.internal.InvokeMethodRunnable.TestNGRuntimeException; 38 import org.testng.internal.ParameterHolder.ParameterOrigin; 39 import org.testng.internal.annotations.AnnotationHelper; 40 import org.testng.internal.annotations.IAnnotationFinder; 41 import org.testng.internal.invokers.InvokedMethodListenerInvoker; 42 import org.testng.internal.invokers.InvokedMethodListenerMethod; 43 import org.testng.internal.thread.ThreadExecutionException; 44 import org.testng.internal.thread.ThreadUtil; 45 import org.testng.internal.thread.graph.IWorker; 46 import org.testng.xml.XmlClass; 47 import org.testng.xml.XmlSuite; 48 import org.testng.xml.XmlTest; 49 50 import static org.testng.internal.invokers.InvokedMethodListenerMethod.AFTER_INVOCATION; 51 import static org.testng.internal.invokers.InvokedMethodListenerMethod.BEFORE_INVOCATION; 52 53 /** 54 * This class is responsible for invoking methods: 55 * - test methods 56 * - configuration methods 57 * - possibly in a separate thread 58 * and then for notifying the result listeners. 59 * 60 * @author <a href="mailto:cedric (at) beust.com">Cedric Beust</a> 61 * @author <a href='mailto:the_mindstorm (at) evolva.ro'>Alexandru Popescu</a> 62 */ 63 public class Invoker implements IInvoker { 64 private final ITestContext m_testContext; 65 private final ITestResultNotifier m_notifier; 66 private final IAnnotationFinder m_annotationFinder; 67 private final SuiteRunState m_suiteState; 68 private final boolean m_skipFailedInvocationCounts; 69 private final Collection<IInvokedMethodListener> m_invokedMethodListeners; 70 private final boolean m_continueOnFailedConfiguration; 71 private final List<IClassListener> m_classListeners; 72 73 /** Group failures must be synced as the Invoker is accessed concurrently */ 74 private Map<String, Boolean> m_beforegroupsFailures = Maps.newHashtable(); 75 76 /** Class failures must be synced as the Invoker is accessed concurrently */ 77 private Map<Class<?>, Set<Object>> m_classInvocationResults = Maps.newHashtable(); 78 79 /** Test methods whose configuration methods have failed. */ 80 private Map<ITestNGMethod, Set<Object>> m_methodInvocationResults = Maps.newHashtable(); 81 private IConfiguration m_configuration; 82 83 /** Predicate to filter methods */ 84 private static Predicate<ITestNGMethod, IClass> CAN_RUN_FROM_CLASS = new CanRunFromClassPredicate(); 85 /** Predicate to filter methods */ 86 private static final Predicate<ITestNGMethod, IClass> SAME_CLASS = new SameClassNamePredicate(); 87 88 private void setClassInvocationFailure(Class<?> clazz, Object instance) { 89 Set<Object> instances = m_classInvocationResults.get( clazz ); 90 if (instances == null) { 91 instances = Sets.newHashSet(); 92 m_classInvocationResults.put(clazz, instances); 93 } 94 instances.add(instance); 95 } 96 97 private void setMethodInvocationFailure(ITestNGMethod method, Object instance) { 98 Set<Object> instances = m_methodInvocationResults.get(method); 99 if (instances == null) { 100 instances = Sets.newHashSet(); 101 m_methodInvocationResults.put(method, instances); 102 } 103 instances.add(getMethodInvocationToken(method, instance)); 104 } 105 106 public Invoker(IConfiguration configuration, 107 ITestContext testContext, 108 ITestResultNotifier notifier, 109 SuiteRunState state, 110 boolean skipFailedInvocationCounts, 111 Collection<IInvokedMethodListener> invokedMethodListeners, 112 List<IClassListener> classListeners) { 113 m_configuration = configuration; 114 m_testContext= testContext; 115 m_suiteState= state; 116 m_notifier= notifier; 117 m_annotationFinder= configuration.getAnnotationFinder(); 118 m_skipFailedInvocationCounts = skipFailedInvocationCounts; 119 m_invokedMethodListeners = invokedMethodListeners; 120 m_continueOnFailedConfiguration = XmlSuite.CONTINUE.equals(testContext.getSuite().getXmlSuite().getConfigFailurePolicy()); 121 m_classListeners = classListeners; 122 } 123 124 /** 125 * Invoke configuration methods if they belong to the same TestClass passed 126 * in parameter.. <p/>TODO: Calculate ahead of time which methods should be 127 * invoked for each class. Might speed things up for users who invoke the 128 * same test class with different parameters in the same suite run. 129 * 130 * If instance is non-null, the configuration will be run on it. If it is null, 131 * the configuration methods will be run on all the instances retrieved 132 * from the ITestClass. 133 */ 134 @Override 135 public void invokeConfigurations(IClass testClass, 136 ITestNGMethod[] allMethods, 137 XmlSuite suite, 138 Map<String, String> params, 139 Object[] parameterValues, 140 Object instance) 141 { 142 invokeConfigurations(testClass, null, allMethods, suite, params, parameterValues, instance, 143 null); 144 } 145 146 private void invokeConfigurations(IClass testClass, 147 ITestNGMethod currentTestMethod, 148 ITestNGMethod[] allMethods, 149 XmlSuite suite, 150 Map<String, String> params, 151 Object[] parameterValues, 152 Object instance, 153 ITestResult testMethodResult) 154 { 155 if(null == allMethods) { 156 log(5, "No configuration methods found"); 157 158 return; 159 } 160 161 ITestNGMethod[] methods= filterMethods(testClass, allMethods, SAME_CLASS); 162 163 for(ITestNGMethod tm : methods) { 164 if(null == testClass) { 165 testClass= tm.getTestClass(); 166 } 167 168 ITestResult testResult= new TestResult(testClass, 169 instance, 170 tm, 171 null, 172 System.currentTimeMillis(), 173 System.currentTimeMillis(), 174 m_testContext); 175 176 IConfigurationAnnotation configurationAnnotation= null; 177 try { 178 Object inst = tm.getInstance(); 179 if (inst == null) { 180 inst = instance; 181 } 182 Class<?> objectClass= inst.getClass(); 183 Method method= tm.getMethod(); 184 185 // Only run the configuration if 186 // - the test is enabled and 187 // - the Configuration method belongs to the same class or a parent 188 configurationAnnotation = AnnotationHelper.findConfiguration(m_annotationFinder, method); 189 boolean alwaysRun= isAlwaysRun(configurationAnnotation); 190 if(MethodHelper.isEnabled(objectClass, m_annotationFinder) || alwaysRun) { 191 192 if (MethodHelper.isEnabled(configurationAnnotation)) { 193 194 if (!confInvocationPassed(tm, currentTestMethod, testClass, instance) && !alwaysRun) { 195 handleConfigurationSkip(tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 196 continue; 197 } 198 199 log(3, "Invoking " + Utils.detailedMethodName(tm, true)); 200 201 Object[] parameters = Parameters.createConfigurationParameters(tm.getMethod(), 202 params, 203 parameterValues, 204 currentTestMethod, 205 m_annotationFinder, 206 suite, 207 m_testContext, 208 testMethodResult); 209 testResult.setParameters(parameters); 210 211 Object newInstance = null != instance ? instance: inst; 212 213 runConfigurationListeners(testResult, true /* before */); 214 215 invokeConfigurationMethod(newInstance, tm, 216 parameters, testResult); 217 218 // TODO: probably we should trigger the event for each instance??? 219 testResult.setEndMillis(System.currentTimeMillis()); 220 runConfigurationListeners(testResult, false /* after */); 221 } 222 else { 223 log(3, 224 "Skipping " 225 + Utils.detailedMethodName(tm, true) 226 + " because it is not enabled"); 227 } 228 } // if is enabled 229 else { 230 log(3, 231 "Skipping " 232 + Utils.detailedMethodName(tm, true) 233 + " because " 234 + objectClass.getName() 235 + " is not enabled"); 236 } 237 } 238 catch(InvocationTargetException ex) { 239 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 240 } catch(Throwable ex) { // covers the non-wrapper exceptions 241 handleConfigurationFailure(ex, tm, testResult, configurationAnnotation, currentTestMethod, instance, suite); 242 } 243 } // for methods 244 } 245 246 /** 247 * Marks the current <code>TestResult</code> as skipped and invokes the listeners. 248 */ 249 private void handleConfigurationSkip(ITestNGMethod tm, 250 ITestResult testResult, 251 IConfigurationAnnotation annotation, 252 ITestNGMethod currentTestMethod, 253 Object instance, 254 XmlSuite suite) { 255 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 256 testResult.setStatus(ITestResult.SKIP); 257 runConfigurationListeners(testResult, false /* after */); 258 } 259 260 /** 261 * Is the <code>IConfiguration</code> marked as alwaysRun. 262 */ 263 private boolean isAlwaysRun(IConfigurationAnnotation configurationAnnotation) { 264 if(null == configurationAnnotation) { 265 return false; 266 } 267 268 boolean alwaysRun= false; 269 if ((configurationAnnotation.getAfterSuite() 270 || configurationAnnotation.getAfterTest() 271 || configurationAnnotation.getAfterTestClass() 272 || configurationAnnotation.getAfterTestMethod() 273 || configurationAnnotation.getBeforeTestMethod() 274 || configurationAnnotation.getBeforeTestClass() 275 || configurationAnnotation.getBeforeTest() 276 || configurationAnnotation.getBeforeSuite()) 277 && configurationAnnotation.getAlwaysRun()) 278 { 279 alwaysRun= true; 280 } 281 282 return alwaysRun; 283 } 284 285 private void handleConfigurationFailure(Throwable ite, 286 ITestNGMethod tm, 287 ITestResult testResult, 288 IConfigurationAnnotation annotation, 289 ITestNGMethod currentTestMethod, 290 Object instance, 291 XmlSuite suite) 292 { 293 Throwable cause= ite.getCause() != null ? ite.getCause() : ite; 294 295 if(isSkipExceptionAndSkip(cause)) { 296 testResult.setThrowable(cause); 297 handleConfigurationSkip(tm, testResult, annotation, currentTestMethod, instance, suite); 298 return; 299 } 300 Utils.log("", 3, "Failed to invoke configuration method " 301 + tm.getRealClass().getName() + "." + tm.getMethodName() + ":" + cause.getMessage()); 302 handleException(cause, tm, testResult, 1); 303 runConfigurationListeners(testResult, false /* after */); 304 305 // 306 // If in TestNG mode, need to take a look at the annotation to figure out 307 // what kind of @Configuration method we're dealing with 308 // 309 if (null != annotation) { 310 recordConfigurationInvocationFailed(tm, testResult.getTestClass(), annotation, currentTestMethod, instance, suite); 311 } 312 } 313 314 /** 315 * @return All the classes that belong to the same <test> tag as @param cls 316 */ 317 private XmlClass[] findClassesInSameTest(Class<?> cls, XmlSuite suite) { 318 Map<String, XmlClass> vResult= Maps.newHashMap(); 319 String className= cls.getName(); 320 for(XmlTest test : suite.getTests()) { 321 for(XmlClass testClass : test.getXmlClasses()) { 322 if(testClass.getName().equals(className)) { 323 324 // Found it, add all the classes in this test in the result 325 for(XmlClass thisClass : test.getXmlClasses()) { 326 vResult.put(thisClass.getName(), thisClass); 327 } 328 // Note: we need to iterate through the entire suite since the same 329 // class might appear in several <test> tags 330 } 331 } 332 } 333 334 XmlClass[] result= vResult.values().toArray(new XmlClass[vResult.size()]); 335 336 return result; 337 } 338 339 /** 340 * Record internally the failure of a Configuration, so that we can determine 341 * later if @Test should be skipped. 342 */ 343 private void recordConfigurationInvocationFailed(ITestNGMethod tm, 344 IClass testClass, 345 IConfigurationAnnotation annotation, 346 ITestNGMethod currentTestMethod, 347 Object instance, 348 XmlSuite suite) { 349 // If beforeTestClass or afterTestClass failed, mark either the config method's 350 // entire class as failed, or the class under tests as failed, depending on 351 // the configuration failure policy 352 if (annotation.getBeforeTestClass() || annotation.getAfterTestClass()) { 353 // tm is the configuration method, and currentTestMethod is null for BeforeClass 354 // methods, so we need testClass 355 if (m_continueOnFailedConfiguration) { 356 setClassInvocationFailure(testClass.getRealClass(), instance); 357 } else { 358 setClassInvocationFailure(tm.getRealClass(), instance); 359 } 360 } 361 362 // If before/afterTestMethod failed, mark either the config method's entire 363 // class as failed, or just the current test method as failed, depending on 364 // the configuration failure policy 365 else if (annotation.getBeforeTestMethod() || annotation.getAfterTestMethod()) { 366 if (m_continueOnFailedConfiguration) { 367 setMethodInvocationFailure(currentTestMethod, instance); 368 } else { 369 setClassInvocationFailure(tm.getRealClass(), instance); 370 } 371 } 372 373 // If beforeSuite or afterSuite failed, mark *all* the classes as failed 374 // for configurations. At this point, the entire Suite is screwed 375 else if (annotation.getBeforeSuite() || annotation.getAfterSuite()) { 376 m_suiteState.failed(); 377 } 378 379 // beforeTest or afterTest: mark all the classes in the same 380 // <test> stanza as failed for configuration 381 else if (annotation.getBeforeTest() || annotation.getAfterTest()) { 382 setClassInvocationFailure(tm.getRealClass(), instance); 383 XmlClass[] classes= findClassesInSameTest(tm.getRealClass(), suite); 384 for(XmlClass xmlClass : classes) { 385 setClassInvocationFailure(xmlClass.getSupportClass(), instance); 386 } 387 } 388 String[] beforeGroups= annotation.getBeforeGroups(); 389 if(null != beforeGroups && beforeGroups.length > 0) { 390 for(String group: beforeGroups) { 391 m_beforegroupsFailures.put(group, Boolean.FALSE); 392 } 393 } 394 } 395 396 /** 397 * @return true if this class or a parent class failed to initialize. 398 */ 399 private boolean classConfigurationFailed(Class<?> cls) { 400 for (Class<?> c : m_classInvocationResults.keySet()) { 401 if (c == cls || c.isAssignableFrom(cls)) { 402 return true; 403 } 404 } 405 return false; 406 } 407 408 /** 409 * @return true if this class has successfully run all its @Configuration 410 * method or false if at least one of these methods failed. 411 */ 412 private boolean confInvocationPassed(ITestNGMethod method, ITestNGMethod currentTestMethod, 413 IClass testClass, Object instance) { 414 boolean result= true; 415 416 Class<?> cls = testClass.getRealClass(); 417 418 if(m_suiteState.isFailed()) { 419 result= false; 420 } 421 else { 422 if (classConfigurationFailed(cls)) { 423 if (! m_continueOnFailedConfiguration) { 424 result = !classConfigurationFailed(cls); 425 } else { 426 result = !m_classInvocationResults.get(cls).contains(instance); 427 } 428 } 429 // if method is BeforeClass, currentTestMethod will be null 430 else if (m_continueOnFailedConfiguration && 431 currentTestMethod != null && 432 m_methodInvocationResults.containsKey(currentTestMethod)) { 433 result = !m_methodInvocationResults.get(currentTestMethod).contains(getMethodInvocationToken(currentTestMethod, instance)); 434 } 435 else if (! m_continueOnFailedConfiguration) { 436 for(Class<?> clazz: m_classInvocationResults.keySet()) { 437 // if (clazz == cls) { 438 if(clazz.isAssignableFrom(cls)) { 439 result= false; 440 break; 441 } 442 } 443 } 444 } 445 446 // check if there are failed @BeforeGroups 447 String[] groups= method.getGroups(); 448 if(null != groups && groups.length > 0) { 449 for(String group: groups) { 450 if(m_beforegroupsFailures.containsKey(group)) { 451 result= false; 452 break; 453 } 454 } 455 } 456 return result; 457 } 458 459 // Creates a token for tracking a unique invocation of a method on an instance. 460 // Is used when configFailurePolicy=continue. 461 private Object getMethodInvocationToken(ITestNGMethod method, Object instance) { 462 return String.format("%s+%d", instance.toString(), method.getCurrentInvocationCount()); 463 } 464 465 /** 466 * Effectively invokes a configuration method on all passed in instances. 467 * TODO: Should change this method to be more like invokeMethod() so that we can 468 * handle calls to {@code IInvokedMethodListener} better. 469 * 470 * @param targetInstance the instance to invoke the configuration method on 471 * @param tm the configuration method 472 * @param params the parameters needed for method invocation 473 * @param testResult 474 * @throws InvocationTargetException 475 * @throws IllegalAccessException 476 */ 477 private void invokeConfigurationMethod(Object targetInstance, 478 ITestNGMethod tm, 479 Object[] params, 480 ITestResult testResult) 481 throws InvocationTargetException, IllegalAccessException 482 { 483 // Mark this method with the current thread id 484 tm.setId(ThreadUtil.currentThreadInfo()); 485 486 { 487 InvokedMethod invokedMethod= new InvokedMethod(targetInstance, 488 tm, 489 params, 490 System.currentTimeMillis(), 491 testResult); 492 493 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 494 m_notifier.addInvokedMethod(invokedMethod); 495 try { 496 Reporter.setCurrentTestResult(testResult); 497 Method method = tm.getMethod(); 498 499 // 500 // If this method is a IConfigurable, invoke its run() method 501 // 502 IConfigurable configurableInstance = 503 IConfigurable.class.isAssignableFrom(tm.getMethod().getDeclaringClass()) ? 504 (IConfigurable) targetInstance : m_configuration.getConfigurable(); 505 if (configurableInstance != null) { 506 MethodInvocationHelper.invokeConfigurable(targetInstance, params, configurableInstance, method, 507 testResult); 508 } 509 else { 510 // 511 // Not a IConfigurable, invoke directly 512 // 513 if (MethodHelper.calculateTimeOut(tm) <= 0) { 514 MethodInvocationHelper.invokeMethod(method, targetInstance, params); 515 } 516 else { 517 MethodInvocationHelper.invokeWithTimeout(tm, targetInstance, params, testResult); 518 if (!testResult.isSuccess()) { 519 // A time out happened 520 throwConfigurationFailure(testResult, testResult.getThrowable()); 521 throw testResult.getThrowable(); 522 } 523 } 524 } 525 } 526 catch (InvocationTargetException | IllegalAccessException ex) { 527 throwConfigurationFailure(testResult, ex); 528 throw ex; 529 } catch (Throwable ex) { 530 throwConfigurationFailure(testResult, ex); 531 throw new TestNGException(ex); 532 } 533 finally { 534 Reporter.setCurrentTestResult(testResult); 535 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 536 Reporter.setCurrentTestResult(null); 537 } 538 } 539 } 540 541 private void throwConfigurationFailure(ITestResult testResult, Throwable ex) 542 { 543 testResult.setStatus(ITestResult.FAILURE);; 544 testResult.setThrowable(ex.getCause() == null ? ex : ex.getCause()); 545 } 546 547 private void runInvokedMethodListeners(InvokedMethodListenerMethod listenerMethod, IInvokedMethod invokedMethod, 548 ITestResult testResult) 549 { 550 if ( noListenersPresent() ) { 551 return; 552 } 553 554 InvokedMethodListenerInvoker invoker = new InvokedMethodListenerInvoker(listenerMethod, testResult, m_testContext); 555 for (IInvokedMethodListener currentListener : m_invokedMethodListeners) { 556 invoker.invokeListener(currentListener, invokedMethod); 557 } 558 } 559 560 private boolean noListenersPresent() { 561 return (m_invokedMethodListeners == null) || (m_invokedMethodListeners.size() == 0); 562 } 563 564 // pass both paramValues and paramIndex to be thread safe in case parallel=true + dataprovider. 565 private ITestResult invokeMethod(Object instance, 566 final ITestNGMethod tm, 567 Object[] parameterValues, 568 int parametersIndex, 569 XmlSuite suite, 570 Map<String, String> params, 571 ITestClass testClass, 572 ITestNGMethod[] beforeMethods, 573 ITestNGMethod[] afterMethods, 574 ConfigurationGroupMethods groupMethods, 575 FailureContext failureContext) { 576 TestResult testResult = new TestResult(); 577 578 // 579 // Invoke beforeGroups configurations 580 // 581 invokeBeforeGroupsConfigurations(testClass, tm, groupMethods, suite, params, 582 instance); 583 584 // 585 // Invoke beforeMethods only if 586 // - firstTimeOnly is not set 587 // - firstTimeOnly is set, and we are reaching at the first invocationCount 588 // 589 invokeConfigurations(testClass, tm, 590 filterConfigurationMethods(tm, beforeMethods, true /* beforeMethods */), 591 suite, params, parameterValues, 592 instance, testResult); 593 594 // 595 // Create the ExtraOutput for this method 596 // 597 InvokedMethod invokedMethod = null; 598 try { 599 testResult.init(testClass, instance, 600 tm, 601 null, 602 System.currentTimeMillis(), 603 0, 604 m_testContext); 605 testResult.setParameters(parameterValues); 606 testResult.setHost(m_testContext.getHost()); 607 testResult.setStatus(ITestResult.STARTED); 608 609 invokedMethod= new InvokedMethod(instance, 610 tm, 611 parameterValues, 612 System.currentTimeMillis(), 613 testResult); 614 615 // Fix from ansgarkonermann 616 // invokedMethod is used in the finally, which can be invoked if 617 // any of the test listeners throws an exception, therefore, 618 // invokedMethod must have a value before we get here 619 runTestListeners(testResult); 620 621 runInvokedMethodListeners(BEFORE_INVOCATION, invokedMethod, testResult); 622 623 m_notifier.addInvokedMethod(invokedMethod); 624 625 Method thisMethod = tm.getConstructorOrMethod().getMethod(); 626 627 if(confInvocationPassed(tm, tm, testClass, instance)) { 628 log(3, "Invoking " + tm.getRealClass().getName() + "." + tm.getMethodName()); 629 630 Reporter.setCurrentTestResult(testResult); 631 632 // If this method is a IHookable, invoke its run() method 633 IHookable hookableInstance = 634 IHookable.class.isAssignableFrom(tm.getRealClass()) ? 635 (IHookable) instance : m_configuration.getHookable(); 636 637 if (MethodHelper.calculateTimeOut(tm) <= 0) { 638 if (hookableInstance != null) { 639 MethodInvocationHelper.invokeHookable(instance, 640 parameterValues, hookableInstance, thisMethod, testResult); 641 } else { 642 // Not a IHookable, invoke directly 643 MethodInvocationHelper.invokeMethod(thisMethod, instance, 644 parameterValues); 645 } 646 testResult.setStatus(ITestResult.SUCCESS); 647 } else { 648 // Method with a timeout 649 MethodInvocationHelper.invokeWithTimeout(tm, instance, parameterValues, testResult, hookableInstance); 650 } 651 } 652 else { 653 testResult.setStatus(ITestResult.SKIP); 654 } 655 } 656 catch(InvocationTargetException ite) { 657 testResult.setThrowable(ite.getCause()); 658 testResult.setStatus(ITestResult.FAILURE); 659 } 660 catch(ThreadExecutionException tee) { // wrapper for TestNGRuntimeException 661 Throwable cause= tee.getCause(); 662 if(TestNGRuntimeException.class.equals(cause.getClass())) { 663 testResult.setThrowable(cause.getCause()); 664 } 665 else { 666 testResult.setThrowable(cause); 667 } 668 testResult.setStatus(ITestResult.FAILURE); 669 } 670 catch(Throwable thr) { // covers the non-wrapper exceptions 671 testResult.setThrowable(thr); 672 testResult.setStatus(ITestResult.FAILURE); 673 } 674 finally { 675 // Set end time ASAP 676 testResult.setEndMillis(System.currentTimeMillis()); 677 678 ExpectedExceptionsHolder expectedExceptionClasses 679 = new ExpectedExceptionsHolder(m_annotationFinder, tm, new RegexpExpectedExceptionsHolder(m_annotationFinder, tm)); 680 List<ITestResult> results = Lists.<ITestResult>newArrayList(testResult); 681 handleInvocationResults(tm, results, expectedExceptionClasses, failureContext); 682 683 // If this method has a data provider and just failed, memorize the number 684 // at which it failed. 685 // Note: we're not exactly testing that this method has a data provider, just 686 // that it has parameters, so might have to revisit this if bugs get reported 687 // for the case where this method has parameters that don't come from a data 688 // provider 689 if (testResult.getThrowable() != null && parameterValues.length > 0) { 690 tm.addFailedInvocationNumber(parametersIndex); 691 } 692 693 // 694 // Increment the invocation count for this method 695 // 696 tm.incrementCurrentInvocationCount(); 697 698 // Run invokedMethodListeners after updating TestResult 699 runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult); 700 runTestListeners(testResult); 701 702 // 703 // Invoke afterMethods only if 704 // - lastTimeOnly is not set 705 // - lastTimeOnly is set, and we are reaching the last invocationCount 706 // 707 invokeConfigurations(testClass, tm, 708 filterConfigurationMethods(tm, afterMethods, false /* beforeMethods */), 709 suite, params, parameterValues, 710 instance, 711 testResult); 712 713 // 714 // Invoke afterGroups configurations 715 // 716 invokeAfterGroupsConfigurations(testClass, tm, groupMethods, suite, 717 params, instance); 718 719 // Reset the test result last. If we do this too early, Reporter.log() 720 // invocations from listeners will be discarded 721 Reporter.setCurrentTestResult(null); 722 } 723 724 return testResult; 725 } 726 727 void collectResults(ITestNGMethod testMethod, Collection<ITestResult> results) { 728 for (ITestResult result : results) { 729 // Collect the results 730 final int status = result.getStatus(); 731 if(ITestResult.SUCCESS == status) { 732 m_notifier.addPassedTest(testMethod, result); 733 } 734 else if(ITestResult.SKIP == status) { 735 m_notifier.addSkippedTest(testMethod, result); 736 } 737 else if(ITestResult.FAILURE == status) { 738 m_notifier.addFailedTest(testMethod, result); 739 } 740 else if(ITestResult.SUCCESS_PERCENTAGE_FAILURE == status) { 741 m_notifier.addFailedButWithinSuccessPercentageTest(testMethod, result); 742 } 743 else { 744 assert false : "UNKNOWN STATUS:" + status; 745 } 746 } 747 } 748 749 /** 750 * The array of methods contains @BeforeMethods if isBefore if true, @AfterMethods 751 * otherwise. This function removes all the methods that should not be run at this 752 * point because they are either firstTimeOnly or lastTimeOnly and we haven't reached 753 * the current invocationCount yet 754 */ 755 private ITestNGMethod[] filterConfigurationMethods(ITestNGMethod tm, 756 ITestNGMethod[] methods, boolean isBefore) 757 { 758 List<ITestNGMethod> result = Lists.newArrayList(); 759 for (ITestNGMethod m : methods) { 760 ConfigurationMethod cm = (ConfigurationMethod) m; 761 if (isBefore) { 762 if (! cm.isFirstTimeOnly() || 763 (cm.isFirstTimeOnly() && tm.getCurrentInvocationCount() == 0)) 764 { 765 result.add(m); 766 } 767 } 768 else { 769 int current = tm.getCurrentInvocationCount(); 770 boolean isLast = false; 771 // If we have parameters, set the boolean if we are about to run 772 // the last invocation 773 if (tm.getParameterInvocationCount() > 0) { 774 isLast = current == tm.getParameterInvocationCount() * tm.getTotalInvocationCount(); 775 } 776 // If we have invocationCount > 1, set the boolean if we are about to 777 // run the last invocation 778 else if (tm.getTotalInvocationCount() > 1) { 779 isLast = current == tm.getTotalInvocationCount(); 780 } 781 if (! cm.isLastTimeOnly() || (cm.isLastTimeOnly() && isLast)) { 782 result.add(m); 783 } 784 } 785 } 786 787 return result.toArray(new ITestNGMethod[result.size()]); 788 } 789 790 /** 791 * invokeTestMethods() eventually converge here to invoke a single @Test method. 792 * <p/> 793 * This method is responsible for actually invoking the method. It decides if the invocation 794 * must be done: 795 * <ul> 796 * <li>through an <code>IHookable</code></li> 797 * <li>directly (through reflection)</li> 798 * <li>in a separate thread (in case it needs to timeout) 799 * </ul> 800 * 801 * <p/> 802 * This method is also responsible for invoking @BeforeGroup, @BeforeMethod, @AfterMethod, @AfterGroup 803 * if it is the case for the passed in @Test method. 804 */ 805 protected ITestResult invokeTestMethod(Object instance, 806 final ITestNGMethod tm, 807 Object[] parameterValues, 808 int parametersIndex, 809 XmlSuite suite, 810 Map<String, String> params, 811 ITestClass testClass, 812 ITestNGMethod[] beforeMethods, 813 ITestNGMethod[] afterMethods, 814 ConfigurationGroupMethods groupMethods, 815 FailureContext failureContext) 816 { 817 // Mark this method with the current thread id 818 tm.setId(ThreadUtil.currentThreadInfo()); 819 820 ITestResult result = invokeMethod(instance, tm, parameterValues, parametersIndex, suite, params, 821 testClass, beforeMethods, afterMethods, groupMethods, 822 failureContext); 823 824 return result; 825 } 826 827 /** 828 * Filter all the beforeGroups methods and invoke only those that apply 829 * to the current test method 830 */ 831 private void invokeBeforeGroupsConfigurations(ITestClass testClass, 832 ITestNGMethod currentTestMethod, 833 ConfigurationGroupMethods groupMethods, 834 XmlSuite suite, 835 Map<String, String> params, 836 Object instance) 837 { 838 synchronized(groupMethods) { 839 List<ITestNGMethod> filteredMethods = Lists.newArrayList(); 840 String[] groups = currentTestMethod.getGroups(); 841 Map<String, List<ITestNGMethod>> beforeGroupMap = groupMethods.getBeforeGroupsMap(); 842 843 for (String group : groups) { 844 List<ITestNGMethod> methods = beforeGroupMap.get(group); 845 if (methods != null) { 846 filteredMethods.addAll(methods); 847 } 848 } 849 850 ITestNGMethod[] beforeMethodsArray = filteredMethods.toArray(new ITestNGMethod[filteredMethods.size()]); 851 // 852 // Invoke the right groups methods 853 // 854 if(beforeMethodsArray.length > 0) { 855 // don't pass the IClass or the instance as the method may be external 856 // the invocation must be similar to @BeforeTest/@BeforeSuite 857 invokeConfigurations(null, beforeMethodsArray, suite, params, 858 null, /* no parameter values */ 859 null); 860 } 861 862 // 863 // Remove them so they don't get run again 864 // 865 groupMethods.removeBeforeGroups(groups); 866 } 867 } 868 869 private void invokeAfterGroupsConfigurations(ITestClass testClass, 870 ITestNGMethod currentTestMethod, 871 ConfigurationGroupMethods groupMethods, 872 XmlSuite suite, 873 Map<String, String> params, 874 Object instance) 875 { 876 // Skip this if the current method doesn't belong to any group 877 // (only a method that belongs to a group can trigger the invocation 878 // of afterGroups methods) 879 if (currentTestMethod.getGroups().length == 0) { 880 return; 881 } 882 883 // See if the currentMethod is the last method in any of the groups 884 // it belongs to 885 Map<String, String> filteredGroups = Maps.newHashMap(); 886 String[] groups = currentTestMethod.getGroups(); 887 synchronized(groupMethods) { 888 for (String group : groups) { 889 if (groupMethods.isLastMethodForGroup(group, currentTestMethod)) { 890 filteredGroups.put(group, group); 891 } 892 } 893 894 if(filteredGroups.isEmpty()) { 895 return; 896 } 897 898 // The list of afterMethods to run 899 Map<ITestNGMethod, ITestNGMethod> afterMethods = Maps.newHashMap(); 900 901 // Now filteredGroups contains all the groups for which we need to run the afterGroups 902 // method. Find all the methods that correspond to these groups and invoke them. 903 Map<String, List<ITestNGMethod>> map = groupMethods.getAfterGroupsMap(); 904 for (String g : filteredGroups.values()) { 905 List<ITestNGMethod> methods = map.get(g); 906 // Note: should put them in a map if we want to make sure the same afterGroups 907 // doesn't get run twice 908 if (methods != null) { 909 for (ITestNGMethod m : methods) { 910 afterMethods.put(m, m); 911 } 912 } 913 } 914 915 // Got our afterMethods, invoke them 916 ITestNGMethod[] afterMethodsArray = afterMethods.keySet().toArray(new ITestNGMethod[afterMethods.size()]); 917 // don't pass the IClass or the instance as the method may be external 918 // the invocation must be similar to @BeforeTest/@BeforeSuite 919 invokeConfigurations(null, afterMethodsArray, suite, params, 920 null, /* no parameter values */ 921 null); 922 923 // Remove the groups so they don't get run again 924 groupMethods.removeAfterGroups(filteredGroups.keySet()); 925 } 926 } 927 928 private Object[] getParametersFromIndex(Iterator<Object[]> parametersValues, int index) { 929 while (parametersValues.hasNext()) { 930 Object[] parameters = parametersValues.next(); 931 932 if (index == 0) { 933 return parameters; 934 } 935 index--; 936 } 937 return null; 938 } 939 940 int retryFailed(Object instance, 941 final ITestNGMethod tm, 942 XmlSuite suite, 943 ITestClass testClass, 944 ITestNGMethod[] beforeMethods, 945 ITestNGMethod[] afterMethods, 946 ConfigurationGroupMethods groupMethods, 947 List<ITestResult> result, 948 int failureCount, 949 ExpectedExceptionsHolder expectedExceptionHolder, 950 ITestContext testContext, 951 Map<String, String> parameters, 952 int parametersIndex) { 953 final FailureContext failure = new FailureContext(); 954 failure.count = failureCount; 955 do { 956 failure.instances = Lists.newArrayList (); 957 Map<String, String> allParameters = Maps.newHashMap(); 958 /** 959 * TODO: This recreates all the parameters every time when we only need 960 * one specific set. Should optimize it by only recreating the set needed. 961 */ 962 ParameterBag bag = createParameters(tm, parameters, 963 allParameters, suite, testContext, null /* fedInstance */); 964 Object[] parameterValues = 965 getParametersFromIndex(bag.parameterHolder.parameters, parametersIndex); 966 967 result.add(invokeMethod(instance, tm, parameterValues, parametersIndex, suite, 968 allParameters, testClass, beforeMethods, afterMethods, groupMethods, failure)); 969 } 970 while (!failure.instances.isEmpty()); 971 return failure.count; 972 } 973 974 private ParameterBag createParameters(ITestNGMethod testMethod, 975 Map<String, String> parameters, 976 Map<String, String> allParameterNames, 977 XmlSuite suite, 978 ITestContext testContext, 979 Object fedInstance) 980 { 981 Object instance; 982 if (fedInstance != null) { 983 instance = fedInstance; 984 } 985 else { 986 instance = testMethod.getInstance(); 987 } 988 989 ParameterBag bag = handleParameters(testMethod, 990 instance, allParameterNames, parameters, null, suite, testContext, fedInstance, null); 991 992 return bag; 993 } 994 995 /** 996 * Invoke all the test methods. Note the plural: the method passed in 997 * parameter might be invoked several times if the test class it belongs 998 * to has more than one instance (i.e., if an @Factory method has been 999 * declared somewhere that returns several instances of this TestClass). 1000 * If no @Factory method was specified, testMethod will only be invoked 1001 * once. 1002 * <p/> 1003 * Note that this method also takes care of invoking the beforeTestMethod 1004 * and afterTestMethod, if any. 1005 * 1006 * Note (alex): this method can be refactored to use a SingleTestMethodWorker that 1007 * directly invokes 1008 * {@link #invokeTestMethod(Object, ITestNGMethod, Object[], int, XmlSuite, Map, ITestClass, ITestNGMethod[], ITestNGMethod[], ConfigurationGroupMethods, FailureContext)} 1009 * and this would simplify the implementation (see how DataTestMethodWorker is used) 1010 */ 1011 @Override 1012 public List<ITestResult> invokeTestMethods(ITestNGMethod testMethod, 1013 XmlSuite suite, 1014 Map<String, String> testParameters, 1015 ConfigurationGroupMethods groupMethods, 1016 Object instance, 1017 ITestContext testContext) 1018 { 1019 // Potential bug here if the test method was declared on a parent class 1020 assert null != testMethod.getTestClass() 1021 : "COULDN'T FIND TESTCLASS FOR " + testMethod.getRealClass(); 1022 1023 if (!MethodHelper.isEnabled(testMethod.getMethod(), m_annotationFinder)) { 1024 // return if the method is not enabled. No need to do any more calculations 1025 return Collections.emptyList(); 1026 } 1027 1028 // By the time this testMethod to be invoked, 1029 // all dependencies should be already run or we need to skip this method, 1030 // so invocation count should not affect dependencies check 1031 final String okToProceed = checkDependencies(testMethod, testContext.getAllTestMethods()); 1032 1033 if (okToProceed != null) { 1034 // 1035 // Not okToProceed. Test is being skipped 1036 // 1037 ITestResult result = registerSkippedTestResult(testMethod, null, System.currentTimeMillis(), 1038 new Throwable(okToProceed)); 1039 m_notifier.addSkippedTest(testMethod, result); 1040 return Collections.singletonList(result); 1041 } 1042 1043 1044 final Map<String, String> parameters = 1045 testMethod.findMethodParameters(testContext.getCurrentXmlTest()); 1046 1047 // For invocationCount > 1 and threadPoolSize > 1 run this method in its own pool thread. 1048 if (testMethod.getInvocationCount() > 1 && testMethod.getThreadPoolSize() > 1) { 1049 return invokePooledTestMethods(testMethod, suite, parameters, groupMethods, testContext); 1050 } 1051 1052 long timeOutInvocationCount = testMethod.getInvocationTimeOut(); 1053 //FIXME: Is this correct? 1054 boolean onlyOne = testMethod.getThreadPoolSize() > 1 || 1055 timeOutInvocationCount > 0; 1056 1057 int invocationCount = onlyOne ? 1 : testMethod.getInvocationCount(); 1058 1059 ExpectedExceptionsHolder expectedExceptionHolder = 1060 new ExpectedExceptionsHolder(m_annotationFinder, testMethod, 1061 new RegexpExpectedExceptionsHolder(m_annotationFinder, testMethod)); 1062 final ITestClass testClass= testMethod.getTestClass(); 1063 final List<ITestResult> result = Lists.newArrayList(); 1064 final FailureContext failure = new FailureContext(); 1065 final ITestNGMethod[] beforeMethods = filterMethods(testClass, testClass.getBeforeTestMethods(), CAN_RUN_FROM_CLASS); 1066 final ITestNGMethod[] afterMethods = filterMethods(testClass, testClass.getAfterTestMethods(), CAN_RUN_FROM_CLASS); 1067 while(invocationCount-- > 0) { 1068 if(false) { 1069 // Prevent code formatting 1070 } 1071 // 1072 // No threads, regular invocation 1073 // 1074 else { 1075 // Used in catch statement 1076 long start = System.currentTimeMillis(); 1077 1078 Map<String, String> allParameterNames = Maps.newHashMap(); 1079 ParameterBag bag = createParameters(testMethod, 1080 parameters, allParameterNames, suite, testContext, instance); 1081 1082 if (bag.hasErrors()) { 1083 final ITestResult tr = bag.errorResult; 1084 tr.setStatus(ITestResult.SKIP); 1085 runTestListeners(tr); 1086 m_notifier.addSkippedTest(testMethod, tr); 1087 result.add(tr); 1088 continue; 1089 } 1090 1091 Iterator<Object[]> allParameterValues = bag.parameterHolder.parameters; 1092 int parametersIndex = 0; 1093 1094 try { 1095 List<TestMethodWithDataProviderMethodWorker> workers = Lists.newArrayList(); 1096 1097 if (bag.parameterHolder.origin == ParameterOrigin.ORIGIN_DATA_PROVIDER && 1098 bag.parameterHolder.dataProviderHolder.annotation.isParallel()) { 1099 while (allParameterValues.hasNext()) { 1100 Object[] parameterValues = injectParameters(allParameterValues.next(), 1101 testMethod.getMethod(), testContext, null /* test result */); 1102 TestMethodWithDataProviderMethodWorker w = 1103 new TestMethodWithDataProviderMethodWorker(this, 1104 testMethod, parametersIndex, 1105 parameterValues, instance, suite, parameters, testClass, 1106 beforeMethods, afterMethods, groupMethods, 1107 expectedExceptionHolder, testContext, m_skipFailedInvocationCounts, 1108 invocationCount, failure.count, m_notifier); 1109 workers.add(w); 1110 // testng387: increment the param index in the bag. 1111 parametersIndex++; 1112 } 1113 PoolService<List<ITestResult>> ps = 1114 new PoolService<>(suite.getDataProviderThreadCount()); 1115 List<List<ITestResult>> r = ps.submitTasksAndWait(workers); 1116 for (List<ITestResult> l2 : r) { 1117 result.addAll(l2); 1118 } 1119 1120 } else { 1121 while (allParameterValues.hasNext()) { 1122 Object[] parameterValues = injectParameters(allParameterValues.next(), 1123 testMethod.getMethod(), testContext, null /* test result */); 1124 1125 List<ITestResult> tmpResults = Lists.newArrayList(); 1126 1127 try { 1128 tmpResults.add(invokeTestMethod(instance, 1129 testMethod, 1130 parameterValues, 1131 parametersIndex, 1132 suite, 1133 parameters, 1134 testClass, 1135 beforeMethods, 1136 afterMethods, 1137 groupMethods, failure)); 1138 } 1139 finally { 1140 if (failure.instances.isEmpty()) { 1141 result.addAll(tmpResults); 1142 } else { 1143 for (Object failedInstance : failure.instances) { 1144 List<ITestResult> retryResults = Lists.newArrayList(); 1145 1146 failure.count = retryFailed( 1147 failedInstance, testMethod, suite, testClass, beforeMethods, 1148 afterMethods, groupMethods, retryResults, 1149 failure.count, expectedExceptionHolder, 1150 testContext, parameters, parametersIndex); 1151 result.addAll(retryResults); 1152 } 1153 } 1154 1155 // 1156 // If we have a failure, skip all the 1157 // other invocationCounts 1158 // 1159 if (failure.count > 0 1160 && (m_skipFailedInvocationCounts 1161 || testMethod.skipFailedInvocations())) { 1162 while (invocationCount-- > 0) { 1163 result.add(registerSkippedTestResult(testMethod, instance, System.currentTimeMillis(), null)); 1164 } 1165 break; 1166 } 1167 }// end finally 1168 parametersIndex++; 1169 } 1170 } 1171 } 1172 catch (Throwable cause) { 1173 ITestResult r = 1174 new TestResult(testMethod.getTestClass(), 1175 instance, 1176 testMethod, 1177 cause, 1178 start, 1179 System.currentTimeMillis(), 1180 m_testContext); 1181 r.setStatus(TestResult.FAILURE); 1182 result.add(r); 1183 runTestListeners(r); 1184 m_notifier.addFailedTest(testMethod, r); 1185 } // catch 1186 } 1187 } 1188 1189 return result; 1190 1191 } // invokeTestMethod 1192 1193 private ITestResult registerSkippedTestResult(ITestNGMethod testMethod, Object instance, 1194 long start, Throwable throwable) { 1195 ITestResult result = 1196 new TestResult(testMethod.getTestClass(), 1197 instance, 1198 testMethod, 1199 throwable, 1200 start, 1201 System.currentTimeMillis(), 1202 m_testContext); 1203 result.setStatus(TestResult.SKIP); 1204 runTestListeners(result); 1205 1206 return result; 1207 } 1208 1209 /** 1210 * Gets an array of parameter values returned by data provider or the ones that 1211 * are injected based on parameter type. The method also checks for {@code NoInjection} 1212 * annotation 1213 * @param parameterValues parameter values from a data provider 1214 * @param method method to be invoked 1215 * @param context test context 1216 * @param testResult test result 1217 */ 1218 private Object[] injectParameters(Object[] parameterValues, Method method, 1219 ITestContext context, ITestResult testResult) 1220 throws TestNGException { 1221 List<Object> vResult = Lists.newArrayList(); 1222 int i = 0; 1223 int numValues = parameterValues.length; 1224 int numParams = method.getParameterTypes().length; 1225 1226 if (numValues > numParams && ! method.isVarArgs()) { 1227 throw new TestNGException("The data provider is trying to pass " + numValues 1228 + " parameters but the method " 1229 + method.getDeclaringClass().getName() + "#" + method.getName() 1230 + " takes " + numParams); 1231 } 1232 1233 // beyond this, numValues <= numParams 1234 for (Class<?> cls : method.getParameterTypes()) { 1235 Annotation[] annotations = method.getParameterAnnotations()[i]; 1236 boolean noInjection = false; 1237 for (Annotation a : annotations) { 1238 if (a instanceof NoInjection) { 1239 noInjection = true; 1240 break; 1241 } 1242 } 1243 Object injected = Parameters.getInjectedParameter(cls, method, context, testResult); 1244 if (injected != null && ! noInjection) { 1245 vResult.add(injected); 1246 } else { 1247 try { 1248 if (method.isVarArgs()) vResult.add(parameterValues); 1249 else vResult.add(parameterValues[i++]); 1250 } catch (ArrayIndexOutOfBoundsException ex) { 1251 throw new TestNGException("The data provider is trying to pass " + numValues 1252 + " parameters but the method " 1253 + method.getDeclaringClass().getName() + "#" + method.getName() 1254 + " takes " + numParams 1255 + " and TestNG is unable in inject a suitable object", ex); 1256 } 1257 } 1258 } 1259 return vResult.toArray(new Object[vResult.size()]); 1260 } 1261 1262 private ParameterBag handleParameters(ITestNGMethod testMethod, 1263 Object instance, 1264 Map<String, String> allParameterNames, 1265 Map<String, String> parameters, 1266 Object[] parameterValues, 1267 XmlSuite suite, 1268 ITestContext testContext, 1269 Object fedInstance, 1270 ITestResult testResult) 1271 { 1272 try { 1273 return new ParameterBag( 1274 Parameters.handleParameters(testMethod, 1275 allParameterNames, 1276 instance, 1277 new Parameters.MethodParameters(parameters, 1278 testMethod.findMethodParameters(testContext.getCurrentXmlTest()), 1279 parameterValues, 1280 testMethod.getMethod(), testContext, testResult), 1281 suite, 1282 m_annotationFinder, 1283 fedInstance)); 1284 } 1285 // catch(TestNGException ex) { 1286 // throw ex; 1287 // } 1288 catch(Throwable cause) { 1289 return new ParameterBag( 1290 new TestResult( 1291 testMethod.getTestClass(), 1292 instance, 1293 testMethod, 1294 cause, 1295 System.currentTimeMillis(), 1296 System.currentTimeMillis(), 1297 m_testContext)); 1298 } 1299 } 1300 1301 /** 1302 * Invokes a method that has a specified threadPoolSize. 1303 */ 1304 private List<ITestResult> invokePooledTestMethods(ITestNGMethod testMethod, 1305 XmlSuite suite, 1306 Map<String, String> parameters, 1307 ConfigurationGroupMethods groupMethods, 1308 ITestContext testContext) 1309 { 1310 // 1311 // Create the workers 1312 // 1313 List<IWorker<ITestNGMethod>> workers = Lists.newArrayList(); 1314 1315 // Create one worker per invocationCount 1316 for (int i = 0; i < testMethod.getInvocationCount(); i++) { 1317 // we use clones for reporting purposes 1318 ITestNGMethod clonedMethod= testMethod.clone(); 1319 clonedMethod.setInvocationCount(1); 1320 clonedMethod.setThreadPoolSize(1); 1321 1322 MethodInstance mi = new MethodInstance(clonedMethod); 1323 workers.add(new SingleTestMethodWorker(this, 1324 mi, 1325 suite, 1326 parameters, 1327 testContext, 1328 m_classListeners)); 1329 } 1330 1331 return runWorkers(testMethod, workers, testMethod.getThreadPoolSize(), groupMethods, suite, 1332 parameters); 1333 } 1334 1335 static class FailureContext { 1336 int count = 0; 1337 List<Object> instances = Lists.newArrayList(); 1338 } 1339 1340 void handleInvocationResults(ITestNGMethod testMethod, 1341 List<ITestResult> result, 1342 ExpectedExceptionsHolder expectedExceptionsHolder, 1343 FailureContext failure) 1344 { 1345 // 1346 // Go through all the results and create a TestResult for each of them 1347 // 1348 List<ITestResult> resultsToRetry = Lists.newArrayList(); 1349 1350 for (ITestResult testResult : result) { 1351 Throwable ite= testResult.getThrowable(); 1352 int status= testResult.getStatus(); 1353 1354 boolean handled = false; 1355 1356 // Exception thrown? 1357 if (ite != null) { 1358 1359 // Invocation caused an exception, see if the method was annotated with @ExpectedException 1360 if (expectedExceptionsHolder != null) { 1361 if (expectedExceptionsHolder.isExpectedException(ite)) { 1362 testResult.setStatus(ITestResult.SUCCESS); 1363 status = ITestResult.SUCCESS; 1364 } else { 1365 if (isSkipExceptionAndSkip(ite)){ 1366 status = ITestResult.SKIP; 1367 } else { 1368 testResult.setThrowable(expectedExceptionsHolder.wrongException(ite)); 1369 status = ITestResult.FAILURE; 1370 } 1371 } 1372 } else { 1373 handleException(ite, testMethod, testResult, failure.count++); 1374 handled = true; 1375 status = testResult.getStatus(); 1376 } 1377 } 1378 1379 // No exception thrown, make sure we weren't expecting one 1380 else if(status != ITestResult.SKIP && expectedExceptionsHolder != null) { 1381 TestException exception = expectedExceptionsHolder.noException(testMethod); 1382 if (exception != null) { 1383 testResult.setThrowable(exception); 1384 status= ITestResult.FAILURE; 1385 } 1386 } 1387 1388 IRetryAnalyzer retryAnalyzer = testMethod.getRetryAnalyzer(); 1389 boolean willRetry = retryAnalyzer != null && status == ITestResult.FAILURE && failure.instances != null && retryAnalyzer.retry(testResult); 1390 1391 if (willRetry) { 1392 resultsToRetry.add(testResult); 1393 failure.count++; 1394 failure.instances.add(testResult.getInstance()); 1395 testResult.setStatus(ITestResult.SKIP); 1396 } else { 1397 testResult.setStatus(status); 1398 if (status == ITestResult.FAILURE && !handled) { 1399 handleException(ite, testMethod, testResult, failure.count++); 1400 } 1401 } 1402 collectResults(testMethod, Collections.singleton(testResult)); 1403 } // for results 1404 1405 removeResultsToRetryFromResult(resultsToRetry, result, failure); 1406 } 1407 1408 private boolean isSkipExceptionAndSkip(Throwable ite) { 1409 return SkipException.class.isAssignableFrom(ite.getClass()) && ((SkipException) ite).isSkip(); 1410 } 1411 1412 private void removeResultsToRetryFromResult(List<ITestResult> resultsToRetry, 1413 List<ITestResult> result, FailureContext failure) { 1414 if (resultsToRetry != null) { 1415 for (ITestResult res : resultsToRetry) { 1416 result.remove(res); 1417 failure.count--; 1418 } 1419 } 1420 } 1421 1422 /** 1423 * To reduce thread contention and also to correctly handle thread-confinement 1424 * this method invokes the @BeforeGroups and @AfterGroups corresponding to the current @Test method. 1425 */ 1426 private List<ITestResult> runWorkers(ITestNGMethod testMethod, 1427 List<IWorker<ITestNGMethod>> workers, 1428 int threadPoolSize, 1429 ConfigurationGroupMethods groupMethods, 1430 XmlSuite suite, 1431 Map<String, String> parameters) 1432 { 1433 // Invoke @BeforeGroups on the original method (reduce thread contention, 1434 // and also solve thread confinement) 1435 ITestClass testClass= testMethod.getTestClass(); 1436 Object[] instances = testClass.getInstances(true); 1437 for(Object instance: instances) { 1438 invokeBeforeGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1439 } 1440 1441 1442 long maxTimeOut= -1; // 10 seconds 1443 1444 for(IWorker<ITestNGMethod> tmw : workers) { 1445 long mt= tmw.getTimeOut(); 1446 if(mt > maxTimeOut) { 1447 maxTimeOut= mt; 1448 } 1449 } 1450 1451 ThreadUtil.execute(workers, threadPoolSize, maxTimeOut, true); 1452 1453 // 1454 // Collect all the TestResults 1455 // 1456 List<ITestResult> result = Lists.newArrayList(); 1457 for (IWorker<ITestNGMethod> tmw : workers) { 1458 if (tmw instanceof TestMethodWorker) { 1459 result.addAll(((TestMethodWorker)tmw).getTestResults()); 1460 } 1461 } 1462 1463 for(Object instance: instances) { 1464 invokeAfterGroupsConfigurations(testClass, testMethod, groupMethods, suite, parameters, instance); 1465 } 1466 1467 return result; 1468 } 1469 1470 /** 1471 * Checks to see of the test method has certain dependencies that prevents 1472 * TestNG from executing it 1473 * @param testMethod test method being checked for 1474 * @return error message or null if dependencies have been run successfully 1475 */ 1476 private String checkDependencies(ITestNGMethod testMethod, 1477 ITestNGMethod[] allTestMethods) 1478 { 1479 // If this method is marked alwaysRun, no need to check for its dependencies 1480 if (testMethod.isAlwaysRun()) { 1481 return null; 1482 } 1483 1484 // Any missing group? 1485 if (testMethod.getMissingGroup() != null 1486 && !testMethod.ignoreMissingDependencies()) { 1487 return "Method " + testMethod + " depends on nonexistent group \"" + testMethod.getMissingGroup() + "\""; 1488 } 1489 1490 // If this method depends on groups, collect all the methods that 1491 // belong to these groups and make sure they have been run successfully 1492 final String[] groups = testMethod.getGroupsDependedUpon(); 1493 if (null != groups && groups.length > 0) { 1494 // Get all the methods that belong to the group depended upon 1495 for (String element : groups) { 1496 ITestNGMethod[] methods = 1497 MethodGroupsHelper.findMethodsThatBelongToGroup(testMethod, 1498 m_testContext.getAllTestMethods(), 1499 element); 1500 if (methods.length == 0 && !testMethod.ignoreMissingDependencies()) { 1501 // Group is missing 1502 return "Method " + testMethod + " depends on nonexistent group \"" + element + "\""; 1503 } 1504 if (!haveBeenRunSuccessfully(testMethod, methods)) { 1505 return "Method " + testMethod + 1506 " depends on not successfully finished methods in group \"" + element + "\""; 1507 } 1508 } 1509 } // depends on groups 1510 1511 // If this method depends on other methods, make sure all these other 1512 // methods have been run successfully 1513 if (dependsOnMethods(testMethod)) { 1514 ITestNGMethod[] methods = 1515 MethodHelper.findDependedUponMethods(testMethod, allTestMethods); 1516 1517 if (!haveBeenRunSuccessfully(testMethod, methods)) { 1518 return "Method " + testMethod + " depends on not successfully finished methods"; 1519 } 1520 } 1521 1522 return null; 1523 } 1524 1525 /** 1526 * @return the test results that apply to one of the instances of the testMethod. 1527 */ 1528 private Set<ITestResult> keepSameInstances(ITestNGMethod method, Set<ITestResult> results) { 1529 Set<ITestResult> result = Sets.newHashSet(); 1530 for (ITestResult r : results) { 1531 final Object o = method.getInstance(); 1532 // Keep this instance if 1) It's on a different class or 2) It's on the same class 1533 // and on the same instance 1534 Object instance = r.getInstance() != null 1535 ? r.getInstance() : r.getMethod().getInstance(); 1536 if (r.getTestClass() != method.getTestClass() || instance == o) result.add(r); 1537 } 1538 return result; 1539 } 1540 1541 /** 1542 * @return true if all the methods have been run successfully 1543 */ 1544 private boolean haveBeenRunSuccessfully(ITestNGMethod testMethod, ITestNGMethod[] methods) { 1545 // Make sure the method has been run successfully 1546 for (ITestNGMethod method : methods) { 1547 Set<ITestResult> results = keepSameInstances(testMethod, m_notifier.getPassedTests(method)); 1548 Set<ITestResult> failedAndSkippedMethods = Sets.newHashSet(); 1549 failedAndSkippedMethods.addAll(m_notifier.getFailedTests(method)); 1550 failedAndSkippedMethods.addAll(m_notifier.getSkippedTests(method)); 1551 Set<ITestResult> failedresults = keepSameInstances(testMethod, failedAndSkippedMethods); 1552 1553 // If failed results were returned on the same instance, then these tests didn't pass 1554 if (failedresults != null && failedresults.size() > 0) { 1555 return false; 1556 } 1557 1558 for (ITestResult result : results) { 1559 if(!result.isSuccess()) { 1560 return false; 1561 } 1562 } 1563 } 1564 1565 return true; 1566 } 1567 1568 // private boolean containsInstance(Set<ITestResult> failedresults, Object[] instances) { 1569 // for (ITestResult tr : failedresults) { 1570 // for (Object o : instances) { 1571 // if (o == tr.getInstance()) { 1572 // return true; 1573 // } 1574 // } 1575 // } 1576 // return false; 1577 // } 1578 1579 /** 1580 * An exception was thrown by the test, determine if this method 1581 * should be marked as a failure or as failure_but_within_successPercentage 1582 */ 1583 private void handleException(Throwable throwable, 1584 ITestNGMethod testMethod, 1585 ITestResult testResult, 1586 int failureCount) { 1587 if (throwable != null) { 1588 testResult.setThrowable(throwable); 1589 } 1590 int successPercentage= testMethod.getSuccessPercentage(); 1591 int invocationCount= testMethod.getInvocationCount(); 1592 float numberOfTestsThatCanFail= ((100 - successPercentage) * invocationCount) / 100f; 1593 1594 if(failureCount < numberOfTestsThatCanFail) { 1595 testResult.setStatus(ITestResult.SUCCESS_PERCENTAGE_FAILURE); 1596 } 1597 else { 1598 testResult.setStatus(ITestResult.FAILURE); 1599 } 1600 1601 } 1602 1603 static interface Predicate<K, T> { 1604 boolean isTrue(K k, T v); 1605 } 1606 1607 static class CanRunFromClassPredicate implements Predicate <ITestNGMethod, IClass> { 1608 @Override 1609 public boolean isTrue(ITestNGMethod m, IClass v) { 1610 return m.canRunFromClass(v); 1611 } 1612 } 1613 1614 static class SameClassNamePredicate implements Predicate<ITestNGMethod, IClass> { 1615 @Override 1616 public boolean isTrue(ITestNGMethod m, IClass c) { 1617 return c == null || m.getTestClass().getName().equals(c.getName()); 1618 } 1619 } 1620 1621 /** 1622 * @return Only the ITestNGMethods applicable for this testClass 1623 */ 1624 private ITestNGMethod[] filterMethods(IClass testClass, ITestNGMethod[] methods, 1625 Predicate<ITestNGMethod, IClass> predicate) { 1626 List<ITestNGMethod> vResult= Lists.newArrayList(); 1627 1628 for(ITestNGMethod tm : methods) { 1629 if (predicate.isTrue(tm, testClass)) { 1630 log(10, "Keeping method " + tm + " for class " + testClass); 1631 vResult.add(tm); 1632 } else { 1633 log(10, "Filtering out method " + tm + " for class " + testClass); 1634 } 1635 } 1636 1637 ITestNGMethod[] result= vResult.toArray(new ITestNGMethod[vResult.size()]); 1638 1639 return result; 1640 } 1641 1642 /** 1643 * @return true if this method depends on certain methods. 1644 */ 1645 private boolean dependsOnMethods(ITestNGMethod tm) { 1646 String[] methods = tm.getMethodsDependedUpon(); 1647 return null != methods && methods.length > 0; 1648 } 1649 1650 private void runConfigurationListeners(ITestResult tr, boolean before) { 1651 if (before) { 1652 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1653 if (icl instanceof IConfigurationListener2) { 1654 ((IConfigurationListener2) icl).beforeConfiguration(tr); 1655 } 1656 } 1657 } else { 1658 for(IConfigurationListener icl: m_notifier.getConfigurationListeners()) { 1659 switch(tr.getStatus()) { 1660 case ITestResult.SKIP: 1661 icl.onConfigurationSkip(tr); 1662 break; 1663 case ITestResult.FAILURE: 1664 icl.onConfigurationFailure(tr); 1665 break; 1666 case ITestResult.SUCCESS: 1667 icl.onConfigurationSuccess(tr); 1668 break; 1669 } 1670 } 1671 } 1672 } 1673 1674 void runTestListeners(ITestResult tr) { 1675 runTestListeners(tr, m_notifier.getTestListeners()); 1676 } 1677 1678 // TODO: move this from here as it is directly called from TestNG 1679 public static void runTestListeners(ITestResult tr, List<ITestListener> listeners) { 1680 for (ITestListener itl : listeners) { 1681 switch(tr.getStatus()) { 1682 case ITestResult.SKIP: { 1683 itl.onTestSkipped(tr); 1684 break; 1685 } 1686 case ITestResult.SUCCESS_PERCENTAGE_FAILURE: { 1687 itl.onTestFailedButWithinSuccessPercentage(tr); 1688 break; 1689 } 1690 case ITestResult.FAILURE: { 1691 itl.onTestFailure(tr); 1692 break; 1693 } 1694 case ITestResult.SUCCESS: { 1695 itl.onTestSuccess(tr); 1696 break; 1697 } 1698 1699 case ITestResult.STARTED: { 1700 itl.onTestStart(tr); 1701 break; 1702 } 1703 1704 default: { 1705 assert false : "UNKNOWN STATUS:" + tr; 1706 } 1707 } 1708 } 1709 } 1710 1711 private void log(int level, String s) { 1712 Utils.log("Invoker " + Thread.currentThread().hashCode(), level, s); 1713 } 1714 1715 /** 1716 * This class holds a {@code ParameterHolder} or in case of an error, a non-null 1717 * {@code TestResult} containing the cause 1718 */ 1719 private static class ParameterBag { 1720 final ParameterHolder parameterHolder; 1721 final ITestResult errorResult; 1722 1723 public ParameterBag(ParameterHolder parameterHolder) { 1724 this.parameterHolder = parameterHolder; 1725 this.errorResult = null; 1726 } 1727 1728 public ParameterBag(ITestResult errorResult) { 1729 this.parameterHolder = null; 1730 this.errorResult = errorResult; 1731 } 1732 1733 public boolean hasErrors() { 1734 return errorResult != null; 1735 } 1736 } 1737 1738 } 1739