Skip to content

Commit 2f4b7c7

Browse files
bj-9527bj-9527
committed
configfailurepolicy=continue only works for BeforeTest when using TestNG XML file
Fixes #2731
1 parent 0438740 commit 2f4b7c7

File tree

9 files changed

+239
-50
lines changed

9 files changed

+239
-50
lines changed

‎CHANGES.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
Current
22
7.6.0
3+
Fixed: GITHUB-2731: configfailurepolicy=continue only works for BeforeTest when using TestNG XML file (Nan Liang)
34
Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang)
45
Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek)
56
Fixed: GITHUB-2709: Testnames not working together with suites in suite (Martin Aldrin)

‎testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ public void invokeConfigurations(ConfigMethodArguments arguments) {
274274
tm.getGroups(),
275275
arguments.getTestClass(),
276276
arguments.getInstance())
277-
&& !alwaysRun) {
277+
&& !alwaysRun
278+
&& !m_continueOnFailedConfiguration) {
278279
log(3, "Skipping " + Utils.detailedMethodName(tm, true));
279280
InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), testResult);
280281
runConfigurationListeners(testResult, arguments.getTestMethod(), true /* before */);

‎testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -597,7 +597,8 @@ private ITestResult invokeMethod(
597597
arguments.getTestMethod(),
598598
arguments.getTestMethod().getGroups(),
599599
arguments.getTestClass(),
600-
arguments.getInstance())) {
600+
arguments.getInstance())
601+
&& suite.getConfigFailurePolicy() == XmlSuite.FailurePolicy.SKIP) {
601602
Throwable exception =
602603
ExceptionUtils.getExceptionDetails(m_testContext, arguments.getInstance());
603604
ITestResult result =

‎testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package test.configurationfailurepolicy;
2+
3+
import static org.testng.Assert.assertEquals;
4+
import static test.SimpleBaseTest.getPathToResource;
5+
6+
import org.testng.ITestContext;
7+
import org.testng.TestListenerAdapter;
8+
import org.testng.TestNG;
9+
import org.testng.annotations.BeforeClass;
10+
import org.testng.annotations.DataProvider;
11+
import org.testng.annotations.Test;
12+
import org.testng.xml.XmlSuite;
13+
import test.configurationfailurepolicy.issue2731.ConfigFailTestSample;
14+
import testhelper.OutputDirectoryPatch;
15+
16+
public class FailureContinuePolicyTest {
17+
// only if this is run from an xml file that sets this on the suite
18+
@BeforeClass(enabled = false)
19+
public void setupClass(ITestContext testContext) {
20+
assertEquals(
21+
testContext.getSuite().getXmlSuite().getConfigFailurePolicy(),
22+
XmlSuite.FailurePolicy.CONTINUE);
23+
}
24+
25+
@DataProvider(name = "dp")
26+
public Object[][] getData() {
27+
return new Object[][] {
28+
// params - confFail, confSkip, skippedTests
29+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
30+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 0, 0},
31+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 0},
32+
new Object[] {
33+
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
34+
1,
35+
0,
36+
0
37+
},
38+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 0},
39+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 0, 0},
40+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
41+
new Object[] {
42+
new Class[] {
43+
ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class
44+
},
45+
2,
46+
0,
47+
0
48+
},
49+
new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 0},
50+
new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 0},
51+
new Object[] {
52+
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 0
53+
},
54+
new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 0, 0},
55+
new Object[] {new Class[] {ConfigFailTestSample.class}, 4, 0, 0}
56+
};
57+
}
58+
59+
@Test(dataProvider = "dp")
60+
public void confFailureTest(
61+
Class[] classesUnderTest,
62+
int configurationFailures,
63+
int configurationSkips,
64+
int skippedTests) {
65+
66+
TestListenerAdapter tla = new TestListenerAdapter();
67+
TestNG testng = new TestNG();
68+
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
69+
testng.setTestClasses(classesUnderTest);
70+
testng.addListener(tla);
71+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
72+
testng.run();
73+
74+
verify(tla, configurationFailures, configurationSkips, skippedTests);
75+
}
76+
77+
@Test
78+
public void confFailureTestInvolvingGroups() {
79+
Class[] classesUnderTest =
80+
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeGroupsAfterClassAfterGroups.class};
81+
82+
TestListenerAdapter tla = new TestListenerAdapter();
83+
TestNG testng = new TestNG();
84+
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
85+
testng.setTestClasses(classesUnderTest);
86+
testng.addListener(tla);
87+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
88+
testng.setGroups("group1");
89+
testng.run();
90+
verify(tla, 1, 0, 0);
91+
}
92+
93+
@Test
94+
public void commandLineTest_policyAsSkip() {
95+
String[] argv =
96+
new String[] {
97+
"-log",
98+
"0",
99+
"-d",
100+
OutputDirectoryPatch.getOutputDirectory(),
101+
"-configfailurepolicy",
102+
"skip",
103+
"-testclass",
104+
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
105+
};
106+
TestListenerAdapter tla = new TestListenerAdapter();
107+
TestNG.privateMain(argv, tla);
108+
109+
verify(tla, 1, 1, 2);
110+
}
111+
112+
@Test
113+
public void commandLineTest_policyAsContinue() {
114+
String[] argv =
115+
new String[] {
116+
"-log",
117+
"0",
118+
"-d",
119+
OutputDirectoryPatch.getOutputDirectory(),
120+
"-configfailurepolicy",
121+
"continue",
122+
"-testclass",
123+
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
124+
};
125+
TestListenerAdapter tla = new TestListenerAdapter();
126+
TestNG.privateMain(argv, tla);
127+
128+
verify(tla, 2, 0, 0);
129+
}
130+
131+
@Test
132+
public void commandLineTestWithXMLFile_policyAsSkip() {
133+
String[] argv =
134+
new String[] {
135+
"-log",
136+
"0",
137+
"-d",
138+
OutputDirectoryPatch.getOutputDirectory(),
139+
"-configfailurepolicy",
140+
"skip",
141+
getPathToResource("testng-configfailure.xml")
142+
};
143+
TestListenerAdapter tla = new TestListenerAdapter();
144+
TestNG.privateMain(argv, tla);
145+
146+
verify(tla, 1, 1, 2);
147+
}
148+
149+
@Test
150+
public void commandLineTestWithXMLFile_policyAsContinue() {
151+
String[] argv =
152+
new String[] {
153+
"-log",
154+
"0",
155+
"-d",
156+
OutputDirectoryPatch.getOutputDirectory(),
157+
"-configfailurepolicy",
158+
"continue",
159+
getPathToResource("testng-configfailure.xml")
160+
};
161+
TestListenerAdapter tla = new TestListenerAdapter();
162+
TestNG.privateMain(argv, tla);
163+
164+
verify(tla, 2, 0, 0);
165+
}
166+
167+
private void verify(
168+
TestListenerAdapter tla,
169+
int configurationFailures,
170+
int configurationSkips,
171+
int skippedTests) {
172+
assertEquals(
173+
tla.getConfigurationFailures().size(),
174+
configurationFailures,
175+
"wrong number of configuration failures");
176+
assertEquals(
177+
tla.getConfigurationSkips().size(),
178+
configurationSkips,
179+
"wrong number of configuration skips");
180+
assertEquals(tla.getSkippedTests().size(), skippedTests, "wrong number of skipped tests");
181+
}
182+
}

‎testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java‎

Lines changed: 7 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ public class FailurePolicyTest {
1818
@BeforeClass(enabled = false)
1919
public void setupClass(ITestContext testContext) {
2020
assertEquals(
21-
testContext.getSuite().getXmlSuite().getConfigFailurePolicy(),
22-
XmlSuite.FailurePolicy.CONTINUE);
21+
testContext.getSuite().getXmlSuite().getConfigFailurePolicy(), XmlSuite.FailurePolicy.SKIP);
2322
}
2423

2524
@DataProvider(name = "dp")
@@ -28,15 +27,15 @@ public Object[][] getData() {
2827
// params - confFail, confSkip, skippedTests
2928
new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1},
3029
new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1},
31-
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2},
30+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 1, 1, 2},
3231
new Object[] {
3332
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
3433
1,
3534
3,
3635
1
3736
},
38-
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4},
39-
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2},
37+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 1, 3, 4},
38+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 1, 3, 2},
4039
new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2},
4140
new Object[] {
4241
new Class[] {
@@ -49,7 +48,7 @@ public Object[][] getData() {
4948
new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1},
5049
new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2},
5150
new Object[] {
52-
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8
51+
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 2, 6, 8
5352
},
5453
new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2},
5554
};
@@ -67,7 +66,7 @@ public void confFailureTest(
6766
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
6867
testng.setTestClasses(classesUnderTest);
6968
testng.addListener(tla);
70-
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
69+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP);
7170
testng.run();
7271

7372
verify(tla, configurationFailures, configurationSkips, skippedTests);
@@ -83,7 +82,7 @@ public void confFailureTestInvolvingGroups() {
8382
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
8483
testng.setTestClasses(classesUnderTest);
8584
testng.addListener(tla);
86-
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
85+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP);
8786
testng.setGroups("group1");
8887
testng.run();
8988
verify(tla, 1, 3, 1);
@@ -108,25 +107,6 @@ public void commandLineTest_policyAsSkip() {
108107
verify(tla, 1, 1, 2);
109108
}
110109

111-
@Test
112-
public void commandLineTest_policyAsContinue() {
113-
String[] argv =
114-
new String[] {
115-
"-log",
116-
"0",
117-
"-d",
118-
OutputDirectoryPatch.getOutputDirectory(),
119-
"-configfailurepolicy",
120-
"continue",
121-
"-testclass",
122-
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
123-
};
124-
TestListenerAdapter tla = new TestListenerAdapter();
125-
TestNG.privateMain(argv, tla);
126-
127-
verify(tla, 2, 0, 2);
128-
}
129-
130110
@Test
131111
public void commandLineTestWithXMLFile_policyAsSkip() {
132112
String[] argv =
@@ -145,24 +125,6 @@ public void commandLineTestWithXMLFile_policyAsSkip() {
145125
verify(tla, 1, 1, 2);
146126
}
147127

148-
@Test
149-
public void commandLineTestWithXMLFile_policyAsContinue() {
150-
String[] argv =
151-
new String[] {
152-
"-log",
153-
"0",
154-
"-d",
155-
OutputDirectoryPatch.getOutputDirectory(),
156-
"-configfailurepolicy",
157-
"continue",
158-
getPathToResource("testng-configfailure.xml")
159-
};
160-
TestListenerAdapter tla = new TestListenerAdapter();
161-
TestNG.privateMain(argv, tla);
162-
163-
verify(tla, 2, 0, 2);
164-
}
165-
166128
private void verify(
167129
TestListenerAdapter tla,
168130
int configurationFailures,

‎testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package test.configurationfailurepolicy.issue2731;
2+
3+
import org.testng.Assert;
4+
import org.testng.annotations.BeforeClass;
5+
import org.testng.annotations.BeforeMethod;
6+
import org.testng.annotations.BeforeSuite;
7+
import org.testng.annotations.BeforeTest;
8+
import org.testng.annotations.Test;
9+
10+
public class ConfigFailTestSample {
11+
@BeforeSuite
12+
public void beforeSuite() {
13+
Assert.fail("This before suite is fail");
14+
}
15+
16+
@BeforeTest
17+
public void beforeTest() {
18+
Assert.fail("This before test is fail");
19+
}
20+
21+
@BeforeClass
22+
public void beforeClass() {
23+
Assert.fail("This before class is fail");
24+
}
25+
26+
@BeforeMethod
27+
public void beforeMethod() {
28+
Assert.fail("This before method is fail");
29+
}
30+
31+
@Test
32+
public void test() {}
33+
}

‎testng-core/src/test/java/test/listeners/github1602/IssueTest.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,19 +35,27 @@ public Object[][] getData() {
3535
"BeforeInvocation_afterMethod_STARTED",
3636
"AfterInvocation_afterMethod_SUCCESS");
3737
List<String> baseList =
38+
Arrays.asList(
39+
"BeforeInvocation_beforeMethod_STARTED",
40+
"AfterInvocation_beforeMethod_FAILURE",
41+
"BeforeInvocation_testMethod_STARTED",
42+
"AfterInvocation_testMethod_SUCCESS",
43+
"BeforeInvocation_afterMethod_STARTED");
44+
List<String> commonSkipList =
3845
Arrays.asList(
3946
"BeforeInvocation_beforeMethod_STARTED",
4047
"AfterInvocation_beforeMethod_FAILURE",
4148
"BeforeInvocation_testMethod_SKIP",
4249
"AfterInvocation_testMethod_SKIP",
43-
"BeforeInvocation_afterMethod_STARTED");
50+
"BeforeInvocation_afterMethod_STARTED",
51+
"AfterInvocation_afterMethod_SKIP");
4452
List<String> skipList = Lists.newArrayList(baseList);
4553
skipList.add("AfterInvocation_afterMethod_SKIP");
4654
List<String> failList = Lists.newArrayList(baseList);
4755
failList.add("AfterInvocation_afterMethod_FAILURE");
4856
return new Object[][] {
4957
{TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, passList},
50-
{TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, skipList},
58+
{TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, commonSkipList},
5159
{TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, passList},
5260
{TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, failList}
5361
};

‎testng-core/src/test/java/test/listeners/issue1777/IssueTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public void testOnStartInvokedForSkippedTests() {
2828
"before_test_method: test1",
2929
"after_test_method: test1",
3030
"after_test_method: test1",
31-
"testSkipped_test_method: test1",
31+
"testSuccess_test_method: test1",
3232
"testStart_test_method: test2",
3333
"before_test_method: test2",
3434
"before_test_method: test2",

‎testng-core/src/test/resources/testng.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,7 @@
784784
<test name="ConfigFailurePolicy">
785785
<classes>
786786
<class name="test.configurationfailurepolicy.FailurePolicyTest" />
787+
<class name="test.configurationfailurepolicy.FailureContinuePolicyTest"></class>
787788
</classes>
788789
</test>
789790

0 commit comments

Comments
 (0)