Issue #137069: adding @org.netbeans.nbjunit.RandomlyFails to mark randomly failing tests.
diff --git a/nbjunit/src/org/netbeans/junit/NbTestCase.java b/nbjunit/src/org/netbeans/junit/NbTestCase.java
--- a/nbjunit/src/org/netbeans/junit/NbTestCase.java
+++ b/nbjunit/src/org/netbeans/junit/NbTestCase.java
@@ -125,6 +125,18 @@
* @return true if the test can run
*/
public boolean canRun() {
+ if (NbTestSuite.ignoreRandomFailures()) {
+ if (getClass().isAnnotationPresent(RandomlyFails.class)) {
+ return false;
+ }
+ try {
+ if (getClass().getMethod(getName()).isAnnotationPresent(RandomlyFails.class)) {
+ return false;
+ }
+ } catch (NoSuchMethodException x) {
+ // Specially named methods; let it pass.
+ }
+ }
if (null == filter) {
//System.out.println("NBTestCase.canRun(): filter == null name=" + name ());
return true; // no filter was aplied
diff --git a/nbjunit/src/org/netbeans/junit/NbTestSuite.java b/nbjunit/src/org/netbeans/junit/NbTestSuite.java
--- a/nbjunit/src/org/netbeans/junit/NbTestSuite.java
+++ b/nbjunit/src/org/netbeans/junit/NbTestSuite.java
@@ -44,6 +44,7 @@
import java.util.*;
import junit.framework.TestSuite;
import junit.framework.Test;
+import junit.framework.TestCase;
/**
* NetBeans extension to JUnit's TestSuite class.
@@ -52,6 +53,10 @@
private Filter fFilter;
+
+ static boolean ignoreRandomFailures() {
+ return Boolean.getBoolean("ignore.random.failures");
+ }
/**
* Constructs an empty TestSuite.
@@ -66,7 +71,22 @@
*
*/
public NbTestSuite(Class theClass) {
- super(theClass);
+ super(testCaseClassOrDummy(theClass));
+ }
+ private static Class testCaseClassOrDummy(Class testClass) {
+ if (ignoreRandomFailures() && ((Class) testClass).isAnnotationPresent(RandomlyFails.class)) {
+ return APIJail.Dummy.class;
+ } else {
+ return testClass;
+ }
+ }
+ private static class APIJail {
+ public static class Dummy extends TestCase {
+ public Dummy(String name) {
+ super(name);
+ }
+ public void testNothing() {}
+ }
}
/**
@@ -95,6 +115,10 @@
* adds a test suite to this test suite
*/
public void addTestSuite(Class testClass) {
+ // XXX should be expecting Class, perhaps
+ if (ignoreRandomFailures() && ((Class) testClass).isAnnotationPresent(RandomlyFails.class)) {
+ return;
+ }
NbTest t = new NbTestSuite(testClass);
t.setFilter(fFilter);
addTest(t);
diff --git a/nbjunit/src/org/netbeans/junit/RandomlyFails.java b/nbjunit/src/org/netbeans/junit/RandomlyFails.java
new file mode 100644
--- /dev/null
+++ b/nbjunit/src/org/netbeans/junit/RandomlyFails.java
@@ -0,0 +1,64 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License. When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2008 Sun Microsystems, Inc.
+ */
+
+package org.netbeans.junit;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates that this test can fail randomly.
+ * When used on a method in a class extending {@link NbTestCase},
+ * the implicit suite will exclude this test in case the system property
+ * ignore.random.failures is set to true.
+ * When used on a class passed to {@link NbTestSuite#NbTestSuite(Class)} or {@link NbTestSuite#addTestSuite(Class)},
+ * the suite will be empty if the system property is set;
+ * the same if it is used on a class extending {@link NbTestCase}.
+ *
Test runs which must be reliable should define the system property.
+ * (E.g. for NetBeans modules: ant -Dtest-unit-sys-prop.ignore.random.failures=true test)
+ * Developers running tests interactively should not.
+ * @since XXX
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Inherited
+public @interface RandomlyFails {}
diff --git a/nbjunit/test/unit/src/org/netbeans/junit/RandomlyFailsTest.java b/nbjunit/test/unit/src/org/netbeans/junit/RandomlyFailsTest.java
new file mode 100644
--- /dev/null
+++ b/nbjunit/test/unit/src/org/netbeans/junit/RandomlyFailsTest.java
@@ -0,0 +1,145 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License. When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2008 Sun Microsystems, Inc.
+ */
+
+package org.netbeans.junit;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.runner.BaseTestRunner;
+import junit.textui.TestRunner;
+
+public class RandomlyFailsTest extends TestCase {
+
+ private static List runs;
+
+ public RandomlyFailsTest(String name) {
+ super(name);
+ }
+
+ public static class One extends NbTestCase {
+ public One(String n) {super(n);}
+ public void testReliable() {runs.add(10);}
+ @RandomlyFails public void testUnreliable() {runs.add(11);}
+ }
+
+ public static class Two extends NbTestCase {
+ public Two(String n) {super(n);}
+ public void testReliable() {runs.add(20);}
+ }
+
+ @RandomlyFails public static class Three extends NbTestCase {
+ public Three(String n) {super(n);}
+ public void testUnreliable() {runs.add(30);}
+ }
+
+ public static class Four extends NbTestCase {
+ public Four(String n) {super(n);}
+ public void testReliable() {runs.add(40);}
+ @RandomlyFails public void testUnreliable() {runs.add(41);}
+ }
+
+ @RandomlyFails public static class Five extends NbTestCase {
+ public Five(String n) {super(n);}
+ public void testUnreliable() {runs.add(50);}
+ }
+
+ public static class Six {
+ public static Test suite() {return new NbTestSuite(Four.class);}
+ }
+
+ public static class Seven {
+ public static Test suite() {return new NbTestSuite(Five.class);}
+ }
+
+ public static class Eight {
+ public static Test suite() {
+ NbTestSuite suite = new NbTestSuite();
+ suite.addTestSuite(Four.class);
+ suite.addTestSuite(Five.class);
+ return suite;
+ }
+ }
+
+ private void run(Class... tests) {
+ runs = new ArrayList();
+ BaseTestRunner runner = new TestRunner();
+ for (Class test : tests) {
+ TestResult result = new TestResult();
+ //result.addListener(new ResultPrinter(System.err));
+ runner.getTest(test.getName()).run(result);
+ assertEquals("failures in " + test, Collections.emptyList(), Collections.list((Enumeration) result.failures()));
+ assertEquals("errors in " + test, Collections.emptyList(), Collections.list((Enumeration) result.errors()));
+ }
+ }
+
+ private void runAll() throws Exception {
+ run(One.class, Two.class, Three.class, Six.class, Seven.class, Eight.class);
+ }
+
+ public void testRegularMode() throws Exception {
+ runAll();
+ assertEquals(Arrays.asList(10, 11, 20, 30, 40, 41, 50, 40, 41, 50), runs);
+ }
+
+ public void testIgnoreRandomFailuresMode() throws Exception {
+ System.setProperty("ignore.random.failures", "true");
+ runAll();
+ assertEquals(Arrays.asList(10, 20, 40, 40), runs);
+ }
+
+ public static class Sub extends Three {
+ public Sub(String n) {super(n);}
+ }
+
+ public void testHeritability() throws Exception {
+ System.setProperty("ignore.random.failures", "false");
+ run(Sub.class);
+ assertEquals(Collections.singletonList(30), runs);
+ System.setProperty("ignore.random.failures", "true");
+ run(Sub.class);
+ assertEquals(Collections.emptyList(), runs);
+ }
+
+}
diff --git a/settings/test/unit/src/org/netbeans/modules/settings/RecognizeInstanceObjectsTest.java b/settings/test/unit/src/org/netbeans/modules/settings/RecognizeInstanceObjectsTest.java
--- a/settings/test/unit/src/org/netbeans/modules/settings/RecognizeInstanceObjectsTest.java
+++ b/settings/test/unit/src/org/netbeans/modules/settings/RecognizeInstanceObjectsTest.java
@@ -41,15 +41,15 @@
package org.netbeans.modules.settings;
-import junit.framework.Test;
-import junit.framework.TestResult;
-import junit.framework.TestSuite;
import org.netbeans.core.startup.layers.NamedFSServicesLookupTest;
-
+import org.netbeans.junit.RandomlyFails;
/** Test finding services from manifest and .instance files using FolderLookup.
* @author Jaroslav Tulach
*/
+// From time to time RecognizeInstanceObjectsTest.testOrderingAttributes fails, no idea why. -jglick
+// And in #417 all the tests deadlocked somewhere in folder ordering, apparently at random.
+@RandomlyFails
public class RecognizeInstanceObjectsTest extends NamedFSServicesLookupTest{
public RecognizeInstanceObjectsTest(String name) {
super(name);
@@ -60,19 +60,4 @@
return 20000;
}
- public static Test suite() {
- // From time to time RecognizeInstanceObjectsTest.testOrderingAttributes fails, no idea why. -jglick
- // And in #417 all the tests deadlocked somewhere in folder ordering, apparently at random.
- if (Boolean.getBoolean("ignore.random.failures")) {
- return new Test() {
- public int countTestCases() {
- return 0;
- }
- public void run(TestResult r) {}
- };
- } else {
- return new TestSuite(RecognizeInstanceObjectsTest.class);
- }
- }
-
}
diff --git a/settings/test/unit/src/org/netbeans/modules/settings/convertors/LayersTest.java b/settings/test/unit/src/org/netbeans/modules/settings/convertors/LayersTest.java
--- a/settings/test/unit/src/org/netbeans/modules/settings/convertors/LayersTest.java
+++ b/settings/test/unit/src/org/netbeans/modules/settings/convertors/LayersTest.java
@@ -51,6 +51,7 @@
import java.util.logging.Level;
import org.netbeans.junit.Log;
import org.netbeans.junit.NbTestCase;
+import org.netbeans.junit.RandomlyFails;
import org.openide.filesystems.FileObject;
import org.openide.filesystems.FileSystem;
import org.openide.filesystems.Repository;
@@ -107,14 +108,11 @@
}
}
+ // This is supposed to be run in ide mode.
+ // It is meaningless if run in code mode,
+ // and it will furthermore fail if the internet connection is down.
+ @RandomlyFails
public void testCorrectContentOfSettingsFiles() throws Exception {
- if (Boolean.getBoolean("ignore.random.failures")) {
- // This is supposed to be run in ide mode.
- // It is meaningless if run in code mode,
- // and it will furthermore fail if the internet connection is down.
- return;
- }
-
ClassLoader l = Lookup.getDefault().lookup(ClassLoader.class);
assertNotNull ("In the IDE mode, there always should be a classloader", l);