[GR-16295] Parallelize test execution for tagged unittests

PullRequest: graalpython/537

Author: timfel

graalpython/com.oracle.graal.python.test/src/graalpytest.py

@@ -40,6 +40,8 @@
 #!/usr/bin/env mx python
 import _io
 import sys
+import time
+import _thread
 
 os = sys.modules.get("posix", sys.modules.get("nt", None))
 if os is None:
@@ -51,6 +53,49 @@
 
 verbose = False
 
+print_lock = _thread.RLock()
+class ThreadPool():
+    cnt_lock = _thread.RLock()
+    cnt = 0
+    if os.environ.get(b"ENABLE_THREADED_GRAALPYTEST") == b"true":
+        maxcnt = min(os.cpu_count(), 16)
+        sleep = time.sleep
+        print("Running with %d threads" % maxcnt)
+    else:
+        sleep = lambda x: x
+        maxcnt = 1
+
+    @classmethod
+    def start(self, function):
+        self.acquire_token()
+        def runner():
+            try:
+                function()
+            finally:
+                self.release_token()
+        _thread.start_new_thread(runner, ())
+        self.sleep(0.5)
+
+    @classmethod
+    def acquire_token(self):
+        while True:
+            with self.cnt_lock:
+                if self.cnt < self.maxcnt:
+                    self.cnt += 1
+                    break
+            self.sleep(1)
+
+    @classmethod
+    def release_token(self):
+        with self.cnt_lock:
+            self.cnt -= 1
+
+    @classmethod
+    def shutdown(self):
+        self.sleep(2)
+        while self.cnt > 0:
+            self.sleep(2)
+
 
 def dump_truffle_ast(func):
     try:
@@ -72,7 +117,8 @@ def __init__(self):
 
     def run_safely(self, func, print_immediately=False):
         if verbose:
-            print(u"\n\t\u21B3 ", func.__name__, " ", end="")
+            with print_lock:
+                print(u"\n\t\u21B3 ", func.__name__, " ", end="")
         try:
             func()
         except BaseException as e:
@@ -102,10 +148,12 @@ def run_test(self, func):
             pass
         elif not hasattr(func, "__call__"):
             pass
-        elif self.run_safely(func):
-            self.success()
         else:
-            self.failure()
+            def do_run():
+                r = self.run_safely(func)
+                with print_lock:
+                    self.success() if r else self.failure()
+            ThreadPool.start(do_run)
 
     def success(self):
         self.passed += 1
@@ -318,6 +366,7 @@ def run(self):
                 self.failed += testcase.failed
             if verbose:
                 print()
+        ThreadPool.shutdown()
         print("\n\nRan %d tests (%d passes, %d failures)" % (self.passed + self.failed, self.passed, self.failed))
         for e in self.exceptions:
             print(e)

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -72,10 +72,14 @@ def working_tests():
     return working_tests
 
 
+class TestAllWorkingTests():
+    pass
+
+
 WORKING_TESTS = working_tests()
 for idx, working_test in enumerate(WORKING_TESTS):
     def make_test_func(working_test):
-        def fun():
+        def fun(self):
             cmd = [sys.executable, "-S", "-m", "unittest"]
             for testpattern in working_test[1]:
                 cmd.extend(["-k", testpattern])
@@ -88,7 +92,7 @@ def fun():
         return fun
 
     test_f = make_test_func(working_test)
-    globals()[test_f.__name__] = test_f
+    setattr(TestAllWorkingTests, test_f.__name__, test_f)
     del test_f
 
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -121,6 +121,13 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
 
     private static final Object[] CONTEXT_INSENSITIVE_SINGLETONS = new Object[]{PNone.NONE, PNone.NO_VALUE, PEllipsis.INSTANCE, PNotImplemented.NOT_IMPLEMENTED};
 
+    /*
+     * We need to store this here, because the check is on the language and can come from a thread
+     * that has no context, but we enable or disable threads with a context option. So we store this
+     * here when a context is created.
+     */
+    private Boolean isWithThread = null;
+
     public static int getNumberOfSpecialSingletons() {
         return CONTEXT_INSENSITIVE_SINGLETONS.length;
     }
@@ -153,14 +160,14 @@ protected void finalizeContext(PythonContext context) {
     protected boolean areOptionsCompatible(OptionValues firstOptions, OptionValues newOptions) {
         // internal sources were marked during context initialization
         return (firstOptions.get(PythonOptions.ExposeInternalSources).equals(newOptions.get(PythonOptions.ExposeInternalSources)) &&
+                        // we cache WithThread on the lanugage
+                        firstOptions.get(PythonOptions.WithThread).equals(newOptions.get(PythonOptions.WithThread)) &&
                         // we cache CatchAllExceptions hard on TryExceptNode
                         firstOptions.get(PythonOptions.CatchAllExceptions).equals(newOptions.get(PythonOptions.CatchAllExceptions)));
     }
 
     private boolean areOptionsCompatibleWithPreinitializedContext(OptionValues firstOptions, OptionValues newOptions) {
         return (areOptionsCompatible(firstOptions, newOptions) &&
-                        // we cache WithThread in SysConfigModuleBuiltins
-                        firstOptions.get(PythonOptions.WithThread).equals(newOptions.get(PythonOptions.WithThread)) &&
                         // disabling TRegex has an effect on the _sre Python functions that are
                         // dynamically created
                         firstOptions.get(PythonOptions.WithTRegex).equals(newOptions.get(PythonOptions.WithTRegex)));
@@ -180,6 +187,8 @@ protected boolean patchContext(PythonContext context, Env newEnv) {
 
     @Override
     protected PythonContext createContext(Env env) {
+        assert this.isWithThread == null || this.isWithThread == PythonOptions.isWithThread(env) : "conflicting thread options in the same language!";
+        this.isWithThread = PythonOptions.isWithThread(env);
         ensureHomeInOptions(env);
         Python3Core newCore = new Python3Core(new PythonParserImpl());
         return new PythonContext(this, env, newCore);
@@ -604,7 +613,7 @@ protected boolean isThreadAccessAllowed(Thread thread, boolean singleThreaded) {
         if (singleThreaded) {
             return super.isThreadAccessAllowed(thread, singleThreaded);
         }
-        return PythonOptions.isWithThread();
+        return isWithThread;
     }
 
     @Override

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysConfigModuleBuiltins.java

@@ -67,7 +67,7 @@ public class SysConfigModuleBuiltins extends PythonBuiltins {
     @Override
     public void initialize(PythonCore core) {
         super.initialize(core);
-        STATIC_CONFIG_OPTIONS.put("WITH_THREAD", PythonOptions.isWithThread() ? 1 : 0);
+        STATIC_CONFIG_OPTIONS.put("WITH_THREAD", PythonOptions.isWithThread(core.getContext().getEnv()) ? 1 : 0);
     }
 
     @Override

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/referencetype/ReferenceTypeBuiltins.java

@@ -103,10 +103,15 @@ public int hash(PReferenceType self) {
             return self.getHash();
         }
 
-        @Fallback
-        public int hash(@SuppressWarnings("unused") Object self) {
+        @Specialization(guards = "self.getObject() == null")
+        public int hashGone(@SuppressWarnings("unused") PReferenceType self) {
             throw raise(PythonErrorType.TypeError, "weak object has gone away");
         }
+
+        @Fallback
+        public int hashWrong(@SuppressWarnings("unused") Object self) {
+            throw raise(PythonErrorType.TypeError, "descriptor '__hash__' requires a 'weakref' object but received a '%p'", self);
+        }
     }
 
     // ref.__repr__

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -28,6 +28,7 @@
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.Option;
+import com.oracle.truffle.api.TruffleLanguage.Env;
 
 import org.graalvm.options.OptionCategory;
 import org.graalvm.options.OptionDescriptors;
@@ -162,6 +163,11 @@ public static <T> T getOption(PythonContext context, OptionKey<T> key) {
         return context.getOptions().get(key);
     }
 
+    @TruffleBoundary
+    public static <T> T getOption(Env env, OptionKey<T> key) {
+        return env.getOptions().get(key);
+    }
+
     @TruffleBoundary
     public static int getIntOption(PythonContext context, OptionKey<Integer> key) {
         if (context == null) {
@@ -198,8 +204,8 @@ public static int getMinLazyStringLength() {
         return getOption(PythonLanguage.getContextRef().get(), MinLazyStringLength);
     }
 
-    public static boolean isWithThread() {
-        return getOption(PythonLanguage.getContextRef().get(), WithThread);
+    public static boolean isWithThread(Env env) {
+        return getOption(env, WithThread);
     }
 
     public static boolean getEnableForcedSplits() {

graalpython/lib-graalpython/_thread.py

@@ -55,10 +55,10 @@ def _set_sentinel():
 
 
 def load():
-    import sys
-    filename = sys.graal_python_stdlib_home + ("/_dummy_thread.py")
-    _dummy_thread = __import__(filename, "_thread")
     if not _sysconfig.get_config_vars().get('WITH_THREAD'):
+        import sys
+        filename = sys.graal_python_stdlib_home + ("/_dummy_thread.py")
+        _dummy_thread = __import__(filename, "_thread")
         sys.modules["_thread"] = _dummy_thread
 load()
 del load

mx.graalpython/mx_graalpython.py

@@ -360,8 +360,8 @@ def graalpython_gate_runner(args, tasks):
 
     with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.tagged]) as task:
         if task:
-            with set_env(ENABLE_CPYTHON_TAGGED_UNITTESTS="true"):
-                run_python_unittests(python_gvm(), paths=["test_tagged_unittests.py"])
+            with set_env(ENABLE_CPYTHON_TAGGED_UNITTESTS="true", ENABLE_THREADED_GRAALPYTEST="true"):
+                run_python_unittests(python_gvm(), args=["--python.WithThread=true"], paths=["test_tagged_unittests.py"])
 
     # Unittests on SVM
     with Task('GraalPython tests on SVM', tasks, tags=[GraalPythonTags.svmunit]) as task: