Merge branch 'master' into update/GR-21590/190321

Author: msimacek

CHANGELOG.md

@@ -5,6 +5,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 21.1.0
 
+* Support multi-threading with a global interpreter lock by default.
 * Added SSL/TLS support (the `ssl` module)
 * Added subclassing of Java classes in JVM mode
 * Use native posix functions in the GraalPython Launcher

docs/contributor/IMPLEMENTATION_DETAILS.md

@@ -198,4 +198,53 @@ this order:
 * `P.patch`
 
 The same applies to `*.dir` files, and the search is independent of the search
-for the `*.patch` file, i.e., multiple versions of `*.patch` may share one `*.dir`.
+for the `*.patch` file, i.e., multiple versions of `*.patch` may share one `*.dir`.
+
+## The GIL
+
+We always run with a GIL, because C extensions in CPython expect to do so and
+are usually not written to be reentrant. The reason to always have the GIL
+enabled is that when using Python, at least Sulong/LLVM is always available in
+the same context and we cannot know if someone may be using that (or another
+polyglot language or the Java host interop) to start additional threads that
+could call back into Python. This could legitimately happen in C extensions when
+the C extension authors use knowledge of how CPython works to do something
+GIL-less in a C thread that is fine to do on CPython's data structures, but not
+for ours.
+
+Suppose we were running GIL-less until a second thread appears. There is now two
+options: the second thread immediately takes the GIL, but both threads might be
+executing in parallel for a little while before the first thread gets to the
+next safepoint where they try to release and re-acquire the GIL. Option two is
+that we block the second thread on some other semaphore until the first thread
+has acquired the GIL. This scenario may deadlock if the first thread is
+suspended in a blocking syscall. This could be a legitimate use case when
+e.g. one thread is supposed to block on a `select` call that the other thread
+would unblock by operating on the selected resource. Now the second thread
+cannot start to run because it is waiting for the first thread to acquire the
+GIL. To get around this potential deadlock, we would have to "remember" around
+blocking calls that the first thread would have released the GIL before and
+re-acquired it after that point. Since this is equivalent to just releasing and
+re-acquiring the GIL, we might as well always do that.
+
+The implication of this is that we may have to acquire and release the GIL
+around all library messages that may be invoked without already holding the
+GIL. The pattern here is, around every one of those messages:
+
+```
+@ExportMessage xxx(..., @Cached GilNode gil) {
+    boolean mustRelease = gil.acquire();
+    try {
+        ...
+    } finally {
+        gil.release(mustRelease);
+    }
+}
+```
+
+The `GilNode` when used in this pattern ensures that we only release the GIL if
+we acquired it before. The `GilNode` internally uses profiles to ensure that if
+we are always running single threaded or always own the GIL already when we get
+to a specific message we never emit a boundary call to lock and unlock the
+GIL. This implies that we may deopt in some places if, after compiling some
+code, we later start a second thread.

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/ThreadPoolTests.java

@@ -44,7 +44,7 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
-import java.util.Collections;
+import java.util.HashMap;
 
 import org.junit.Test;
 
@@ -70,7 +70,7 @@ public void threadPool() {
                         "print(res)\n";
         final ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
         final PrintStream printStream = new PrintStream(byteArray);
-        PythonTests.runScript(Collections.singletonMap("python.WithThread", "true"), new String[0], source, printStream, System.err, () -> PythonTests.closeContext());
+        PythonTests.runScript(new HashMap<>(), new String[0], source, printStream, System.err, () -> PythonTests.closeContext());
         String result = byteArray.toString().replaceAll("\r\n", "\n");
         assertEquals("[True, True, True, True, True, True, True, True, True, True]\n", result);
     }

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py

@@ -37,11 +37,11 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import sys
-import time
-import gc
 import itertools
+import sys
+
 from . import CPyExtTestCase, CPyExtFunction, CPyExtType, unhandled_error_compare_with_message, unhandled_error_compare
+
 __dir__ = __file__.rpartition("/")[0]
 
 indices = (-2, 2, 10)
@@ -407,11 +407,6 @@ def test_memoryview_acquire_release(self):
         assert mv2[3] == 126
         mv2.release()
         assert mv3[2] == 126
-        del mv1
-        del mv3
-        for i in range(120):
-            gc.collect()
-            if obj.get_bufcount() == 0:
-                break
-            time.sleep(1)
+        mv1.release()
+        mv3.release()
         assert obj.get_bufcount() == 0

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_support.txt

@@ -12,7 +12,6 @@
 *graalpython.lib-python.3.test.test_support.TestSupport.test_check_syntax_error
 *graalpython.lib-python.3.test.test_support.TestSupport.test_detect_api_mismatch
 *graalpython.lib-python.3.test.test_support.TestSupport.test_detect_api_mismatch__ignore
-*graalpython.lib-python.3.test.test_support.TestSupport.test_find_unused_port
 *graalpython.lib-python.3.test.test_support.TestSupport.test_forget
 *graalpython.lib-python.3.test.test_support.TestSupport.test_gc_collect
 *graalpython.lib-python.3.test.test_support.TestSupport.test_get_attribute

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_thread.txt

@@ -1,3 +1,23 @@
 *graalpython.lib-python.3.test.test_thread.BarrierTest.test_barrier
+*graalpython.lib-python.3.test.test_thread.LockTests.test_acquire_contended
+*graalpython.lib-python.3.test.test_thread.LockTests.test_acquire_destroy
+*graalpython.lib-python.3.test.test_thread.LockTests.test_acquire_release
+*graalpython.lib-python.3.test.test_thread.LockTests.test_constructor
+*graalpython.lib-python.3.test.test_thread.LockTests.test_different_thread
+*graalpython.lib-python.3.test.test_thread.LockTests.test_locked_repr
+*graalpython.lib-python.3.test.test_thread.LockTests.test_reacquire
+*graalpython.lib-python.3.test.test_thread.LockTests.test_repr
+*graalpython.lib-python.3.test.test_thread.LockTests.test_state_after_timeout
+*graalpython.lib-python.3.test.test_thread.LockTests.test_thread_leak
+*graalpython.lib-python.3.test.test_thread.LockTests.test_timeout
+*graalpython.lib-python.3.test.test_thread.LockTests.test_try_acquire
+*graalpython.lib-python.3.test.test_thread.LockTests.test_try_acquire_contended
+*graalpython.lib-python.3.test.test_thread.LockTests.test_weakref_deleted
+*graalpython.lib-python.3.test.test_thread.LockTests.test_weakref_exists
+*graalpython.lib-python.3.test.test_thread.LockTests.test_with
 *graalpython.lib-python.3.test.test_thread.TestForkInThread.test_forkinthread
+*graalpython.lib-python.3.test.test_thread.ThreadRunningTests.test__count
 *graalpython.lib-python.3.test.test_thread.ThreadRunningTests.test_nt_and_posix_stack_size
+*graalpython.lib-python.3.test.test_thread.ThreadRunningTests.test_stack_size
+*graalpython.lib-python.3.test.test_thread.ThreadRunningTests.test_starting_threads
+*graalpython.lib-python.3.test.test_thread.ThreadRunningTests.test_unraisable_exception