[GR-49557] Implement multiprocessing spawn backend

PullRequest: graalpython/3025

Author: msimacek

CHANGELOG.md

@@ -14,6 +14,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Support installing some packages with native extensions on Windows. Simple packages like `ujson` or `kiwisolver` will now work when installed from a venv inside a Visual Studio command prompt.
 * Raise `KeyboardInterrupt` exception when the process is interrupted. Enabled only when using the launcher.
 * Add option `python.InitialLocale` to change the default locale. If not set, then Java Locale#getDefault is used.
+* `multiprocessing` module now uses the `spawn` method (creates new processes) by default. The formerly default method that uses threads and multiple Truffle contexts can be selected using `multiprocessing.set_start_method('graalpy')`.
 
 ## Version 23.1.0
 * Oracle GraalPy distributions (previously known as GraalPy Enterprise) are now available under the [GFTC license](https://www.oracle.com/downloads/licenses/graal-free-license.html). The community builds published on Github have been renamed to `graalpy-community-<version>-<os>-<arch>.tar.gz`.

graalpython/com.oracle.graal.python.test/src/tests/test_multiprocessing.py

@@ -37,41 +37,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import multiprocessing
-import sys
-import time
-from multiprocessing.connection import wait
-
-
-def test_wait_timeout():
-    timeout = 3
-    a, b = multiprocessing.Pipe()
-    x, y = multiprocessing.connection.Pipe(False)  # Truffle multiprocessing pipe
-    for fds in [[a, b], [x, y], [a, b, x, y]]:
-        start = time.monotonic()
-        res = wait(fds, timeout)
-        delta = time.monotonic() - start
-        assert not res
-        assert delta < timeout * 2
-        assert delta > timeout / 2
 
-
-def test_wait():
-    a, b = multiprocessing.Pipe()
-    x, y = multiprocessing.connection.Pipe(False)  # Truffle multiprocessing pipe
-    a.send(42)
-    res = wait([b, y], 3)
-    assert res == [b], "res1"
-    assert b.recv() == 42, "res2"
-    y.send(33)
-    res = wait([b, x], 3)
-    assert res == [x], "res3"
-    assert x.recv() == 33, "res4"
-    a.send(1)
-    y.send(2)
-    res = wait([b, x], 3)
-    assert set(res) == set([b, x])
-    assert b.recv() == 1
-    assert x.recv() == 2
+import sys
 
 
 def test_array_read():

…al.python.test/src/tests/test_semlock.py …rc/tests/test_multiprocessing_graalpy.pygraalpython/com.oracle.graal.python.test/src/tests/test_semlock.py renamed to graalpython/com.oracle.graal.python.test/src/tests/test_multiprocessing_graalpy.py graalpython/com.oracle.graal.python.test/src/tests/test_semlock.py renamed to graalpython/com.oracle.graal.python.test/src/tests/test_multiprocessing_graalpy.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -36,11 +36,53 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import multiprocessing
+from multiprocessing.connection import wait
 
-def test_SemLock_raises_on_non_string_name():    
-    from _multiprocessing import SemLock
-    try :
-        SemLock(kind=1, value=1, name={1:2}, maxvalue=1, unlink=1)
-    except TypeError:
-        raised = True
-    assert raised
+import sys
+import time
+
+if sys.implementation.name == 'graalpy':
+    multiprocessing.set_start_method('graalpy', force=True)
+
+
+    def test_SemLock_raises_on_non_string_name():
+        from _multiprocessing_graalpy import SemLock
+        try:
+            SemLock(kind=1, value=1, name={1: 2}, maxvalue=1, unlink=1)
+        except TypeError:
+            pass
+        else:
+            assert False
+
+
+    def test_wait_timeout():
+        timeout = 3
+        a, b = multiprocessing.Pipe()
+        x, y = multiprocessing.connection.Pipe(False)  # Truffle multiprocessing pipe
+        for fds in [[a, b], [x, y], [a, b, x, y]]:
+            start = time.monotonic()
+            res = wait(fds, timeout)
+            delta = time.monotonic() - start
+            assert not res
+            assert delta < timeout * 2
+            assert delta > timeout / 2
+
+
+    def test_wait():
+        a, b = multiprocessing.Pipe()
+        x, y = multiprocessing.connection.Pipe(False)  # Truffle multiprocessing pipe
+        a.send(42)
+        res = wait([b, y], 3)
+        assert res == [b], "res1"
+        assert b.recv() == 42, "res2"
+        y.send(33)
+        res = wait([b, x], 3)
+        assert res == [x], "res3"
+        assert x.recv() == 33, "res4"
+        a.send(1)
+        y.send(2)
+        res = wait([b, x], 3)
+        assert set(res) == {b, x}
+        assert b.recv() == 1
+        assert x.recv() == 2

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -64,7 +64,9 @@
     (None, "arm", "test_smtplib"),
     (None, "aarch64", "test_imaplib"),
     (None, "arm", "test_multiprocessing_spawn"),
+    (None, "arm", "test_multiprocessing_graalpy"),
     (None, "aarch64", "test_multiprocessing_spawn"),
+    (None, "aarch64", "test_multiprocessing_graalpy"),
 )
 
 

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_concurrent_futures.txt

@@ -70,4 +70,31 @@
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolShutdownTest.test_shutdown_no_wait
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolShutdownTest.test_submit_after_interpreter_shutdown
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_20369
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_all_completed
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_first_completed
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_first_completed_some_already_completed
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_first_exception
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_first_exception_one_already_failed
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_first_exception_some_already_complete
+*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnWaitTest.test_timeout
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolAsCompletedTest.test_correct_timeout_exception_msg
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolAsCompletedTest.test_duplicate_futures
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolAsCompletedTest.test_no_timeout
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolAsCompletedTest.test_zero_timeout
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_default_workers
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_executor_map_current_future_cancel
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_idle_thread_reuse
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_map
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_map_exception
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_map_submits_without_iteration
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_map_timeout
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_max_workers_negative
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_no_stale_references
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_saturation
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_shutdown_race_issue12456
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_submit
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolExecutorTest.test_submit_keyword
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolFailingInitializerTest.test_initializer
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolInitializerTest.test_initializer
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolShutdownTest.test_cancel_futures
+*graalpython.lib-python.3.test.test_concurrent_futures.ThreadPoolShutdownTest.test_context_manager_shutdown