[GR-33794] Multiprocessing fixes

PullRequest: graalpython/1961

Author: timfel

CHANGELOG.md

@@ -3,6 +3,18 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 21.3.0
+
+* Remove PYPY_VERSION from our C extension emulation, enabling PyGame 2.0 and other extensions to work out of the box.
+* Intrinsify and optimize more of the core language for better startup and reduced footprint.
+* Implement a new binary compatible backend for HPy 0.0.3, which allows binary HPy wheels to run unmodified on CPython and GraalPython
+* Support the `multiprocessing` module via in-process nested contexts, allowing execution on multiple cores within the same process using the Python multiprocessing API
+* Add support for the `ctypes` module, enabling more native extensions to run that use the ctypes API
+* Fix multiple REPL issues reported on Github, you can now paste blocks of code and use the numpad in the REPL.
+* Make our marshal format compatible with CPython, so binary data can now be exchanged between CPython and GraalPython processes.
+* Make most `socket` module tests pass in native mode by using a native extension, allowing usage of all POSIX socket APIs where before only those supported on Java could be used.
+* Various compatibility fixes to make the `psutil` package work.
+
 ## Version 21.2.0
 
 * Support the `dict` type properly in interop using the new hash interop messages.

graalpython/com.oracle.graal.python.benchmarks/python/meso/multiprocessing-benchmark.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from multiprocessing import Pool
+from time import time
+from itertools import permutations
+from pprint import pprint
+
+
+NGRAMS = 4
+
+
+def jaccard(fileA, fileB):
+    with open(fileA) as f:
+        x = f.read().split()
+        x = set(zip(*[x[i:] for i in range(NGRAMS)]))
+    with open(fileB) as f:
+        y = f.read().split()
+        y = set(zip(*[y[i:] for i in range(NGRAMS)]))
+    intersection_cardinality = len(x & y)
+    union_cardinality = len(x | y)
+    return (intersection_cardinality, union_cardinality, intersection_cardinality / union_cardinality)
+
+
+def parallel_jaccard(pair):
+    return pair, jaccard(*pair)
+
+
+if __name__ == '__main__':
+    import sys, os
+
+    iterations = 50
+    pairs = list(permutations([os.path.join(sys.argv[1], f) for f in os.listdir(sys.argv[1])], 2))
+
+    for i in range(4):
+        print(f"Warmup run #{i}")
+        start = time()
+        for res in map(parallel_jaccard, pairs):
+            print(f"\t{time() - start}s => {res[0]}: {res[1]}", flush=True)
+
+    with Pool(8) as p:
+        for i in range(iterations):
+            start = time()
+            print(f"#{i + 1}:", end="", flush=True)
+            for res in p.imap_unordered(parallel_jaccard, pairs):
+                print(f"\t{time() - start}s => {res[0]}: {res[1]}", flush=True)
+            # for pair in pairs:
+            #     value = jaccard(*pair)
+            #     print(f"\t{time() - start}s => {pair}: {value}", flush=True)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

@@ -357,6 +357,7 @@ private PythonModule getBuiltinModule(String name) {
     @GenerateNodeFactory
     public abstract static class ExecBuiltin extends PythonBuiltinNode {
         @Specialization
+        @TruffleBoundary
         public Object exec(PythonModule pythonModule) {
             final Python3Core core = getCore();
             if (!ImageInfo.inImageBuildtimeCode()) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MultiprocessingModuleBuiltins.java

@@ -361,18 +361,18 @@ Object doGeneric(VirtualFrame frame, Object rlist,
                         @Cached GilNode gil) {
             ArrayBuilder<Integer> notEmpty = new ArrayBuilder<>();
             SharedMultiprocessingData sharedData = getContext().getSharedMultiprocessingData();
-            gil.release(true);
-            try {
-                PSequence pSequence = constructListNode.execute(frame, rlist);
-                for (int i = 0; i < sizeNode.execute(frame, pSequence); i++) {
-                    Object pythonObject = callGetItemNode.executeObject(frame, pSequence, i);
-                    int fd = toInt(castToJava, pythonObject);
+            PSequence pSequence = constructListNode.execute(frame, rlist);
+            for (int i = 0; i < sizeNode.execute(frame, pSequence); i++) {
+                Object pythonObject = callGetItemNode.executeObject(frame, pSequence, i);
+                int fd = toInt(castToJava, pythonObject);
+                gil.release(true);
+                try {
                     if (!sharedData.isBlocking(fd)) {
                         notEmpty.add(fd);
                     }
+                } finally {
+                    gil.acquire();
                 }
-            } finally {
-                gil.acquire();
             }
             return factory().createList(notEmpty.toObjectArray(new Object[0]));
         }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/PythonAbstractObject.java

@@ -113,7 +113,6 @@
 import com.oracle.graal.python.nodes.call.CallNode;
 import com.oracle.graal.python.nodes.call.special.CallBinaryMethodNode;
 import com.oracle.graal.python.nodes.call.special.CallTernaryMethodNode;
-import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
 import com.oracle.graal.python.nodes.call.special.CallVarargsMethodNode;
 import com.oracle.graal.python.nodes.classes.IsSubtypeNode;
 import com.oracle.graal.python.nodes.expression.CastToListExpressionNode.CastToListInteropNode;
@@ -534,7 +533,7 @@ public Object getMembers(boolean includeInternal,
                     @Shared("getClassThis") @Cached GetClassNode getClass,
                     @Cached PyMappingCheckNode checkMapping,
                     @Shared("lookup") @Cached PyObjectLookupAttr lookupKeys,
-                    @Cached CallUnaryMethodNode callKeys,
+                    @Cached CallNode callKeys,
                     @Shared("getItemNode") @Cached PInteropSubscriptNode getItemNode,
                     @Cached SequenceNodes.LenNode lenNode,
                     @Cached TypeNodes.GetMroNode getMroNode,
@@ -557,7 +556,7 @@ public Object getMembers(boolean includeInternal,
                 if (checkMapping.execute(this)) {
                     Object keysMethod = lookupKeys.execute(null, this, KEYS);
                     if (keysMethod != PNone.NO_VALUE) {
-                        PList mapKeys = castToList.executeWithGlobalState(callKeys.executeObject(keysMethod, this));
+                        PList mapKeys = castToList.executeWithGlobalState(callKeys.execute(keysMethod));
                         int len = lenNode.execute(mapKeys);
                         for (int i = 0; i < len; i++) {
                             Object key = getItemNode.execute(mapKeys, i);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/queue/SimpleQueueBuiltins.java

@@ -165,7 +165,7 @@ Object doNoTimeout(PSimpleQueue self, boolean block, @SuppressWarnings("unused")
                 } catch (InterruptedException e) {
                     CompilerDirectives.transferToInterpreter();
                     Thread.currentThread().interrupt();
-                    return PNone.NO_VALUE;
+                    return PNone.NONE;
                 } finally {
                     ensureGil().acquire();
                 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/thread/SemLockBuiltins.java

@@ -57,13 +57,11 @@
 import com.oracle.graal.python.nodes.ErrorMessages;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
-import com.oracle.graal.python.nodes.function.builtins.PythonQuaternaryBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
-import com.oracle.graal.python.nodes.util.CannotCastException;
-import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
 import com.oracle.graal.python.runtime.PythonContext.SharedMultiprocessingData;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -201,19 +199,17 @@ Object doEnter(VirtualFrame frame, AbstractPythonLock self, Object blocking, Obj
     }
 
     @Builtin(name = "_rebuild", minNumOfPositionalArgs = 4, parameterNames = {"handle", "kind", "maxvalue", "name"})
+    @ArgumentClinic(name = "kind", conversion = ArgumentClinic.ClinicConversion.Int)
+    @ArgumentClinic(name = "name", conversion = ArgumentClinic.ClinicConversion.String)
     @GenerateNodeFactory
-    abstract static class RebuildNode extends PythonQuaternaryBuiltinNode {
-        @Specialization
-        Object doEnter(@SuppressWarnings("unused") Object handle, int kind, @SuppressWarnings("unused") Object maxvalue, Object nameObj,
-                        @Cached CastToJavaStringNode castNameNode) {
-
-            String name;
-            try {
-                name = castNameNode.execute(nameObj);
-            } catch (CannotCastException e) {
-                throw raise(PythonBuiltinClassType.TypeError, ErrorMessages.ARG_D_MUST_BE_S_NOT_P, "_rebuild", 4, "str", nameObj);
-            }
+    abstract static class RebuildNode extends PythonClinicBuiltinNode {
+        @Override
+        protected ArgumentClinicProvider getArgumentClinic() {
+            return SemLockBuiltinsClinicProviders.RebuildNodeClinicProviderGen.INSTANCE;
+        }
 
+        @Specialization
+        Object doEnter(@SuppressWarnings("unused") Object handle, int kind, @SuppressWarnings("unused") Object maxvalue, String name) {
             SharedMultiprocessingData multiprocessing = getContext().getSharedMultiprocessingData();
             Semaphore semaphore = multiprocessing.getNamedSemaphore(name);
             if (semaphore == null) {