[GR-22298] Properly implement 'select.select' for 'os.pipe' pipes.

fangerer · fangerer · commit 1adc69218481 · 2020-04-09T19:20:33.000Z
PullRequest: graalpython/880
diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/PythonTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/PythonTests.java
@@ -45,6 +45,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Map;
 
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Engine;
@@ -97,10 +99,14 @@ public class PythonTests {
     }
 
     public static void enterContext(String... newArgs) {
+        enterContext(Collections.emptyMap(), newArgs);
+    }
+
+    public static void enterContext(Map<String, String> options, String[] args) {
         PythonTests.outArray.reset();
         PythonTests.errArray.reset();
         Context prevContext = context;
-        context = Context.newBuilder().engine(engine).allowExperimentalOptions(true).allowAllAccess(true).arguments("python", newArgs).option("python.Executable", executable).build();
+        context = Context.newBuilder().engine(engine).allowExperimentalOptions(true).allowAllAccess(true).options(options).arguments("python", args).option("python.Executable", executable).build();
         context.initialize("python");
         if (prevContext != null) {
             closeContext(prevContext);
@@ -333,6 +339,15 @@ public static Value runScript(String[] args, File path, OutputStream out, Output
         }
     }
 
+    public static Value runScript(Map<String, String> options, String[] args, String source, OutputStream out, OutputStream err) {
+        try {
+            enterContext(options, args);
+            return context.eval(org.graalvm.polyglot.Source.create("python", source));
+        } finally {
+            flush(out, err);
+        }
+    }
+
     public static Value runScript(String[] args, String source, OutputStream out, OutputStream err) {
         try {
             enterContext(args);
@@ -352,8 +367,12 @@ public static Value runScript(String[] args, org.graalvm.polyglot.Source source,
     }
 
     public static Value runScript(String[] args, String source, OutputStream out, OutputStream err, Runnable cb) {
+        return runScript(Collections.emptyMap(), args, source, out, err, cb);
+    }
+
+    public static Value runScript(Map<String, String> options, String[] args, String source, OutputStream out, OutputStream err, Runnable cb) {
         try {
-            enterContext(args);
+            enterContext(options, args);
             return context.eval(org.graalvm.polyglot.Source.create("python", source));
         } finally {
             cb.run();
diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/ThreadPoolTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/module/ThreadPoolTests.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.module;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.Collections;
+
+import org.junit.Test;
+
+import com.oracle.graal.python.test.PythonTests;
+
+public class ThreadPoolTests {
+
+    @Test
+    public void threadPool() {
+        String source = "import _sysconfig\n" +
+                        "assert _sysconfig.get_config_vars().get('WITH_THREAD'), 'context was not started for threading'\n" +
+                        "from multiprocessing.pool import ThreadPool\n" +
+                        "\n" +
+                        "def fun(item):\n" +
+                        "    return item != None\n" +
+                        "\n" +
+                        "items = list(range(0, 10))\n" +
+                        "\n" +
+                        "pool = ThreadPool(2)\n" +
+                        "res = list(pool.imap(fun, items))\n" +
+                        "pool.close()\n" +
+                        "\n" +
+                        "print(res)\n";
+        final ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
+        final PrintStream printStream = new PrintStream(byteArray);
+        PythonTests.runScript(Collections.singletonMap("python.WithThread", "true"), new String[0], source, printStream, System.err, () -> PythonTests.closeContext());
+        String result = byteArray.toString().replaceAll("\r\n", "\n");
+        assertEquals("[True, True, True, True, True, True, True, True, True, True]\n", result);
+    }
+}
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SelectModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SelectModuleBuiltins.java
@@ -40,20 +40,43 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
-import java.io.PrintStream;
+import java.io.IOException;
+import java.nio.channels.Channel;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
 import java.util.List;
 
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
+import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.exception.OSErrorEnum;
+import com.oracle.graal.python.builtins.objects.function.PArguments;
+import com.oracle.graal.python.builtins.objects.list.PList;
+import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
 import com.oracle.graal.python.builtins.objects.tuple.PTuple;
+import com.oracle.graal.python.nodes.PGuards;
+import com.oracle.graal.python.nodes.SpecialMethodNames;
+import com.oracle.graal.python.nodes.builtins.ListNodes.FastConstructListNode;
+import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
-import com.oracle.graal.python.runtime.PythonOptions;
+import com.oracle.graal.python.nodes.util.CoerceToDoubleNode;
+import com.oracle.graal.python.nodes.util.CoerceToFileDescriptorNode;
+import com.oracle.graal.python.runtime.sequence.PSequence;
+import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
+import com.oracle.truffle.api.CompilerDirectives.ValueType;
+import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.library.CachedLibrary;
+import com.oracle.truffle.api.nodes.ControlFlowException;
 
 @CoreFunctions(defineModule = "select")
 public class SelectModuleBuiltins extends PythonBuiltins {
@@ -65,13 +88,184 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
     @Builtin(name = "select", minNumOfPositionalArgs = 3, parameterNames = {"rlist", "wlist", "xlist", "timeout"})
     @GenerateNodeFactory
     abstract static class SelectNode extends PythonBuiltinNode {
-        @Specialization
+
+        @Specialization(limit = "3")
+        PTuple doWithoutTimeout(VirtualFrame frame, Object rlist, Object wlist, Object xlist, @SuppressWarnings("unused") PNone timeout,
+                        @CachedLibrary("rlist") PythonObjectLibrary rlistLibrary,
+                        @CachedLibrary("wlist") PythonObjectLibrary wlistLibrary,
+                        @CachedLibrary("xlist") PythonObjectLibrary xlistLibrary,
+                        @Cached CoerceToDoubleNode coerceToDoubleNode,
+                        @Cached("createGetItem()") LookupAndCallBinaryNode callGetItemNode,
+                        @Cached FastConstructListNode constructListNode,
+                        @Cached CoerceToFileDescriptorNode coerceToFDNode) {
+            return doGeneric(frame, rlist, wlist, xlist, PNone.NONE, rlistLibrary, wlistLibrary, xlistLibrary, coerceToDoubleNode, callGetItemNode, constructListNode, coerceToFDNode);
+        }
+
+        @Specialization(replaces = "doWithoutTimeout", limit = "3")
+        PTuple doGeneric(VirtualFrame frame, Object rlist, Object wlist, Object xlist, Object timeout,
+                        @CachedLibrary("rlist") PythonObjectLibrary rlistLibrary,
+                        @CachedLibrary("wlist") PythonObjectLibrary wlistLibrary,
+                        @CachedLibrary("xlist") PythonObjectLibrary xlistLibrary,
+                        @Cached CoerceToDoubleNode coerceToDoubleNode,
+                        @Cached("createGetItem()") LookupAndCallBinaryNode callGetItemNode,
+                        @Cached FastConstructListNode constructListNode,
+                        @Cached CoerceToFileDescriptorNode coerceToFDNode) {
+
+            ChannelFD[] readFDs;
+            ChannelFD[] writeFDs;
+            ChannelFD[] xFDs;
+            try {
+                readFDs = seq2set(frame, rlist, rlistLibrary, coerceToFDNode, callGetItemNode, constructListNode);
+                writeFDs = seq2set(frame, wlist, wlistLibrary, coerceToFDNode, callGetItemNode, constructListNode);
+                xFDs = seq2set(frame, xlist, xlistLibrary, coerceToFDNode, callGetItemNode, constructListNode);
+            } catch (NonSelectableChannel e) {
+                // If one of the channels is not selectable, we do what we did before: just return
+                // everything.
+                return factory().createTuple(new Object[]{rlist, wlist, xlist});
+            }
+
+            // IMPORTANT: The meaning of the timeout value is slightly different:
+            // 'timeout == 0.0' means we should not block and return immediately. However, the Java
+            // API does not allow a non-blocking select. So we set the timeout to 1 ms.
+            //
+            // 'timeout == None' means we should wait indefinitely, i.e., we need to pass 0 to the
+            // Java API.
+            long timeoutMillis;
+            if (!PGuards.isPNone(timeout)) {
+                double timeoutSecs = coerceToDoubleNode.execute(frame, timeout);
+                timeoutMillis = timeoutSecs != 0.0 ? (long) (timeoutSecs * 1000.0) : 1L;
+            } else {
+                timeoutMillis = 0;
+            }
+
+            if (timeoutMillis < 0) {
+                throw raise(PythonBuiltinClassType.ValueError, "timeout must be non-negative");
+            }
+
+            try {
+                doSelect(readFDs, writeFDs, xFDs, timeoutMillis);
+            } catch (ClosedChannelException e) {
+                // If the channel was closed (this can only happen concurrently between resolving
+                // the FD to the channel and registration), we provided an incorrect file
+                // descriptor. The errno code for that is EBADF.
+                throw raiseOSError(frame, OSErrorEnum.EBADF);
+            } catch (IOException e) {
+                throw raiseOSError(frame, e);
+            } catch (RuntimeException e) {
+                throw raise(PythonBuiltinClassType.SystemError, e);
+            }
+
+            return factory().createTuple(new PList[]{toList(readFDs), toList(writeFDs), toList(xFDs)});
+        }
+
         @TruffleBoundary
-        PTuple select(Object rlist, Object wlist, Object xlist, @SuppressWarnings("unused") Object timeout) {
-            if (getContext().getOption(PythonOptions.VerboseFlag)) {
-                new PrintStream(getContext().getEnv().err()).println("select() will always return immediately, we only support blocking I/O for now");
+        private static void doSelect(ChannelFD[] readFDs, ChannelFD[] writeFDs, ChannelFD[] xFDs, long timeoutMillis) throws IOException {
+            Selector selector = Selector.open();
+
+            for (ChannelFD readFD : readFDs) {
+                readFD.channel.configureBlocking(false);
+                readFD.channel.register(selector, SelectionKey.OP_READ);
             }
-            return factory().createTuple(new Object[]{rlist, wlist, xlist});
+
+            for (ChannelFD writeFD : writeFDs) {
+                writeFD.channel.configureBlocking(false);
+                writeFD.channel.register(selector, SelectionKey.OP_WRITE);
+            }
+
+            for (ChannelFD xFD : xFDs) {
+                // TODO(fa): not sure if these ops are representing
+                // "exceptional condition pending"
+                xFD.channel.configureBlocking(false);
+                xFD.channel.register(selector, SelectionKey.OP_ACCEPT | SelectionKey.OP_CONNECT);
+            }
+
+            int selected = selector.select(timeoutMillis);
+
+            // remove non-selected channels from given lists
+            int deleted = 0;
+            for (int i = 0; i < readFDs.length; i++) {
+                ChannelFD readFD = readFDs[i];
+                SelectionKey selectionKey = readFD.channel.keyFor(selector);
+                if (!selectionKey.isReadable()) {
+                    readFDs[i] = null;
+                    deleted++;
+                }
+            }
+
+            for (int i = 0; i < writeFDs.length; i++) {
+                ChannelFD writeFD = writeFDs[i];
+                SelectionKey selectionKey = writeFD.channel.keyFor(selector);
+                if (!selectionKey.isWritable()) {
+                    writeFDs[i] = null;
+                    deleted++;
+                }
+            }
+
+            for (int i = 0; i < xFDs.length; i++) {
+                ChannelFD xFD = xFDs[i];
+                SelectionKey selectionKey = xFD.channel.keyFor(selector);
+                if (!(selectionKey.isAcceptable() || selectionKey.isConnectable())) {
+                    xFDs[i] = null;
+                    deleted++;
+                }
+            }
+            assert selected == (readFDs.length + writeFDs.length + xFDs.length) - deleted;
+        }
+
+        private ChannelFD[] seq2set(VirtualFrame frame, Object sequence, PythonObjectLibrary lib, CoerceToFileDescriptorNode coerceToFDNode, LookupAndCallBinaryNode callGetItemNode,
+                        FastConstructListNode constructListNode) {
+            int len = lib.lengthWithState(sequence, PArguments.getThreadState(frame));
+            ChannelFD[] result = new ChannelFD[len];
+
+            PSequence pSequence = constructListNode.execute(sequence);
+
+            for (int i = 0; i < len; i++) {
+                int fd = coerceToFDNode.execute(frame, callGetItemNode.executeObject(frame, pSequence, i));
+                Channel fileChannel = getContext().getResources().getFileChannel(fd);
+                if (!(fileChannel instanceof SelectableChannel)) {
+                    throw NonSelectableChannel.INSTANCE;
+                }
+                result[i] = new ChannelFD(fd, (SelectableChannel) fileChannel);
+            }
+            return result;
+        }
+
+        private PList toList(ChannelFD[] arr) {
+            int cnt = 0;
+            for (ChannelFD channelFD : arr) {
+                if (channelFD != null) {
+                    cnt++;
+                }
+            }
+            int[] fds = new int[cnt];
+            for (ChannelFD channelFD : arr) {
+                if (channelFD != null) {
+                    fds[fds.length - (cnt--)] = channelFD.fd;
+                }
+            }
+            return factory().createList(new IntSequenceStorage(fds));
+        }
+
+        static LookupAndCallBinaryNode createGetItem() {
+            return LookupAndCallBinaryNode.create(SpecialMethodNames.__GETITEM__);
         }
+
+        @ValueType
+        private static final class ChannelFD {
+            private final int fd;
+            private final SelectableChannel channel;
+
+            private ChannelFD(int fd, SelectableChannel channel) {
+                this.fd = fd;
+                this.channel = channel;
+            }
+        }
+
+        private static final class NonSelectableChannel extends ControlFlowException {
+            private static final long serialVersionUID = 1L;
+
+            static final NonSelectableChannel INSTANCE = new NonSelectableChannel();
+        }
+
     }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/SpecialMethodNames.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/SpecialMethodNames.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -164,6 +164,7 @@ public abstract class SpecialMethodNames {
     public static final String DECODE = "decode";
     public static final String __SIZEOF__ = "__sizeof__";
     public static final String __CLASS_GETITEM__ = "__class_getitem__";
+    public static final String FILENO = "fileno";
 
     public static final String RICHCMP = "__truffle_richcompare__";
     public static final String TRUFFLE_SOURCE = "__truffle_source__";
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/util/CoerceToFileDescriptorNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/util/CoerceToFileDescriptorNode.java
diff --git a/graalpython/lib-graalpython/_collections.py b/graalpython/lib-graalpython/_collections.py
