Update CHANGELOG, produce a warning when using native backend without launcher, handle options without the "python" prefix, untag test_repl.test_close_stdin, handle truffle cwd

Author: otethal

CHANGELOG.md

@@ -6,6 +6,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 ## Version 21.1.0
 
 * Added subclassing of Java classes in JVM mode
+* Use native posix functions in the GraalPython Launcher
 
 ## Version 21.0.0
 

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -116,6 +116,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
         List<String> arguments = new ArrayList<>(inputArgs);
         List<String> subprocessArgs = new ArrayList<>();
         programArgs = new ArrayList<>();
+        boolean posixBackendSpecified = false;
         for (int i = 0; i < arguments.size(); i++) {
             String arg = arguments.get(i);
             switch (arg) {
@@ -285,12 +286,15 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                         }
                     } else {
                         if (arg.startsWith("--llvm.") ||
-                                        arg.startsWith("--python.CoreHome") ||
-                                        arg.startsWith("--python.StdLibHome") ||
-                                        arg.startsWith("--python.CAPI") ||
-                                        arg.startsWith("--PosixModuleBackend")) {
+                                        matchesPythonOption(arg, "CoreHome") ||
+                                        matchesPythonOption(arg, "StdLibHome") ||
+                                        matchesPythonOption(arg, "CAPI") ||
+                                        matchesPythonOption(arg, "PosixModuleBackend")) {
                             addRelaunchArg(arg);
                         }
+                        if (matchesPythonOption(arg, "PosixModuleBackend")) {
+                            posixBackendSpecified = true;
+                        }
                         // possibly a polyglot argument
                         unrecognized.add(arg);
                     }
@@ -313,7 +317,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
         if (!subprocessArgs.isEmpty()) {
             subExec(inputArgs, subprocessArgs);
         }
-
+        if (!posixBackendSpecified) {
+            polyglotOptions.put("python.PosixModuleBackend", "native");
+        }
         return unrecognized;
     }
 
@@ -524,7 +530,7 @@ protected void launch(Builder contextBuilder) {
         if (warnOptions == null || warnOptions.isEmpty()) {
             warnOptions = "";
         }
-        String executable = getContextOptionIfSetViaCommandLine("python.Executable");
+        String executable = getContextOptionIfSetViaCommandLine("Executable");
         if (executable != null) {
             contextBuilder.option("python.ExecutableList", executable);
         } else {
@@ -562,13 +568,7 @@ protected void launch(Builder contextBuilder) {
         contextBuilder.option("python.TerminalHeight", Integer.toString(consoleHandler.getTerminalHeight()));
 
         contextBuilder.option("python.CheckHashPycsMode", checkHashPycsMode);
-
-        if (getContextOptionIfSetViaCommandLine("python.PosixModuleBackend") == null && getContextOptionIfSetViaCommandLine("PosixModuleBackend") == null) {
-            // TODO find a proper way to force emulated backend in llvm.managed mode
-            if (getClass() == GraalPythonMain.class) {
-                contextBuilder.option("python.PosixModuleBackend", "native");
-            }
-        }
+        contextBuilder.option("python.RunViaLauncher", "true");
 
         if (multiContext) {
             contextBuilder.engine(Engine.newBuilder().allowExperimentalOptions(true).options(enginePolyglotOptions).build());
@@ -617,12 +617,17 @@ protected void launch(Builder contextBuilder) {
         System.exit(rc);
     }
 
+    private static boolean matchesPythonOption(String arg, String key) {
+        assert !key.startsWith("python.");
+        return arg.startsWith("--python." + key) || arg.startsWith("--" + key);
+    }
+
     private String getContextOptionIfSetViaCommandLine(String key) {
-        if (System.getProperty("polyglot." + key) != null) {
-            return System.getProperty("polyglot." + key);
+        if (System.getProperty("polyglot.python." + key) != null) {
+            return System.getProperty("polyglot.python." + key);
         }
         for (String f : givenArguments) {
-            if (f.startsWith("--" + key)) {
+            if (matchesPythonOption(f, key)) {
                 String[] splits = f.split("=", 2);
                 if (splits.length > 1) {
                     return splits[1];

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_repl.txt

@@ -1,3 +1,2 @@
-*graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_close_stdin
 *graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_multiline_string_parsing
 *graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_no_memory

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIPosixSupport.java

@@ -305,16 +305,29 @@ public NFIPosixSupport(PythonContext context, String nfiBackend) {
         this.cachedFunctions = new AtomicReferenceArray<>(PosixNativeFunction.values().length);
     }
 
+    @Override
+    public void setEnv(Env env) {
+        // Java NIO (and TruffleFile) do not expect/support changing native working directory since
+        // it is inherently thread-unsafe operation. It is not defined how NIO behaves when native
+        // cwd changes, thus we need to prevent TruffleFile from resolving relative paths using
+        // NIO by setting Truffle cwd to a know value. This cannot be done lazily in chdir() because
+        // native cwd is global, but Truffle cwd is per context.
+        // TruffleFile will be unaware of the real working directory and keep resolving against the
+        // original working directory. This should not matter since we do not use TruffleFile for
+        // ordinary I/O when using NFI backend.
+        try {
+            TruffleFile truffleFile = context.getEnv().getInternalTruffleFile(".").getAbsoluteFile();
+            context.getEnv().setCurrentWorkingDirectory(truffleFile);
+        } catch (Exception e) {
+            LOGGER.log(Level.WARNING, "Unable to change Truffle working directory", e);
+        }
+    }
+
     @ExportMessage
     public String getBackend() {
         return nfiBackend;
     }
 
-    @Override
-    public void setEnv(Env env) {
-        // TODO check that env does not attempt to redirect std streams
-    }
-
     @ExportMessage
     public String strerror(int errorCode,
                     @Shared("invoke") @Cached InvokeNativeFunction invokeNode) {
@@ -645,13 +658,6 @@ public void chdir(Object path,
         if (result != 0) {
             throw newPosixException(invokeNode, getErrno(invokeNode));
         }
-        // TODO we don;t need to do this more than once
-        try {
-            TruffleFile truffleFile = context.getEnv().getInternalTruffleFile(".").getAbsoluteFile();
-            context.getEnv().setCurrentWorkingDirectory(truffleFile);
-        } catch (Exception e) {
-            LOGGER.log(Level.WARNING, "Unable to change Truffle working directory", e);
-        }
     }
 
     @ExportMessage

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -586,6 +586,10 @@ private void initalizePosixSupport() {
                     resources = new EmulatedPosixSupport(this);
                     resources.setEnv(env);
                 } else {
+                    if (!getOption(PythonOptions.RunViaLauncher)) {
+                        writeWarning("Native Posix backend is not fully supported when embedding. For example, standard I/O always uses file " +
+                                        "descriptors 0, 1 and 2 regardless of stream redirection specified in Truffle environment");
+                    }
                     result = new NFIPosixSupport(this, option);
                     resources = new EmulatedPosixSupport(this);
                     resources.setEnv(env);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -241,6 +241,9 @@ private PythonOptions() {
     @Option(category = OptionCategory.EXPERT, help = "Max native memory heap size (default: 2 GB).") //
     public static final OptionKey<Long> MaxNativeMemory = new OptionKey<>(1L << 31);
 
+    @Option(category = OptionCategory.EXPERT, help = "Set by the launcher to true (false means that GraalPython is being embedded in an application).") //
+    public static final OptionKey<Boolean> RunViaLauncher = new OptionKey<>(false);
+
     public static final OptionDescriptors DESCRIPTORS = new PythonOptionsOptionDescriptors();
 
     @CompilationFinal(dimensions = 1) private static final OptionKey<?>[] ENGINE_OPTION_KEYS;