[GR-44623][GR-44415] Address unsatisfied link errors.

PullRequest: graalpython/2667

Author: timfel

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -1270,7 +1270,8 @@ PyAPI_FUNC(void) initialize_graal_capi(ptr_cache_t _pythonToNative, void_ptr_cac
     initialize_hashes();
     initialize_bufferprocs();
 
-	// TODO: initialize during cext initialization doesn't work at the moment
+    // TODO: initialize during cext initialization doesn't work at the moment
+    // This is hardcoded the same way in capi_native.c
     Py_FileSystemDefaultEncoding = "utf-8"; // strdup(PyUnicode_AsUTF8(GraalPyTruffle_FileSystemDefaultEncoding()));
 
     PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "initialize_graal_capi: %fs", ((double) (clock() - t)) / CLOCKS_PER_SEC);

graalpython/com.oracle.graal.python.jni/src/capi_native.c

@@ -562,3 +562,9 @@ char _PyByteArray_empty_string[] = "";
 #include "complexobject.c"
 #include "dictobject.c"
 #include "modsupport_shared.c"
+
+/*
+ * This mirrors the definition in capi.c that we us on Sulong, and needs to be
+ * fixed when that is.
+ */
+const char *Py_FileSystemDefaultEncoding = "utf-8";

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -617,6 +617,15 @@ protected void launch(Builder contextBuilder) {
             }
         }
 
+        if (isLLVMToolchainLauncher()) {
+            if (!hasContextOptionSetViaCommandLine("UseSystemToolchain")) {
+                contextBuilder.option("python.UseSystemToolchain", "false");
+            }
+            if (!hasContextOptionSetViaCommandLine("NativeModules")) {
+                contextBuilder.option("python.NativeModules", "");
+            }
+        }
+
         if (relaunchArgs != null) {
             Iterator<String> it = origArgs.iterator();
             while (it.hasNext()) {
@@ -886,6 +895,9 @@ protected String getLanguageId() {
 
     @Override
     protected void printHelp(OptionCategory maxCategory) {
+        if (isLLVMToolchainLauncher()) {
+            System.out.println("GraalPy launcher to use LLVM toolchain and Sulong execution of native extensions.\n");
+        }
         System.out.println("usage: python [option] ... (-c cmd | file) [arg] ...\n" +
                         "Options and arguments (and corresponding environment variables):\n" +
                         "-B     : this disables writing .py[co] files on import\n" +
@@ -958,6 +970,11 @@ protected void printHelp(OptionCategory maxCategory) {
                                         "   This is required when starting subprocesses of python.\n" : ""));
     }
 
+    private boolean isLLVMToolchainLauncher() {
+        String exeName = getResolvedExecutableName();
+        return exeName != null && exeName.endsWith("-lt");
+    }
+
     @Override
     protected String[] getDefaultLanguages() {
         return new String[]{getLanguageId(), "llvm", "regex"};
@@ -1186,6 +1203,39 @@ private void subExec(List<String> args, List<String> subProcessDefs) {
         }
     }
 
+    protected String getResolvedExecutableName() {
+        // Note that with thin launchers, both graalpy and graalpy-managed are actual executables
+        // that load and start GraalPy from a shared library
+        if (ImageInfo.inImageRuntimeCode()) {
+            // This is the fastest, most straightforward way to get the name of the actual
+            // executable, i.e., with symlinks resolved all the way down to either graalpy or
+            // graalpy-managed.
+            String executableName = ProcessProperties.getExecutableName();
+            if (executableName != null) {
+                return executableName;
+            }
+        }
+        // The program name can be a symlink, or a command resolved via $PATH
+        // We use getLauncherExecName, which should resolve $PATH commands for us
+        String launcherExecName = getLauncherExecName();
+        if (launcherExecName == null) {
+            return null;
+        }
+        Path programPath = Paths.get(launcherExecName);
+        Path realPath;
+        try {
+            // toRealPath resolves symlinks along the way
+            realPath = programPath.toRealPath();
+        } catch (IOException ex) {
+            throw abort(String.format("Cannot determine the executable name from path: '%s'", programPath), 72);
+        }
+        Path f = realPath.getFileName();
+        if (f == null) {
+            throw abort(String.format("Cannot determine the executable name from path: '%s'", realPath), 73);
+        }
+        return f.toString();
+    }
+
     private List<String> getCmdline(List<String> args, List<String> subProcessDefs) {
         List<String> cmd = new ArrayList<>();
         if (isAOT()) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -229,6 +229,7 @@ public void postInitialize(Python3Core core) {
         PList executableList = PythonObjectFactory.getUncached().createList(arr);
         mod.setAttribute(tsLiteral("executable_list"), executableList);
         mod.setAttribute(tsLiteral("ForeignType"), core.lookupType(PythonBuiltinClassType.ForeignObject));
+        mod.setAttribute(tsLiteral("use_system_toolchain"), context.getOption(PythonOptions.UseSystemToolchain));
 
         if (!context.getOption(PythonOptions.EnableDebuggingBuiltins)) {
             mod.setAttribute(tsLiteral("dump_truffle_ast"), PNone.NO_VALUE);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -391,6 +391,9 @@ private PythonOptions() {
     @Option(category = OptionCategory.INTERNAL, help = "The list of the original command line arguments passed to the Python executable.") //
     public static final OptionKey<TruffleString> OrigArgv = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);
 
+    @Option(category = OptionCategory.EXPERT, help = "If true, use the system's toolchain for native extension compilation. Otherwise, use the LLVM Toolchain included with GraalVM.") //
+    public static final OptionKey<Boolean> UseSystemToolchain = new OptionKey<>(true);
+
     public static final OptionDescriptors DESCRIPTORS = new PythonOptionsOptionDescriptors();
 
     @CompilationFinal(dimensions = 1) private static final OptionKey<?>[] ENGINE_OPTION_KEYS;

graalpython/lib-graalpython/_sysconfig.py

@@ -56,8 +56,8 @@ def _get_posix_vars():
         so_ext = ".so"
     assert _imp.extension_suffixes()[0] == "." + so_abi + so_ext, "mismatch between extension suffix to _imp.extension_suffixes"
 
-    use_native_toolchain = os.environ.get('NATIVE_TOOLCHAIN', '').lower() in {'1', 'true'}
-    if use_native_toolchain:
+    use_system_toolchain = __graalpython__.use_system_toolchain
+    if use_system_toolchain:
         def get_toolchain(name):
             return dict(CC='cc',CXX='c++',AR='ar',RANLIB='ranlib',LD='ld',NM='nm')[name]
     else:
@@ -78,7 +78,7 @@ def get_toolchain(name):
     g['CC'] = get_toolchain('CC')
     g['CXX'] = toolchain_cxx if have_cxx else get_toolchain('CC') + ' --driver-mode=g++'
     opt_flags = ["-DNDEBUG"]
-    if not use_native_toolchain:
+    if not use_system_toolchain:
         opt_flags += ["-stdlib=libc++"]
     g['OPT'] = ' '.join(opt_flags)
     g['INCLUDEPY'] = python_inc
@@ -87,7 +87,7 @@ def get_toolchain(name):
     gnu_source = "-D_GNU_SOURCE=1"
     g['USE_GNU_SOURCE'] = gnu_source
     cflags_default = list(opt_flags)
-    if not use_native_toolchain:
+    if not use_system_toolchain:
         cflags_default += ["-Wno-unused-command-line-argument", "-DGRAALVM_PYTHON_LLVM"]
     if win32_native:
         cflags_default += ["-DMS_WINDOWS", "-DPy_ENABLE_SHARED", "-DHAVE_DECLSPEC_DLL"]

graalpython/lib-python/3/venv/__init__.py

@@ -74,7 +74,8 @@ def create(self, env_dir):
         if self.with_pip:
             self._setup_pip(context)
         # Truffle change
-        self._install_compilers(context)
+        if not __graalpython__.use_system_toolchain:
+            self._install_compilers(context)
         # End of Truffle change
         if not self.upgrade:
             self.setup_scripts(context)
@@ -183,7 +184,7 @@ def create_if_needed(d):
 
 
     def _install_compilers(self, context):
-        """Puts the Graal LLVM compiler tools on the path"""
+        """Puts the Graal LLVM compiler tools on the path."""
         bin_dir = os.path.join(context.env_dir, context.bin_name)
         for (tool_path, names) in __graalpython__.get_toolchain_tools_for_venv().items():
             for name in names:

mx.graalpython/mx_graalpython.py

@@ -1937,7 +1937,7 @@ def verify_ci(dest_suite, common_ci_dir="ci_common", args=None, ext=('.jsonnet',
     ],
     library_configs=[
         mx_sdk.LanguageLibraryConfig(
-            launchers=['bin/<exe:graalpy>', 'bin/<exe:python>', 'bin/<exe:python3>'],
+            launchers=['bin/<exe:graalpy>', 'bin/<exe:graalpy-lt>', 'bin/<exe:python>', 'bin/<exe:python3>'],
             jar_distributions=['graalpython:GRAALPYTHON-LAUNCHER'],
             main_class=GRAALPYTHON_MAIN_CLASS,
             build_args=[
@@ -2381,6 +2381,9 @@ def run(self, args, env=None, cwd=None, **kwargs):
         env.update(mx.dependency("com.oracle.graal.python.cext").getBuildEnv())
         env.update(self.subject.getBuildEnv())
 
+        # we need to use the sulong toolchain for this
+        args.insert(0, "--python.UseSystemToolchain=false")
+
         # distutils will honor env variables CC, CFLAGS, LDFLAGS but we won't allow to change them,
         # besides keeping custom sysroot, since our toolchain forwards to the system headers
         for var in ["CC", "CFLAGS", "LDFLAGS"]: