[GR-48689] Use VirtualFileSystem from embedding utils in standalone native tool.

PullRequest: graalpython/3080

Author: timfel

graalpython/com.oracle.graal.python.test/src/tests/test_standalone.py

@@ -226,7 +226,7 @@ def test_native_executable_one_file():
             f.write("print('hello world, argv[1:]:', sys.argv[1:])")
 
         target_file = os.path.join(tmpdir, "hello")
-        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-m", source_file, "-o", target_file]
+        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-m", source_file, "-o", target_file]
 
         out, return_code = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out
@@ -261,7 +261,7 @@ def test_native_executable_venv_and_one_file():
         out, return_code = run_cmd(cmd, env)
 
         target_file = os.path.join(target_dir, "hello")
-        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", source_file, "--venv", venv_dir, "-o", target_file]
+        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-Os", "-m", source_file, "--venv", venv_dir, "-o", target_file]
         out, return_code = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out
 
@@ -294,7 +294,7 @@ def test_native_executable_module():
             f.write("hello.print_hello()\n")
 
         target_file = os.path.join(tmp_dir, "hello")
-        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-Os", "-m", module_dir, "-o", target_file]
+        cmd = [graalpy, "-m", "standalone", "--verbose", "native", "-ce", "-Os", "-m", module_dir, "-o", target_file]
 
         out, return_code = run_cmd(cmd, env)
         assert "Bundling Python resources into" in out

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/PythonUtils.java

@@ -48,6 +48,7 @@
 import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING;
 import static com.oracle.truffle.api.CompilerDirectives.shouldNotReachHere;
 
+import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
@@ -65,6 +66,7 @@
 import javax.management.ReflectionException;
 
 import org.graalvm.nativeimage.ImageInfo;
+import org.graalvm.nativeimage.VMRuntime;
 import org.graalvm.polyglot.io.ByteSequence;
 
 import com.oracle.graal.python.PythonLanguage;
@@ -525,7 +527,14 @@ public static void forceFullGC() {
 
     @TruffleBoundary
     public static void dumpHeap(String path) {
-        if (SERVER != null) {
+        if (ImageInfo.inImageCode()) {
+            try {
+                VMRuntime.dumpHeap(path, true);
+            } catch (UnsupportedOperationException | IOException e) {
+                System.err.println("Heap dump creation failed." + e.getMessage());
+                e.printStackTrace();
+            }
+        } else if (SERVER != null) {
             try {
                 Class<?> mxBeanClass = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
                 Object mxBean = ManagementFactory.newPlatformMXBeanProxy(SERVER,

graalpython/lib-graalpython/modules/standalone/__main__.py

@@ -54,25 +54,28 @@
 import subprocess
 import sys
 import tempfile
+import pathlib
 import platform
 import socket
 import urllib
 import urllib.request
 import tarfile
 import zipfile
-import glob 
+import glob
 
 assert sys.pycache_prefix is None
 
+# Prefix and filelist match the defaults in org.graalvm.python.embedding.utils.VirtualFileSystem
 VFS_PREFIX = "vfs"
+FILES_LIST_NAME = "fileslist.txt"
+FILES_LIST_PATH = VFS_PREFIX + "/" + FILES_LIST_NAME
+
+# Global configuration
 VFS_HOME = "home"
 VFS_HOME_PREFIX = f"{VFS_PREFIX}/{VFS_HOME}"
 VFS_VENV_PREFIX = VFS_PREFIX + "/venv"
 VFS_PROJ_PREFIX = VFS_PREFIX + "/proj"
 
-VFS_JAVA_FILE = "VirtualFileSystem.java"
-VFS_JAVA_FILE_TEMPLATE = f"resources/{VFS_JAVA_FILE}"
-
 NATIVE_EXEC_LAUNCHER = "Py2BinLauncher"
 NATIVE_EXEC_LAUNCHER_FILE = f"{NATIVE_EXEC_LAUNCHER}.java"
 NATIVE_EXEC_LAUNCHER_TEMPLATE_PATH = f"resources/{NATIVE_EXEC_LAUNCHER_FILE}"
@@ -83,20 +86,17 @@
 GRAALVM_URL_BASE = "https://download.oracle.com/graalvm/"
 
 MVN_REPOSITORY = os.getenv("MVN_REPOSITORY")
-MVN_GRAALPY_ARTEFACT_ID = "python-community"
 MVN_GRAALPY_VERSION = os.getenv("MVN_GRAALPY_VERSION") if os.getenv("MVN_GRAALPY_VERSION") else __graalpython__.get_graalvm_version()
-PYTHON_LANGUAGE_JAR = f"org.graalvm.python-python-language-{MVN_GRAALPY_VERSION}.jar"
 
-FILES_LIST_NAME = "fileslist.txt"
-FILES_LIST_PATH = VFS_PREFIX + "/" + FILES_LIST_NAME
-
 CMD_NATIVE_EXECUTABLE = "native"
 CMD_JAVA_PYTHON_APP = "polyglot_app"
 ATTR_STANDALONE_CMD = "command"
 
+
 def get_file(*paths):
     return os.path.join(os.path.dirname(__file__), *paths)
 
+
 def get_executable(file):
     if os.path.isfile(file):
         return file
@@ -108,6 +108,7 @@ def get_executable(file):
         return exe
     return None
 
+
 def create_polyglot_app(parsed_args):
     if hasattr(parsed_args, "module") and os.path.abspath(parsed_args.output_directory).startswith(os.path.abspath(parsed_args.module)):
         print(
@@ -145,23 +146,39 @@ def create_polyglot_app(parsed_args):
         exit(1)
 
 
-def get_modules_path(target_dir):
-    mp = os.path.join(__graalpython__.home, "graalpy_downloaded_modules")  
+def get_download_dir(parsed_args):
+    subdir = "downloaded_standalone_resources"
+    mp = os.path.join(__graalpython__.home, subdir)  
     try:
         if not os.path.exists(mp):
             os.mkdir(mp)
+        Pathlib(mp).touch() # Ensure if we can write to that location
+        parsed_args.keep_temp = True # Keep this location
         return mp
     except Exception as e:
-        pass
-    return os.path.join(target_dir, "modules")
+        if parsed_args.verbose:
+            print("Cannot store native standalone dependencies permanently, storing to tmpdir")
+            import traceback
+            traceback.print_exception(e)
+        mp = os.path.join(tempfile.mkdtemp(), subdir)
+        os.mkdir(mp)
+        return mp
+
 
 def create_native_exec(parsed_args):
-    target_dir = tempfile.mkdtemp()
+    artifacts = ["org.graalvm.python.python-embedding"]
+    if parsed_args.ce:
+        artifacts.append("org.graalvm.polyglot.python-community")
+    else:
+        artifacts.append("org.graalvm.polyglot.python")
+
+    target_dir = get_download_dir(parsed_args)
     try:
         ni, jc = get_tools(target_dir, parsed_args)
 
-        modules_path = get_modules_path(target_dir)
-        download_python(modules_path, parsed_args)
+        modules_path = target_dir
+        for artifact in artifacts:
+            download_maven_artifact(modules_path, artifact, parsed_args)
 
         launcher_file = os.path.join(target_dir, NATIVE_EXEC_LAUNCHER_FILE)    
         create_target_directory(target_dir, launcher_file, parsed_args)
@@ -171,7 +188,8 @@ def create_native_exec(parsed_args):
     finally:
         if not parsed_args.keep_temp:
             shutil.rmtree(target_dir)
-   
+
+
 def index_vfs(target_dir):   
     files_list_path = os.path.join(target_dir, FILES_LIST_PATH)
     dir_to_list = os.path.join(target_dir, VFS_PREFIX)        
@@ -186,16 +204,7 @@ def f(dir_path, names, line_end):
         for (dir_path, dir_names, file_names) in w:
             f(dir_path, dir_names, "/\n")
             f(dir_path, file_names, "\n")
-                
-def create_virtual_filesystem_file(vfs_file):
-    lines = open(get_file(VFS_JAVA_FILE_TEMPLATE), 'r').readlines()
-    with open(vfs_file, 'w') as f:
-        for line in lines:
-            if "{vfs-prefix}" in line:
-                line = line.replace("{vfs-prefix}", VFS_PREFIX)
-            if "{files-list-name}" in line:
-                line = line.replace("{files-list-name}", FILES_LIST_NAME)
-            f.write(line)
+
 
 def create_launcher_file(template, launcher):
     lines = open(template, 'r').readlines()
@@ -209,6 +218,7 @@ def create_launcher_file(template, launcher):
                 line = line.replace("{vfs-proj-prefix}", VFS_PROJ_PREFIX)
             f.write(line)
 
+
 def create_target_directory(target_dir, launcher_file, parsed_args):
     if parsed_args.verbose:
         print(f"Bundling Python resources into {target_dir}")
@@ -222,11 +232,9 @@ def create_target_directory(target_dir, launcher_file, parsed_args):
     os.makedirs(os.path.dirname(launcher_file), exist_ok=True)
     create_launcher_file(get_file(NATIVE_EXEC_LAUNCHER_TEMPLATE_PATH), launcher_file)
 
-    virtual_filesystem_java_file = os.path.join(target_dir, VFS_JAVA_FILE)
-    create_virtual_filesystem_file(virtual_filesystem_java_file)
-
     shutil.copy(get_file(NATIVE_IMAGE_RESOURCES_PATH), os.path.join(target_dir, NATIVE_IMAGE_RESOURCES_FILE))
 
+
 def bundle_python_resources(target_dir, project, venv=None):
     """
     Copy the Python core, stdlib, venv, and module into one folder.
@@ -276,6 +284,7 @@ def bundle_python_resources(target_dir, project, venv=None):
             copy_folder_to_target(target_dir, tmpdir, VFS_PROJ_PREFIX)
             os.unlink(name)
 
+
 def copy_folder_to_target(resource_root, folder, prefix, path_filter=lambda file=None, dir=None: False):
     """
     Store a folder with Python modules.
@@ -292,6 +301,7 @@ def copy_folder_to_target(resource_root, folder, prefix, path_filter=lambda file
             os.makedirs(resource_parent_path, exist_ok=True)
             shutil.copy(fullname, os.path.join(resource_root, arcname))
 
+
 def get_graalvm_url():
     jdk_version = __graalpython__.get_jdk_version()
     if "." in jdk_version:
@@ -321,6 +331,7 @@ def get_graalvm_url():
 
     return f"{GRAALVM_URL_BASE}{major_version}/archive/graalvm-jdk-{jdk_version}_{system}-{machine}_bin.{sufix}"
 
+
 def get_tools(target_dir, parsed_args):
     if os.getenv("JAVA_HOME"):
         graalvm_home = os.getenv("JAVA_HOME")
@@ -372,16 +383,15 @@ def get_tools(target_dir, parsed_args):
 
     return ni, jc
 
-def download_python(modules_path, parsed_args):
-    if os.path.exists((os.path.join(modules_path, PYTHON_LANGUAGE_JAR))):
-        return
-            
+
+def download_maven_artifact(modules_path, artifact, parsed_args):
     mvnd = get_executable(os.path.join(__graalpython__.home, "libexec", "graalpy-polyglot-get"))
     cmd = [mvnd]
 
     if MVN_REPOSITORY:
         cmd += ["-r", MVN_REPOSITORY]
-    cmd += ["-a", MVN_GRAALPY_ARTEFACT_ID]
+    cmd += ["-a", artifact.rsplit(".", 1)[1]]
+    cmd += ["-g", artifact.rsplit(".", 1)[0]]
     cmd += ["-v", MVN_GRAALPY_VERSION]
     cmd += ["-o", modules_path]
     if parsed_args.verbose:
@@ -398,13 +408,14 @@ def download_python(modules_path, parsed_args):
             print(p.stderr.decode())
         exit(1)
 
+
 def build_binary(target_dir, ni, jc, modules_path, launcher_file, parsed_args):
     cwd = os.getcwd()
     output = os.path.abspath(parsed_args.output)
     os.chdir(target_dir)
 
     try:        
-        cmd = [jc, "-cp", f"{modules_path}/*", os.path.join(target_dir, "VirtualFileSystem.java"), launcher_file]
+        cmd = [jc, "-cp", f"{modules_path}/*", launcher_file]
         if parsed_args.verbose:
             print(f"Compiling code for Python standalone entry point: {' '.join(cmd)}")
         p = subprocess.run(cmd, cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -439,6 +450,7 @@ def build_binary(target_dir, ni, jc, modules_path, launcher_file, parsed_args):
     finally:
         os.chdir(cwd)
 
+
 def main(args):
     parser = argparse.ArgumentParser(prog=f"{sys.executable} -m standalone")
     parser.add_argument(
@@ -471,6 +483,9 @@ def main(args):
         metavar="<arg>",
         default=[],
     )
+    parser_bin.add_argument(
+        "-ce", action="store_true", help="Use GraalPy Community Edition instead of Oracle GraalPy"
+    )
 
     parser_app = subparsers.add_parser(
         CMD_JAVA_PYTHON_APP,
@@ -510,5 +525,6 @@ def main(args):
     else :
         create_native_exec(parsed_args)
 
+
 if __name__ == "__main__":
     main(sys.argv[1:])

graalpython/lib-graalpython/modules/standalone/resources/Py2BinLauncher.java

@@ -47,6 +47,7 @@
 import org.graalvm.polyglot.PolyglotException;
 import org.graalvm.polyglot.Source;
 import org.graalvm.polyglot.io.IOAccess;
+import org.graalvm.python.embedding.utils.VirtualFileSystem;
 
 /**
  * A simple launcher for Python. The launcher sets the filesystem up to read the Python core,
@@ -65,10 +66,12 @@ public class Py2BinLauncher {
     private static final String PROJ_PREFIX = "/{vfs-proj-prefix}";
 
     public static void main(String[] args) throws IOException {
-        VirtualFileSystem vfs = new VirtualFileSystem(p -> {
-            String s = p.toString();
-            return s.endsWith(".so") || s.endsWith(".dylib") || s.endsWith(".pyd") || s.endsWith(".dll");
-        });
+        VirtualFileSystem vfs = VirtualFileSystem.newBuilder()
+            .extractFilter(p -> {
+                String s = p.toString();
+                return s.endsWith(".ttf");
+            })
+            .build();
         IOAccess ioAccess = IOAccess.newBuilder().fileSystem(vfs).allowHostSocketAccess(true).build();
         var builder = Context.newBuilder()
                 .allowExperimentalOptions(true)