register the vms and benchmark suites after the suite is initialized

- at this time the mx projects / distributions are available
- extend the GraalPythonVM with extra arguments to extend the classpath

Author: cosminbasca

mx.graalpython/mx_graalpython.py

@@ -37,24 +37,25 @@
 import mx_urlrewrites
 from mx_gate import Task
 from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS
-from mx_graalpython_benchmark import PythonBenchmarkSuite
+from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, GraalPythonVm, \
+    CONFIGURATION_DEFAULT
 from mx_unittest import unittest
 
-_suite = mx.suite('graalpython')
-_mx_graal = mx.suite("compiler", fatalIfMissing=False)
-_sulong = mx.suite("sulong")
+SUITE = mx.suite('graalpython')
+SUITE_COMPILER = mx.suite("compiler", fatalIfMissing=False)
+SUITE_SULONG = mx.suite("sulong")
 
 
 def _get_core_home():
-    return os.path.join(_suite.dir, "graalpython", "lib-graalpython")
+    return os.path.join(SUITE.dir, "graalpython", "lib-graalpython")
 
 
 def _get_stdlib_home():
-    return os.path.join(_suite.dir, "graalpython", "lib-python", "3")
+    return os.path.join(SUITE.dir, "graalpython", "lib-python", "3")
 
 
 def _get_svm_binary():
-    return os.path.join(_suite.dir, "graalpython-svm")
+    return os.path.join(SUITE.dir, "graalpython-svm")
 
 
 def __get_svm_binary_from_graalvm():
@@ -131,7 +132,7 @@ def _extract_graalpython_internal_options(args):
         elif arg == '-force-long':
             internal += ["-Dcom.oracle.graal.python.forceLongType=true"]  # false
 
-        elif arg == '-debug-perf' and _mx_graal:
+        elif arg == '-debug-perf' and SUITE_COMPILER:
             # internal += ['-Dgraal.InliningDepthError=500']
             # internal += ['-Dgraal.EscapeAnalysisIterations=3']
             # internal += ['-XX:JVMCINMethodSizeLimit=1000000']
@@ -149,7 +150,7 @@ def _extract_graalpython_internal_options(args):
             # internal += ['-Dgraal.MaximumLoopExplosionCount=1000']
             # internal += ['-Dgraal.TruffleCompilationThreshold=100000']
 
-        elif arg == '-compile-truffle-immediately' and _mx_graal:
+        elif arg == '-compile-truffle-immediately' and SUITE_COMPILER:
             internal += ['-Dgraal.TruffleCompileImmediately=true']
             internal += ['-Dgraal.TruffleCompilationExceptionsAreThrown=true']
 
@@ -160,7 +161,7 @@ def _extract_graalpython_internal_options(args):
 
 
 def check_vm(vm_warning=True, must_be_jvmci=False):
-    if not _mx_graal:
+    if not SUITE_COMPILER:
         if must_be_jvmci:
             print '** Error ** : graal compiler was not found!'
             sys.exit(1)
@@ -170,7 +171,7 @@ def check_vm(vm_warning=True, must_be_jvmci=False):
 
 
 def get_jdk():
-    if _mx_graal:
+    if SUITE_COMPILER:
         tag = 'jvmci'
     else:
         tag = None
@@ -201,7 +202,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, **kwargs):
     if '--python.WithJavaStacktrace' not in graalpython_args:
         graalpython_args.insert(0, '--python.WithJavaStacktrace')
 
-    if _sulong:
+    if SUITE_SULONG:
         dists.append('SULONG')
         if mx.suite("sulong-managed", fatalIfMissing=False):
             dists.append('SULONG_MANAGED')
@@ -217,9 +218,9 @@ def _is_user(user, home=None):
             return os.environ.get("USER") == user
 
         if _is_user("tim", "MAGLEV_HOME") or _is_user("cbasca") or _is_user("fa"):
-            _suite.import_suite("tools", version=None, urlinfos=None, in_subdir=True)
+            SUITE.import_suite("tools", version=None, urlinfos=None, in_subdir=True)
             dists.append('CHROMEINSPECTOR')
-            if _sulong:
+            if SUITE_SULONG:
                 vm_args.append("-Dpolyglot.llvm.enableLVI=true")
 
     vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk)
@@ -254,7 +255,7 @@ def nativebuild(args):
 
 
 def nativeclean(args):
-    mx.run(['find', _suite.dir, '-name', '*.bc', '-delete'])
+    mx.run(['find', SUITE.dir, '-name', '*.bc', '-delete'])
 
 
 def python3_unittests(args):
@@ -291,17 +292,17 @@ def python_gate(args):
 
 
 def find_jdt():
-    pardir = os.path.abspath(os.path.join(_suite.dir, ".."))
-    for f in [os.path.join(_suite.dir, f) for f in os.listdir(_suite.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:
+    pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))
+    for f in [os.path.join(SUITE.dir, f) for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:
         if os.path.basename(f).startswith("ecj-") and os.path.basename(f).endswith(".jar"):
             mx.log("Automatically choosing %s for JDT" % f)
             os.environ["JDT"] = f
             return
 
 
 def find_eclipse():
-    pardir = os.path.abspath(os.path.join(_suite.dir, ".."))
-    for f in [os.path.join(_suite.dir, f) for f in os.listdir(_suite.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:
+    pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))
+    for f in [os.path.join(SUITE.dir, f) for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:
         if os.path.basename(f) == "eclipse" and os.path.isdir(f):
             mx.log("Automatically choosing %s for Eclipse" % f)
             eclipse_exe = os.path.join(f, "eclipse")
@@ -351,7 +352,7 @@ def gate_unittests(args=[], subdir=""):
 
 def run_python_unittests(python_binary, args=[], aot_compatible=True, exclude=[]):
     # tests root directory
-    tests_folder = os.path.join(_suite.dir, "graalpython", "com.oracle.graal.python.test", "src", "tests")
+    tests_folder = os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "tests")
 
     # list of excluded tests
     if aot_compatible:
@@ -377,7 +378,7 @@ def is_included(path):
             except OSError:
                 pass
 
-    args += [os.path.join(_suite.dir, "graalpython", "com.oracle.graal.python.test", "src", "graalpytest.py"), "-v"]
+    args += [os.path.join(SUITE.dir, "graalpython", "com.oracle.graal.python.test", "src", "graalpytest.py"), "-v"]
     args += testfiles
     return mx.run([python_binary] + args, nonZeroIsFatal=True)
 
@@ -414,15 +415,15 @@ def graalpython_gate_runner(args, tasks):
             if not os.path.exists(svm_image_name):
                 svm_image_name = python_svm(["-h"])
             llvm_home = mx_subst.path_substitutions.substitute('--native.Dllvm.home=<path:SULONG_LIBS>')
-            args = ["--python.CoreHome=%s" % os.path.join(_suite.dir, "graalpython", "lib-graalpython"),
-                    "--python.StdLibHome=%s" % os.path.join(_suite.dir, "graalpython", "lib-python/3"),
+            args = ["--python.CoreHome=%s" % os.path.join(SUITE.dir, "graalpython", "lib-graalpython"),
+                    "--python.StdLibHome=%s" % os.path.join(SUITE.dir, "graalpython", "lib-python/3"),
                     llvm_home]
             run_python_unittests(svm_image_name, args)
 
     with Task('GraalPython apptests', tasks, tags=[GraalPythonTags.apptests]) as task:
         if task:
             apprepo = os.environ["GRAALPYTHON_APPTESTS_REPO_URL"]
-            _apptest_suite = _suite.import_suite(
+            _apptest_suite = SUITE.import_suite(
                 "graalpython-apptests",
                 version="fe1458e3af707f5996f192aedc7b76585501e8ba",
                 urlinfos=[mx.SuiteImportURLInfo(mx_urlrewrites.rewriteurl(apprepo), "git", mx.vc_system("git"))]
@@ -453,13 +454,13 @@ def graalpython_gate_runner(args, tasks):
             if success not in out.data:
                 mx.abort('Output from generated SVM image "' + svm_image + '" did not match success pattern:\n' + success)
             llvm_home = mx_subst.path_substitutions.substitute('--native.Dllvm.home=<path:SULONG_LIBS>')
-            args = ["--python.CoreHome=%s" % os.path.join(_suite.dir, "graalpython", "lib-graalpython"),
-                    "--python.StdLibHome=%s" % os.path.join(_suite.dir, "graalpython", "lib-python/3"),
+            args = ["--python.CoreHome=%s" % os.path.join(SUITE.dir, "graalpython", "lib-graalpython"),
+                    "--python.StdLibHome=%s" % os.path.join(SUITE.dir, "graalpython", "lib-python/3"),
                     llvm_home]
             run_python_unittests(svm_image, args)
 
 
-mx_gate.add_gate_runner(_suite, graalpython_gate_runner)
+mx_gate.add_gate_runner(SUITE, graalpython_gate_runner)
 
 
 def run_shared_lib_test(args=None):
@@ -593,7 +594,7 @@ def run_shared_lib_test(args=None):
         }
         """)
         os.close(fd)
-        progname = os.path.join(_suite.dir, "graalpython-embedded-tool")
+        progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")
         mx.log("".join(["Running ", "'clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"]))
         mx.run(["clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o%s" % progname, "-lpolyglot"], nonZeroIsFatal=True)
         mx.log("Running " + progname + " with LD_LIBRARY_PATH " + svm_lib_path)
@@ -650,7 +651,7 @@ def getResults(self):
 def deploy_binary_if_master(args):
     """if the active branch is 'master', deploy binaries for the primary suite to remote maven repository."""
     master_branch = 'master'
-    active_branch = mx.VC.get_vc(_suite.dir).active_branch(_suite.dir)
+    active_branch = mx.VC.get_vc(SUITE.dir).active_branch(SUITE.dir)
     if active_branch == master_branch:
         if sys.platform == "darwin":
             args.insert(0, "--platform-dependent")
@@ -684,8 +685,8 @@ def delete_self_if_testdownstream(args):
     """
     A helper for downstream testing with binary dependencies
     """
-    if str(_suite.dir).endswith("testdownstream/graalpython"):
-        shutil.rmtree(_suite.dir, ignore_errors=True)
+    if str(SUITE.dir).endswith("testdownstream/graalpython"):
+        shutil.rmtree(SUITE.dir, ignore_errors=True)
 
 
 def update_import(name, rev="origin/master", callback=None):
@@ -718,11 +719,11 @@ def update_import_cmd(args):
         if name == "sulong":
             join = os.path.join
             callback=lambda: shutil.copy(
-                join(_sulong.dir, "include", "truffle.h"),
-                join(_suite.dir, "graalpython", "com.oracle.graal.python.cext", "include", "truffle.h")
+                join(SUITE_SULONG.dir, "include", "truffle.h"),
+                join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext", "include", "truffle.h")
             ) and shutil.copy(
                 join(mx.dependency("SULONG_LIBS").output, "polyglot.h"),
-                join(_suite.dir, "graalpython", "com.oracle.graal.python.cext", "include", "polyglot.h")
+                join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext", "include", "polyglot.h")
             )
         # make sure that sulong and regex are the same version
         if name == "regex":
@@ -819,11 +820,11 @@ def import_python_sources(args):
         files = [line.split(",")[0] for line in f.read().split("\n") if len(line.split(",")) > 1 and line.split(",")[1] == "python.copyright"]
 
     # move to orphaned branch with sources
-    if _suite.vc.isDirty(_suite.dir):
+    if SUITE.vc.isDirty(SUITE.dir):
         mx.abort("Working dir must be clean")
-    tip = _suite.vc.tip(_suite.dir).strip()
-    _suite.vc.git_command(_suite.dir, ["checkout", "python-import"])
-    _suite.vc.git_command(_suite.dir, ["clean", "-fdx"])
+    tip = SUITE.vc.tip(SUITE.dir).strip()
+    SUITE.vc.git_command(SUITE.dir, ["checkout", "python-import"])
+    SUITE.vc.git_command(SUITE.dir, ["clean", "-fdx"])
     shutil.rmtree("graalpython")
 
     for inlined_file in files:
@@ -849,27 +850,18 @@ def import_python_sources(args):
             mx.warn("Could not update %s - original file not found" % inlined_file)
 
     # re-copy lib-python
-    libdir = os.path.join(_suite.dir, "graalpython/lib-python/3")
+    libdir = os.path.join(SUITE.dir, "graalpython/lib-python/3")
     shutil.copytree(os.path.join(pypy_sources, "lib-python", "3"), libdir)
 
     # commit and check back
-    _suite.vc.git_command(_suite.dir, ["add", "."])
+    SUITE.vc.git_command(SUITE.dir, ["add", "."])
     raw_input("Check that the updated files look as intended, then press RETURN...")
-    _suite.vc.commit(_suite.dir, "Update Python inlined files: %s" % import_version)
+    SUITE.vc.commit(SUITE.dir, "Update Python inlined files: %s" % import_version)
     answer = raw_input("Should we push python-import (y/N)? ")
     if answer and answer in "Yy":
-        _suite.vc.git_command(_suite.dir, ["push", "origin", "python-import:python-import"])
-    _suite.vc.update(_suite.dir, rev=tip)
-    _suite.vc.git_command(_suite.dir, ["merge", "python-import"])
-
-
-# ----------------------------------------------------------------------------------------------------------------------
-#
-# add the defined python benchmark suites
-#
-# ----------------------------------------------------------------------------------------------------------------------
-for py_bench_suite in PythonBenchmarkSuite.get_benchmark_suites(BENCHMARKS):
-    mx_benchmark.add_bm_suite(py_bench_suite)
+        SUITE.vc.git_command(SUITE.dir, ["push", "origin", "python-import:python-import"])
+    SUITE.vc.update(SUITE.dir, rev=tip)
+    SUITE.vc.git_command(SUITE.dir, ["merge", "python-import"])
 
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -878,7 +870,7 @@ def import_python_sources(args):
 #
 # ----------------------------------------------------------------------------------------------------------------------
 mx_sdk.register_graalvm_component(mx_sdk.GraalVmLanguage(
-    suite=_suite,
+    suite=SUITE,
     name='Graal.Python',
     short_name='pyn',
     dir_name='python',
@@ -911,18 +903,41 @@ def import_python_sources(args):
 #
 # ----------------------------------------------------------------------------------------------------------------------
 if not os.getenv("GRAAL_PYTHONHOME"):
-    home = os.path.join(_suite.dir, "graalpython")
+    home = os.path.join(SUITE.dir, "graalpython")
     if not os.path.exists(home):
-        home = [d for d in _suite.dists if d.name == "GRAALPYTHON_GRAALVM_SUPPORT"][0].output
+        home = [d for d in SUITE.dists if d.name == "GRAALPYTHON_GRAALVM_SUPPORT"][0].output
     os.environ["GRAAL_PYTHONHOME"] = home
 
 
+# ----------------------------------------------------------------------------------------------------------------------
+#
+# post init
+#
+# ----------------------------------------------------------------------------------------------------------------------
+
+def _register_vms(namespace):
+    python_vm_registry.add_vm(CPythonVm(CONFIGURATION_DEFAULT), SUITE)
+    python_vm_registry.add_vm(PyPyVm(CONFIGURATION_DEFAULT), SUITE)
+    python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE, 10)
+
+
+def _register_bench_suites(namespace):
+    for py_bench_suite in PythonBenchmarkSuite.get_benchmark_suites(BENCHMARKS):
+        mx_benchmark.add_bm_suite(py_bench_suite)
+
+
+def mx_post_parse_cmd_line(namespace):
+    # all projects are now available at this time
+    _register_vms(namespace)
+    _register_bench_suites(namespace)
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # register the suite commands (if any)
 #
 # ----------------------------------------------------------------------------------------------------------------------
-mx.update_commands(_suite, {
+mx.update_commands(SUITE, {
     'python': [python, '[Python args|@VM options]'],
     'python3': [python, '[Python args|@VM options]'],
     'deploy-binary-if-master': [deploy_binary_if_master, ''],