[GR-17682] Make mx scripts Python 3 compatible

PullRequest: graalpython/633

Author: timfel

ci.jsonnet

@@ -20,7 +20,7 @@ local builder = import 'ci_common/builder.libsonnet';
     //
     // ------------------------------------------------------------------------------------------------------
     local gates = [
-        // unittests
+        // unittests on JDK8
         builder.testGate(type="unittest", platform="linux"),
         builder.testGate(type="unittest", platform="darwin"),
         builder.testGateTime(type="tagged-unittest", platform="linux", timelimit=const.TIME_LIMIT["2h"]),
@@ -29,6 +29,10 @@ local builder = import 'ci_common/builder.libsonnet';
         builder.testGate(type="svm-unittest", platform="darwin"),
         builder.testGate(type="unittest-jython", platform="linux"),
 
+        // JDK11
+        // builder.testGate11(type="unittest", platform="linux"),
+        // builder.testGate11(type="svm-unittest", platform="linux"),
+
         // junit
         builder.testGate(type="junit", platform="linux"),
         builder.testGate(type="junit", platform="darwin"),

ci_common/builder.libsonnet

@@ -25,6 +25,10 @@ local mixins = import 'mixins.libsonnet';
     local common = base + mixins.labsjdk8 + {
         dynamicImports:: "/compiler",
 
+        environment +: {
+            MX_PYTHON_VERSION: "3",
+        },
+
         setup +: [
             ["mx", "sforceimports"],
             ["mx", "--dynamicimports", self.dynamicImports, "build"],
@@ -56,11 +60,15 @@ local mixins = import 'mixins.libsonnet';
     baseGraalGate:: baseGraalGate,
 
     // specific gates
-    testGate(type, platform)::
+    local testGate = function(type, platform)
         baseGraalGate + {tags:: "python-"+type} + mixins.getPlatform(platform) + {name: "python-"+ type +"-"+platform},
+    testGate:: testGate,
+
+    testGate11(type, platform)::
+        testGate(type, platform) + mixins.labsjdk11 + {name: "python-"+ type +"-jdk11-"+platform},
 
     testGateTime(type, platform, timelimit)::
-        baseGraalGate + {tags:: "python-"+type} + mixins.getPlatform(platform) + {name: "python-"+ type +"-"+platform} + {timelimit: timelimit},
+        testGate(type, platform) + {timelimit: timelimit},
 
     local baseStyleGate = baseGraalGate + mixins.eclipse + mixins.linux + {
         tags:: "style",
@@ -115,4 +123,4 @@ local mixins = import 'mixins.libsonnet';
             name: "python-coverage"
         },
 
-}
+}

ci_common/mixins.libsonnet

@@ -38,6 +38,9 @@ local const = import 'constants.libsonnet';
             "pip:pylint": "==1.1.0",
             "llvm": "==4.0.1",
         },
+        environment: {
+            "LC_CTYPE": "en_US.UTF-8", // make sure we have the correct locale
+        },
     },
     darwin:: darwin,
 
@@ -72,18 +75,28 @@ local const = import 'constants.libsonnet';
     },
     eclipse:: eclipse,
 
-    local labsjdk8 = {
-        downloads +: {
-            JAVA_HOME: utils.download("oraclejdk", "8u212-jvmci-19.2-b01"),
-        },
+    local labsjdk = {
         environment +: {
             CI: "true",
             GRAALVM_CHECK_EXPERIMENTAL_OPTIONS: "true",
             PATH: "$JAVA_HOME/bin:$PATH",
         },
     },
+
+    local labsjdk8 = labsjdk + {
+        downloads +: {
+            JAVA_HOME: utils.download("oraclejdk", "8u221-jvmci-19.3-b01"),
+        },
+    },
     labsjdk8:: labsjdk8,
 
+    local labsjdk11 = labsjdk + {
+        downloads +: {
+            JAVA_HOME: utils.download("labsjdk", "11-20190830-095818"),
+        },
+    },
+    labsjdk11:: labsjdk11,
+
     local graal = labsjdk8 + {
         environment +: {
             HOST_VM: const.JVM.server,

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

@@ -106,6 +106,7 @@ public class CodecsModuleBuiltins extends PythonBuiltins {
         CHARSET_MAP.put("L1", StandardCharsets.ISO_8859_1);
 
         // utf-8
+        CHARSET_MAP.put("UTF-8", StandardCharsets.UTF_8);
         CHARSET_MAP.put("utf-8", StandardCharsets.UTF_8);
         CHARSET_MAP.put("utf_8", StandardCharsets.UTF_8);
         CHARSET_MAP.put("U8", StandardCharsets.UTF_8);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

@@ -746,7 +746,7 @@ private PException fileNoFound(String path) {
 
         @TruffleBoundary(allowInlining = true, transferToInterpreterOnException = false)
         private static long strToLong(String name) throws NumberFormatException {
-            return new Long(name).longValue();
+            return Long.decode(name).longValue();
         }
 
         @TruffleBoundary(allowInlining = true)

mx.graalpython/mx_graalpython.py

@@ -50,6 +50,12 @@
 SUITE_SULONG = mx.suite("sulong")
 
 
+# compatibility between Python versions
+PY3 = sys.version_info[0] == 3
+if PY3:
+    raw_input = input
+
+
 def _get_core_home():
     return os.path.join(SUITE.dir, "graalpython", "lib-graalpython")
 
@@ -113,7 +119,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, **kwargs):
 
     dists = ['GRAALPYTHON', 'TRUFFLE_NFI', 'SULONG']
     env["PYTHONUSERBASE"] = mx_subst.path_substitutions.substitute("<path:PYTHON_USERBASE>")
-    
+
     vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
     graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)
     dists += additional_dists
@@ -317,7 +323,7 @@ def _graalpytest_root():
 
 def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=True, exclude=None):
     args = args or []
-    args = ["--experimental-options=true", 
+    args = ["--experimental-options=true",
             "--python.CatchAllExceptions=true",
             mx_subst.path_substitutions.substitute("--python.CAPI=<path:com.oracle.graal.python.cext>"),
             ] + args
@@ -372,6 +378,7 @@ def graalpython_gate_runner(args, tasks):
                 mx.log("Running tests with CPython")
                 test_args = [_graalpytest_driver(), "-v", _graalpytest_root()]
                 mx.run(["python3"] + test_args, nonZeroIsFatal=True)
+            mx.run(["env"])
             run_python_unittests(python_gvm())
 
     with Task('GraalPython sandboxed tests', tasks, tags=[GraalPythonTags.unittest_sandboxed]) as task:
@@ -498,7 +505,7 @@ def run_shared_lib_test(args=None):
     fd = name = progname = None
     try:
         fd, name = tempfile.mkstemp(suffix='.c')
-        os.write(fd, """
+        os.write(fd, b"""
         #include "stdio.h"
         #include "polyglot_api.h"
 
@@ -605,7 +612,7 @@ def run_shared_lib_test(args=None):
             }
             return test_basic_python_function();
         }
-        """ % ("1" if "sandboxed" in args else "0"))
+        """ % (b"1" if "sandboxed" in args else b"0"))
         os.close(fd)
         progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")
         mx.log("".join(["Running ", "'clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"]))

mx.graalpython/mx_graalpython_benchmark.py

@@ -80,35 +80,54 @@ def is_sandboxed_configuration(conf):
     return conf == CONFIGURATION_SANDBOXED or conf == CONFIGURATION_SANDBOXED_MULTI
 
 
+# from six
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        if hasattr(cls, '__qualname__'):
+            orig_vars['__qualname__'] = cls.__qualname__
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
 @contextmanager
 def environ(env):
-    def _handle_var((k, v)):
+    def _handle_var(key_value):
+        (k, v) = key_value
         if v is None:
             del os.environ[k]
         else:
             os.environ[k] = str(v)
 
     if env:
         prev_env = {v: os.getenv(v) for v in env}
-        map(_handle_var, env.items())
+        list(map(_handle_var, list(env.items())))
     else:
         prev_env = None
 
     try:
         yield
     finally:
         if prev_env:
-            map(_handle_var, prev_env.items())
+            list(map(_handle_var, list(prev_env.items())))
 
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # the vm definitions
 #
 # ----------------------------------------------------------------------------------------------------------------------
+@add_metaclass(ABCMeta)
 class AbstractPythonVm(Vm):
-    __metaclass__ = ABCMeta
-
     def __init__(self, config_name, options=None, env=None):
         super(AbstractPythonVm, self).__init__()
         self._config_name = config_name
@@ -156,9 +175,8 @@ def run(self, cwd, args):
         return ret_code, out.data
 
 
+@add_metaclass(ABCMeta)
 class AbstractPythonIterationsControlVm(AbstractPythonVm):
-    __metaclass__ = ABCMeta
-
     def __init__(self, config_name, options=None, env=None, iterations=None):
         super(AbstractPythonIterationsControlVm, self).__init__(config_name, options=options, env=env)
         try:
@@ -424,7 +442,7 @@ def createVmCommandLineArgs(self, benchmarks, bmSuiteArgs):
             for pth in self._python_path:
                 if hasattr(pth, '__call__'):
                     pth = pth()
-                assert isinstance(pth, (str, unicode))
+                assert isinstance(pth, str)
                 python_path.append(pth)
             cmd_args += ['-p', ",".join(python_path)]
 
@@ -438,7 +456,7 @@ def createVmCommandLineArgs(self, benchmarks, bmSuiteArgs):
         return vm_options + vm_args + cmd_args
 
     def benchmarkList(self, bm_suite_args):
-        return self._benchmarks.keys()
+        return list(self._benchmarks.keys())
 
     def benchmarks(self):
         raise FutureWarning('the benchmarks method has been deprecated for VmBenchmarkSuite instances, '

mx.graalpython/suite.py

@@ -44,15 +44,15 @@
             },
             {
                 "name": "sulong",
-                "version": "432df38e6864205e10e52aa4058ce868aedab929",
+                "version": "01bb0670bac1ab1e87322b0621bc4ff2b76e79bc",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ]
             },
             {
                 "name": "regex",
-                "version": "432df38e6864205e10e52aa4058ce868aedab929",
+                "version": "01bb0670bac1ab1e87322b0621bc4ff2b76e79bc",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},