[GR-13107] Enable building and running scikit-learn examples without CPython support

PullRequest: graalpython/333

Author: timfel

README.md

@@ -36,6 +36,15 @@ If all goes well (you'll need to have `clang`, `llvm-link`, `llvm-extract`,
 `llvm-nm`, and `opt` in your `PATH` in addition to the normal NumPy build
 dependencies), you should be able to `import numpy` afterwards.
 
+Support for more extension modules is high priority for us. We are actively
+building out our support for the Python C API to make extensions such as NumPy,
+SciPy, Scikit-learn, Tensorflow and the like work. This work means that some
+other extensions might also already work, but we're not actively testing other
+extensions right now and cannot promise anything. Note that to try extensions on
+this implementation, you have to download, build, and install them manually for
+now. To do so, we recommend LLVM 6. Other versions might also work, but this
+version is what we're testing with in our CI.
+
 ### Licensing
 
 This Graal/Truffle-based implementation of Python is copyright (c) 2017, 2018

graalpython/com.oracle.graal.python.cext/src/boolobject.c

@@ -42,7 +42,7 @@
 
 #include <stdarg.h>
 
-PyTypeObject PyBool_Type = PY_TRUFFLE_TYPE("bool", &PyType_Type, Py_TPFLAGS_DEFAULT, sizeof(struct _longobject));
+PyTypeObject PyBool_Type = PY_TRUFFLE_TYPE("bool", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_LONG_SUBCLASS, sizeof(struct _longobject));
 
 // taken from CPython "Python/Objects/boolobject.c"
 PyObject *PyBool_FromLong(long ok) {

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -235,13 +235,19 @@ void* get_ob_type(PyObject* obj) {
     if (!truffle_cannot_be_handle(type)) {
         return resolve_handle(cache, (uint64_t)type);
     } else {
-        // we have stored a handle to the Java class in ob_refcnt
-        void* handle = (void*)((PyObject*)type)->ob_refcnt;
-        if (!truffle_cannot_be_handle(handle)) {
-            return resolve_handle(cache, (uint64_t)handle);
+        PyObject* cast_type = ((PyObject*)type);
+        if (!polyglot_is_value(cast_type)) {
+            // we have stored a handle to the Java class in ob_refcnt
+            void* handle = (void*)(cast_type->ob_refcnt);
+            if (!truffle_cannot_be_handle(handle)) {
+                return resolve_handle(cache, (uint64_t)handle);
+            } else {
+                // assume handle is a TruffleObject
+                return handle;
+            }
         } else {
-            // assume handle is a TruffleObject
-            return handle;
+            // the type is already the right value (e.g. on sandboxed it's a managed pointer)
+            return cast_type;
         }
     }
 }
@@ -649,6 +655,21 @@ void* wrap_unsupported(void *fun, ...) {
     return NULL;
 }
 
-int truffle_ptr_compare(void* x, void* y) {
-    return x == y;
+int truffle_ptr_compare(void* x, void* y, int op) {
+    switch (op) {
+    case Py_LT:
+        return x < y;
+    case Py_LE:
+        return x <= y;
+    case Py_EQ:
+        return x == y;
+    case Py_NE:
+        return x != y;
+    case Py_GT:
+        return x > y;
+    case Py_GE:
+        return x >= y;
+    default:
+        return -1;
+    }
 }

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -211,7 +211,7 @@ void* wrap_keywords(PyCFunctionWithKeywords fun, PyObject *module, PyObject *var
 void* wrap_fastcall(_PyCFunctionFast        fun, PyObject *  self, PyObject   **args, PyObject  *nargs, PyObject *kwnames);
 void* wrap_unsupported(void *fun, ...);
 
-#define TDEBUG __asm__("int $3")
+#define TDEBUG __builtin_debugtrap()
 #define get_method_flags_wrapper(flags)                                 \
     (((flags) < 0) ?                                                    \
      truffle_read(PY_TRUFFLE_CEXT, "METH_DIRECT") :                     \

graalpython/com.oracle.graal.python.cext/src/exceptions.c

@@ -82,6 +82,7 @@ PyObject * PyExc_UnboundLocalError = NULL;
 PyObject * PyExc_NotImplementedError = NULL;
 PyObject * PyExc_RecursionError = NULL;
 PyObject * PyExc_UnicodeEncodeError = NULL;
+PyObject * PyExc_GeneratorExit = NULL;
 
 void initialize_exceptions() {
     PyExc_AttributeError = PY_EXCEPTION("AttributeError");
@@ -120,6 +121,7 @@ void initialize_exceptions() {
     PyExc_RecursionError = PY_EXCEPTION("RecursionError");
     PyExc_NotImplementedError = PY_EXCEPTION("NotImplementedError");
     PyExc_UnicodeEncodeError = PY_EXCEPTION("UnicodeEncodeError");
+    PyExc_GeneratorExit = PY_EXCEPTION("GeneratorExit");
 }
 
 

graalpython/com.oracle.graal.python.cext/src/obmalloc.c

@@ -76,6 +76,10 @@ int _PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, uintptr_t ptr, size_t s
     return -2; // we do not track
 }
 
+int _PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, uintptr_t ptr) {
+    return -2; // we do not track
+}
+
 void * PyMem_Realloc(void *ptr, size_t new_size) {
     return PyMem_RawRealloc(ptr, new_size);
 }

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonCC.java

@@ -60,6 +60,7 @@ public class GraalPythonCC extends GraalPythonCompiler {
     private List<String> clangArgs;
     private List<String> execLinkArgs;
     private List<String> fileInputs;
+    private boolean isCpp;
 
     GraalPythonCC() {
     }
@@ -103,6 +104,20 @@ static void main(String[] args) {
 
     private void run(String[] args) {
         parseOptions(args);
+        if (isCpp) {
+            // cannot use streaming API anyMatch for this on SVM
+            for (String s : clangArgs) {
+                if (s.contains("--sysroot")) {
+                    // nasty, nasty
+                    logV("Refusing to compile C++ code in sandboxed mode, because we cannot actually do it");
+                    try {
+                        Files.createFile(Paths.get(outputFilename));
+                    } catch (IOException e) {
+                    }
+                    return;
+                }
+            }
+        }
         launchCC();
     }
 
@@ -116,6 +131,7 @@ private void parseOptions(String[] args) {
         clangArgs = new ArrayList<>(clangPrefix);
         execLinkArgs = new ArrayList<>(execLinkPrefix);
         fileInputs = new ArrayList<>();
+        isCpp = false;
         for (int i = 0; i < args.length; i++) {
             String arg = args[i];
             switch (arg) {
@@ -152,6 +168,9 @@ private void parseOptions(String[] args) {
                         }
                         fileInputs.add(arg);
                     } else if (arg.endsWith(".c") || arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
+                        if (arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
+                            isCpp = true;
+                        }
                         if (compile == null) {
                             compile = true;
                         } else if (compile != true) {
@@ -168,6 +187,9 @@ private void parseOptions(String[] args) {
         if (targetFlags != null) {
             clangArgs.addAll(Arrays.asList(targetFlags.split(" ")));
         }
+        if (isCpp) {
+            clangArgs.add("-stdlib=libc++");
+        }
     }
 
     private void launchCC() {

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonLD.java

@@ -244,11 +244,13 @@ private Collection<? extends String> arMembers(String path) throws IOException,
         // in the build process, because such a smart linker should not be assumed for POSIX, but it
         // seems ok to emulate this at least for the very common case of ar archives with symbol
         // definitions that overlap what's defined in explicitly include .o files
-        for (String f : members) {
+        outer: for (String f : members) {
             if (Files.probeContentType(Paths.get(f)).contains(LLVM_IR_BITCODE)) {
-                HashSet<String> definedHere = new HashSet<>();
+                HashSet<String> definedFuncs = new HashSet<>();
+                HashSet<String> definedGlobals = new HashSet<>();
+
                 ProcessBuilder nm = new ProcessBuilder();
-                nm.command(LLVM_NM, "-g", "--defined-only", f);
+                nm.command(LLVM_NM, "--defined-only", f);
                 nm.redirectInput(Redirect.INHERIT);
                 nm.redirectError(Redirect.INHERIT);
                 nm.redirectOutput(Redirect.PIPE);
@@ -257,23 +259,40 @@ private Collection<? extends String> arMembers(String path) throws IOException,
                     String line = null;
                     while ((line = buffer.readLine()) != null) {
                         String[] symboldef = line.split(" ");
-                        if (symboldef.length >= 2) {
-                            definedHere.add(symboldef[symboldef.length - 1]);
+                        if (symboldef.length == 3) {
+                            // format is ------- CHAR FUNCNAME
+                            if (symboldef[1].toLowerCase().equals("t")) {
+                                definedFuncs.add(symboldef[2].trim());
+                            } else if (symboldef[1].toLowerCase().equals("d")) {
+                                definedGlobals.add(symboldef[2].trim());
+                            } else {
+                                // keep all if we have symbols that we wouldn't know what to do with
+                                logV("Not extracting from ", f, " because there are non-strong function or global symbols");
+                                continue outer;
+                            }
                         }
                     }
                 }
                 nmProc.waitFor();
 
                 ArrayList<String> extractCmd = new ArrayList<>();
                 extractCmd.add("llvm-extract");
-                for (String def : definedHere) {
-                    if (undefinedSymbols.contains(def)) {
+                for (String def : definedFuncs) {
+                    if (!definedSymbols.contains(def)) {
                         definedSymbols.add(def);
                         undefinedSymbols.remove(def);
                         extractCmd.add("-func");
                         extractCmd.add(def);
                     }
                 }
+                for (String def : definedGlobals) {
+                    if (!definedSymbols.contains(def)) {
+                        definedSymbols.add(def);
+                        undefinedSymbols.remove(def);
+                        extractCmd.add("-glob");
+                        extractCmd.add(def);
+                    }
+                }
                 extractCmd.add(f);
                 extractCmd.add("-o");
                 extractCmd.add(f);

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_abstract.py

@@ -240,6 +240,15 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare
     )
 
+    test_PyLong_Check = CPyExtFunction(
+        lambda args: isinstance(args[0], int),
+        _default_unarop_args,
+        resultspec="i",
+        argspec='O',
+        arguments=["PyObject* v"],
+        cmpfunc=unhandled_error_compare
+    )
+
     test_PyNumber_Add = CPyExtFunction(
         lambda args: args[0] + args[1],
         lambda: (

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_long.py

@@ -271,6 +271,33 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare
     )
 
+    # We get a pattern like this in Cython generated code
+    test_PyLong_FromAndToVoidPtrAllocated = CPyExtFunction(
+        lambda args: True,
+        lambda: ((None,),),
+        code="""PyObject* PyLong_FromAndToVoidPtrAllocated(PyObject* none) {
+            unsigned long l = 0;
+            void* unwrappedPtr;
+            PyObject* result;
+            void* dummyPtr = malloc(sizeof(size_t));
+            PyObject* obj = PyLong_FromVoidPtr(dummyPtr);
+            int r = PyObject_RichCompareBool(obj, Py_False, Py_LT);
+            if (r < 0) {
+                return Py_None;
+            }
+            l = PyLong_AsUnsignedLong(obj);
+            unwrappedPtr = (void*)l;
+            result = unwrappedPtr == dummyPtr ? Py_True : Py_False;
+            free(dummyPtr);
+            return result;
+        }
+        """,
+        resultspec="O",
+        argspec='O',
+        arguments=["PyObject* none"],
+        cmpfunc=unhandled_error_compare
+    )
+
     test_PyLong_Check = CPyExtFunction(
         lambda args: isinstance(args[0], int),
         lambda: (