[GR-10719] Progress towards Cython modules

timfel · timfel · commit b1daaf034644 · 2018-07-12T06:32:26.000-07:00
PullRequest: graalpython/112
diff --git a/ci.jsonnet b/ci.jsonnet
@@ -124,8 +124,8 @@
 
   local labsjdk8Mixin = {
     downloads +: {
-      JAVA_HOME: utils.download("labsjdk", "8u161-jvmci-0.42"),
-      EXTRA_JAVA_HOMES : { pathlist: [utils.download("oraclejdk", "9.0.4+11")] },
+      JAVA_HOME: utils.download("labsjdk", "8u172-jvmci-0.46"),
+      EXTRA_JAVA_HOMES : { pathlist: [utils.download("oraclejdk", "11+20")] },
     },
     environment +: {
       CI: "true",
diff --git a/graalpython/com.oracle.graal.python.cext/include/Python.h b/graalpython/com.oracle.graal.python.cext/include/Python.h
@@ -46,6 +46,7 @@
 #define HAVE_SYS_WAIT_H
 
 #define PYPY_VERSION 0
+#define PYPY_VERSION_NUM 0
 
 #include <truffle.h>
 #include <polyglot.h>
@@ -113,6 +114,7 @@
 #include "frameobject.h"
 #include "traceback.h"
 #include "classobject.h"
+#include "pythread.h"
 
 #define PY_TRUFFLE_CEXT ((void*)polyglot_import("python_cext"))
 #define PY_BUILTIN ((void*)polyglot_import("python_builtins"))
diff --git a/graalpython/com.oracle.graal.python.cext/src/ceval.c b/graalpython/com.oracle.graal.python.cext/src/ceval.c
@@ -47,3 +47,14 @@ int _Py_CheckRecursionLimit = Py_DEFAULT_RECURSION_LIMIT;
 PyObject* PyEval_CallObjectWithKeywords(PyObject *func, PyObject *args, PyObject *kwargs) {
     return PyObject_Call(func, args, kwargs);
 }
+
+PyThreadState* PyEval_SaveThread() {
+    return NULL;
+}
+
+void PyEval_RestoreThread(PyThreadState *ptr) {
+}
+
+void* PyThread_allocate_lock() {
+    return NULL;
+}
diff --git a/graalpython/com.oracle.graal.python.cext/src/floatobject.c b/graalpython/com.oracle.graal.python.cext/src/floatobject.c
@@ -62,3 +62,15 @@ double PyFloat_AsDouble(PyObject *op) {
 PyObject* PyFloat_FromDouble(double fval) {
     return UPCALL_CEXT_O("PyFloat_FromDouble", fval);
 }
+
+// not quite as in CPython, this assumes that x is already a double. The rest of
+// the implementation is in the Float constructor in Java
+PyObject* float_subtype_new(PyTypeObject *type, double x) {
+    PyObject* newobj = type->tp_alloc(type, 0);
+    if (newobj == NULL) {
+        Py_DECREF(tmp);
+        return NULL;
+    }
+    ((PyFloatObject *)newobj)->ob_fval = x;
+    return newobj;
+}
diff --git a/graalpython/com.oracle.graal.python.cext/src/functionobject.c b/graalpython/com.oracle.graal.python.cext/src/functionobject.c
@@ -37,3 +37,7 @@
  * SOFTWARE.
  */
 #include "capi.h"
+
+PyObject* PyClassMethod_New(PyObject* method) {
+    return UPCALL_O(PY_BUILTIN, "classmethod", native_to_java(method));
+}
diff --git a/graalpython/com.oracle.graal.python.cext/src/methodobject.c b/graalpython/com.oracle.graal.python.cext/src/methodobject.c
@@ -41,6 +41,7 @@
 typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
 
 PyTypeObject PyCFunction_Type = PY_TRUFFLE_TYPE("builtin_function_or_method", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyCFunctionObject));
+PyTypeObject PyWrapperDescr_Type = PY_TRUFFLE_TYPE("builtin_function_or_method", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyWrapperDescr_Type));;
 
 PyObject* PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) {
     PyObject* func = to_sulong(polyglot_invoke(PY_TRUFFLE_CEXT,
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java
@@ -66,6 +66,8 @@
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
 import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
 import com.oracle.graal.python.builtins.objects.cell.PCell;
+import com.oracle.graal.python.builtins.objects.cext.CExtNodes;
+import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass;
 import com.oracle.graal.python.builtins.objects.code.PCode;
 import com.oracle.graal.python.builtins.objects.common.HashingStorage.DictEntry;
 import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes;
@@ -128,6 +130,7 @@
 import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallTernaryNode;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
+import com.oracle.graal.python.nodes.classes.IsSubtypeNode;
 import com.oracle.graal.python.nodes.control.GetIteratorNode;
 import com.oracle.graal.python.nodes.control.GetNextNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -426,51 +429,55 @@ public Object reversed(PythonClass cls, Object sequence,
     public abstract static class FloatNode extends PythonBuiltinNode {
         private final ConditionProfile isPrimitiveProfile = ConditionProfile.createBinaryProfile();
 
-        private boolean isPrimitiveFloat(Object cls) {
-            return isPrimitiveProfile.profile(cls == getCore().lookupType(PythonBuiltinClassType.PFloat));
+        protected boolean isPrimitiveFloat(Object cls) {
+            return isPrimitiveProfile.profile(cls == getBuiltinFloatClass());
         }
 
-        @Specialization
+        protected PythonBuiltinClass getBuiltinFloatClass() {
+            return getCore().lookupType(PythonBuiltinClassType.PFloat);
+        }
+
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromInt(PythonClass cls, int arg) {
             if (isPrimitiveFloat(cls)) {
                 return (double) arg;
             }
             return factory().createFloat(cls, arg);
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromBoolean(PythonClass cls, boolean arg) {
             if (isPrimitiveFloat(cls)) {
                 return arg ? 1d : 0d;
             }
             return factory().createFloat(cls, arg ? 1d : 0d);
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromLong(PythonClass cls, long arg) {
             if (isPrimitiveFloat(cls)) {
                 return (double) arg;
             }
             return factory().createFloat(cls, arg);
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromPInt(PythonClass cls, PInt arg) {
             if (isPrimitiveFloat(cls)) {
                 return arg.doubleValue();
             }
             return factory().createFloat(cls, arg.doubleValue());
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromFloat(PythonClass cls, double arg) {
             if (isPrimitiveFloat(cls)) {
                 return arg;
             }
             return factory().createFloat(cls, arg);
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromString(PythonClass cls, String arg) {
             double value = JavaTypeConversions.convertStringToDouble(arg);
             if (isPrimitiveFloat(cls)) {
@@ -479,34 +486,59 @@ public Object floatFromString(PythonClass cls, String arg) {
             return factory().createFloat(cls, value);
         }
 
-        @Specialization
+        @Specialization(guards = "!isNativeClass(cls)")
         public Object floatFromNone(PythonClass cls, @SuppressWarnings("unused") PNone arg) {
             if (isPrimitiveFloat(cls)) {
                 return 0.0;
             }
             return factory().createFloat(cls, 0.0);
         }
 
-        @Specialization
-        Object doPythonObject(PythonClass cls, Object obj,
+        @Specialization(guards = "isPrimitiveFloat(cls)")
+        double doubleFromObject(@SuppressWarnings("unused") PythonClass cls, Object obj,
                         @Cached("create(__FLOAT__)") LookupAndCallUnaryNode callFloatNode,
                         @Cached("create()") BranchProfile gotException) {
             try {
-                return floatFromFloat(cls, callFloatNode.executeDouble(obj));
+                return callFloatNode.executeDouble(obj);
             } catch (UnexpectedResultException e) {
                 gotException.enter();
                 Object result = e.getResult();
                 if (result == PNone.NO_VALUE) {
                     throw raise(TypeError, "must be real number, not %p", obj);
-                } else if (PGuards.isPFloat(result)) {
+                } else if (result instanceof PFloat) {
                     // TODO Issue warning if 'result' is a subclass of Python type 'float'
-                    return result;
+                    return ((PFloat) result).getValue();
                 } else {
                     throw raise(TypeError, "%p.__float__ returned non-float (type %p)", obj, result);
                 }
             }
         }
 
+        @Specialization(guards = "!isNativeClass(cls)")
+        Object doPythonObject(PythonClass cls, Object obj,
+                        @Cached("create(__FLOAT__)") LookupAndCallUnaryNode callFloatNode,
+                        @Cached("create()") BranchProfile gotException) {
+            return floatFromFloat(cls, doubleFromObject(cls, obj, callFloatNode, gotException));
+        }
+
+        protected CExtNodes.SubtypeNew createSubtypeNew() {
+            return new CExtNodes.SubtypeNew("float");
+        }
+
+        // logic similar to float_subtype_new(PyTypeObject *type, PyObject *x) from CPython
+        // floatobject.c we have to first create a temporary float, then fill it into
+        // a natively allocated subtype structure
+        @Specialization(guards = "isSubtype.execute(cls, floatCls)", limit = "1")
+        Object doPythonObject(PythonNativeClass cls, Object obj,
+                        @Cached("getBuiltinFloatClass()") PythonBuiltinClass floatCls,
+                        @SuppressWarnings("unused") @Cached("create()") IsSubtypeNode isSubtype,
+                        @Cached("create(__FLOAT__)") LookupAndCallUnaryNode callFloatNode,
+                        @Cached("create()") BranchProfile gotException,
+                        @Cached("createSubtypeNew()") CExtNodes.SubtypeNew subtypeNew) {
+            double realFloat = doubleFromObject(floatCls, obj, callFloatNode, gotException);
+            return subtypeNew.execute(cls, realFloat);
+        }
+
         @Fallback
         @TruffleBoundary
         public Object floatFromObject(@SuppressWarnings("unused") Object cls, Object arg) {
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/CExtNodes.java
@@ -75,6 +75,35 @@
 
 public abstract class CExtNodes {
 
+    /**
+     * For some builtin classes, the CPython approach to creating a subclass instance is to just
+     * call the alloc function and then assign some fields. This needs to be done in C. This node
+     * will call that subtype C function with two arguments, the C type object and an object
+     * argument to fill in from.
+     */
+    public static class SubtypeNew extends PBaseNode {
+        private final TruffleObject subtypeFunc;
+        @Child private Node executeNode = Message.createExecute(2).createNode();
+        @Child private ToSulongNode toSulongNode = ToSulongNode.create();
+        @Child private ToJavaNode toJavaNode = ToJavaNode.create();
+
+        /**
+         * @param typenamePrefix the <code>typename</code> in <code>typename_subtype_new</code>
+         */
+        public SubtypeNew(String typenamePrefix) {
+            subtypeFunc = (TruffleObject) getContext().getEnv().importSymbol(typenamePrefix + "_subtype_new");
+            assert subtypeFunc != null;
+        }
+
+        public Object execute(PythonNativeClass object, Object arg) {
+            try {
+                return toJavaNode.execute(ForeignAccess.sendExecute(executeNode, subtypeFunc, toSulongNode.execute(object), arg));
+            } catch (UnsupportedMessageException | UnsupportedTypeException | ArityException e) {
+                throw new IllegalStateException("C subtype_new function failed", e);
+            }
+        }
+    }
+
     public static class FromNativeSubclassNode extends PBaseNode {
         private final PythonBuiltinClassType expectedType;
         private final String conversionFuncName;
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PGuards.java
@@ -42,6 +42,7 @@
 import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
 import com.oracle.graal.python.builtins.objects.bytes.PByteArray;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
+import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass;
 import com.oracle.graal.python.builtins.objects.dict.PDict;
 import com.oracle.graal.python.builtins.objects.floats.PFloat;
 import com.oracle.graal.python.builtins.objects.function.PArguments;
@@ -259,6 +260,10 @@ public static boolean isPythonBuiltinClass(Object klass) {
         return klass instanceof PythonBuiltinClass;
     }
 
+    public static boolean isNativeClass(Object klass) {
+        return klass instanceof PythonNativeClass;
+    }
+
     public static boolean isPRange(Object obj) {
         return obj instanceof PRange;
     }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/ReadAttributeFromObjectNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/ReadAttributeFromObjectNode.java
@@ -40,6 +40,7 @@
 
 import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.object.PythonObject;
+import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.nodes.PNode;
 import com.oracle.truffle.api.Assumption;
 import com.oracle.truffle.api.CompilerDirectives;
@@ -85,6 +86,14 @@ protected static boolean checkShape(@SuppressWarnings("unused") PythonObject obj
         return cachedObject.getStorage().getShape() == cachedShape;
     }
 
+    protected Object attrKey(Object key) {
+        if (key instanceof PString) {
+            return ((PString) key).getValue();
+        } else {
+            return key;
+        }
+    }
+
     @SuppressWarnings("unused")
     @Specialization(limit = "1", //
                     guards = {
@@ -101,12 +110,13 @@ protected static boolean checkShape(@SuppressWarnings("unused") PythonObject obj
     protected Object readDirectFinal(PythonObject object, Object key,
                     @Cached("object") PythonObject cachedObject,
                     @Cached("key") Object cachedKey,
+                    @Cached("attrKey(key)") Object attrKey,
                     @Cached("object.getStorage().getShape()") Shape cachedShape,
                     @Cached("cachedShape.getValidAssumption()") Assumption layoutAssumption,
-                    @Cached("getLocationOrNull(cachedShape.getProperty(key))") Location loc,
+                    @Cached("getLocationOrNull(cachedShape.getProperty(attrKey))") Location loc,
                     @Cached("loc.getFinalAssumption()") Assumption finalAssumption,
                     @Cached("readFinalValue(object, loc)") Object cachedValue) {
-        assert assertFinal(object, key, cachedValue);
+        assert assertFinal(object, attrKey, cachedValue);
         return cachedValue;
     }
 
@@ -125,9 +135,10 @@ private static boolean assertFinal(PythonObject object, Object key, Object cache
                     assumptions = "layoutAssumption")
     protected Object readDirect(PythonObject object, Object key,
                     @Cached("key") Object cachedKey,
+                    @Cached("attrKey(cachedKey)") Object attrKey,
                     @Cached("object.getStorage().getShape()") Shape cachedShape,
                     @Cached("cachedShape.getValidAssumption()") Assumption layoutAssumption,
-                    @Cached("getLocationOrNull(cachedShape.getProperty(key))") Location loc) {
+                    @Cached("getLocationOrNull(cachedShape.getProperty(attrKey))") Location loc) {
         if (loc == null) {
             return PNone.NO_VALUE;
         } else {
@@ -151,7 +162,7 @@ protected Object updateShapeAndRead(PythonObject object, Object key,
 
     @Specialization(replaces = "readDirect")
     protected Object readIndirect(PythonObject object, Object key) {
-        Object value = object.getStorage().get(key);
+        Object value = object.getStorage().get(attrKey(key));
         if (value == null) {
             return PNone.NO_VALUE;
         } else {
@@ -163,7 +174,7 @@ protected Object readIndirect(PythonObject object, Object key) {
     protected Object readForeign(TruffleObject object, Object key,
                     @Cached("createReadNode()") Node readNode) {
         try {
-            return ForeignAccess.sendRead(readNode, object, key);
+            return ForeignAccess.sendRead(readNode, object, attrKey(key));
         } catch (UnknownIdentifierException | UnsupportedMessageException e) {
             return PNone.NO_VALUE;
         }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java
diff --git a/graalpython/lib-graalpython/python_cext.py b/graalpython/lib-graalpython/python_cext.py
diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py
diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py
