[GR-11913] Support pickle module.

PullRequest: graalpython/215

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -243,6 +243,12 @@ PyObject* PyObject_CallMethod(PyObject* object, const char* method, const char*
     return UPCALL_CEXT_O(_jls_PyObject_CallMethod, native_to_java(object), polyglot_from_string(method, SRC_CS), native_to_java(args));
 }
 
+PyObject* _PyObject_CallMethod_SizeT(PyObject* object, const char* method, const char* fmt, ...) {
+    PyObject* args;
+    CALL_WITH_VARARGS(args, Py_BuildValue, 3, fmt);
+    return UPCALL_CEXT_O(_jls_PyObject_CallMethod, native_to_java(object), polyglot_from_string(method, SRC_CS), native_to_java(args));
+}
+
 UPCALL_ID(type);
 PyObject* PyObject_Type(PyObject* obj) {
     return UPCALL_O(PY_BUILTIN, _jls_type, native_to_java(obj));

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -232,10 +232,7 @@ int PyType_Ready(PyTypeObject* cls) {
     PyDict_SetItemString(native_members, "tp_name", polyglot_from_string(cls->tp_name, SRC_CS));
     PyDict_SetItemString(native_members, "tp_doc", polyglot_from_string(cls->tp_doc ? cls->tp_doc : "", SRC_CS));
     PyDict_SetItemString(native_members, "tp_basicsize", PyLong_FromSsize_t(cls->tp_basicsize));
-    const char* lastDot = strrchr(cls->tp_name, '.');
-    if (lastDot) {
-        PyDict_SetItemString(native_members, "__module__", polyglot_from_string(lastDot + 1, SRC_CS));
-    }
+    const char* class_name = cls->tp_name;
     PyTypeObject* javacls = polyglot_invoke(PY_TRUFFLE_CEXT,
                                             "PyType_Ready",
                                             // no conversion of cls here, because we

graalpython/com.oracle.graal.python.test/src/tests/test_float.py

@@ -96,6 +96,8 @@ def __round__(x, n):
     def test_create(self):
         assert float(99) == 99
         assert float(999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999) == 1e+132
+        assert float(b"0.001") == 0.001
+        assert float("0.001") == 0.001
 
     def test_hex(self):
         data = [

graalpython/com.oracle.graal.python.test/src/tests/test_functions.py

@@ -72,6 +72,10 @@ def f2(a=f(1, 2), b=10):
     return a, b
 
 
+def f3(a, b=f(1,2), c=10, *args, d="hello", e="world"):
+    return a, b, c, args, d, e
+
+
 class MyClass(object):
     def __init__(self, x = 10):
         pass
@@ -82,6 +86,12 @@ def test_defaults():
     assert f2.__defaults__ == ((1, 2, 10, (), {}), 10)
 
 
+def test_kwdefaults():
+    assert f.__kwdefaults__ == None
+    assert f2.__kwdefaults__ == None
+    assert f3.__kwdefaults__ == { "d": "hello", "e": "world"}
+
+
 def test_defaults_method():
     obj = MyClass()
     assert obj.__init__.__defaults__ == (10,)

graalpython/com.oracle.graal.python.test/src/tests/test_metaclass.py

@@ -128,3 +128,17 @@ def __init__(self, a, b):
     assert len(global_log) == 4
     assert global_log[2] == ['__call__', MyClass, (1, 2), {}]
     assert global_log[3] == ['MyKlass object', 1, 2]
+    
+
+class A:
+    class B:
+        pass
+
+
+def test_nested_class():
+    assert A.__name__ == "A"
+    assert A.__qualname__ == "A"
+    assert A.__module__ == __name__
+    assert A.B.__name__ == "B"
+    assert A.B.__qualname__ == "A.B"
+    assert A.B.__module__ == __name__

graalpython/com.oracle.graal.python.test/src/tests/test_pickle.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import unittest
+import pickle
+
+class TestPickle(unittest.TestCase):
+
+    def test_builtin(self):
+        self.pickle_unpickle(len)
+        import sys
+        self.pickle_unpickle(sys.getrecursionlimit)
+
+    def test_local(self):
+        self.pickle_unpickle("test")
+        myvar = 10
+        self.pickle_unpickle(myvar)
+
+    def pickle_unpickle(self, obj):
+        b_obj = pickle.dumps(obj, protocol=0)
+        r_obj = pickle.loads(b_obj)
+        self.assertEqual(r_obj, obj)
+
+if __name__ == '__main__':
+    unittest.main()

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -425,6 +425,7 @@ public String[] builtinModuleNames() {
         return builtinModules.keySet().toArray(new String[0]);
     }
 
+    @Override
     public PythonModule getBuiltins() {
         return builtinsModule;
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

@@ -67,6 +67,7 @@
 import com.oracle.graal.python.builtins.objects.PEllipsis;
 import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.PNotImplemented;
+import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
 import com.oracle.graal.python.builtins.objects.bytes.BytesUtils;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
 import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
@@ -558,6 +559,8 @@ public Object reversed(PythonClass cls, Object sequence,
     @Builtin(name = FLOAT, minNumOfPositionalArgs = 1, maxNumOfPositionalArgs = 2, constructsClass = PythonBuiltinClassType.PFloat)
     @GenerateNodeFactory
     public abstract static class FloatNode extends PythonBuiltinNode {
+        @Child private BytesNodes.ToBytesNode toByteArrayNode;
+
         private final ConditionProfile isPrimitiveProfile = ConditionProfile.createBinaryProfile();
 
         protected boolean isPrimitiveFloat(Object cls) {
@@ -617,6 +620,16 @@ public Object floatFromString(PythonClass cls, String arg) {
             return factory().createFloat(cls, value);
         }
 
+        @Specialization(guards = "!isNativeClass(cls)")
+        @TruffleBoundary
+        public Object floatFromBytes(PythonClass cls, PIBytesLike arg) {
+            double value = convertStringToDouble(new String(getByteArray(arg)));
+            if (isPrimitiveFloat(cls)) {
+                return value;
+            }
+            return factory().createFloat(cls, value);
+        }
+
         // Taken from Jython PyString's atof() method
         // The last statement throw Py.ValueError is modified
         @TruffleBoundary
@@ -722,6 +735,14 @@ Object doPythonObject(PythonNativeClass cls, Object obj,
         public Object floatFromObject(@SuppressWarnings("unused") Object cls, Object arg) {
             throw raise(TypeError, "can't convert %s to float", arg.getClass().getSimpleName());
         }
+
+        private byte[] getByteArray(PIBytesLike pByteArray) {
+            if (toByteArrayNode == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                toByteArrayNode = insert(BytesNodes.ToBytesNode.create());
+            }
+            return toByteArrayNode.execute(pByteArray);
+        }
     }
 
     // frozenset([iterable])
@@ -780,7 +801,7 @@ private HashingCollectionNodes.SetItemNode getSetItemNode() {
     @GenerateNodeFactory
     public abstract static class IntNode extends PythonBuiltinNode {
 
-        @Child private SequenceStorageNodes.ToByteArrayNode toByteArrayNode;
+        @Child private BytesNodes.ToBytesNode toByteArrayNode;
 
         @TruffleBoundary(transferToInterpreterOnException = false)
         private Object stringToInt(String num, int base) {
@@ -980,30 +1001,32 @@ public Object createInt(PythonClass cls, String arg, @SuppressWarnings("unused")
         @Specialization(guards = "isPrimitiveInt(cls)", rewriteOn = NumberFormatException.class)
         @TruffleBoundary
         int parseInt(Object cls, PIBytesLike arg, int keywordArg) throws NumberFormatException {
-            return parseInt(cls, new String(getByteArray(arg)), keywordArg);
+            return parseInt(cls, toString(arg), keywordArg);
         }
 
         @Specialization(guards = "isPrimitiveInt(cls)", rewriteOn = NumberFormatException.class)
         @TruffleBoundary
         long parseLong(Object cls, PIBytesLike arg, int keywordArg) throws NumberFormatException {
-            return parseLong(cls, new String(getByteArray(arg)), keywordArg);
+            return parseLong(cls, toString(arg), keywordArg);
         }
 
-        @Specialization(rewriteOn = NumberFormatException.class)
-        @TruffleBoundary
-        Object parseBytes(PythonClass cls, PIBytesLike arg, int base) {
-            return parsePInt(cls, new String(getByteArray(arg)), base);
-        }
-
-        @Specialization(replaces = "parseBytes")
-        Object parseBytesError(PythonClass cls, PIBytesLike arg, int base) {
+        @Specialization
+        Object parseBytesError(PythonClass cls, PIBytesLike arg, int base,
+                        @Cached("create()") BranchProfile errorProfile) {
             try {
-                return parseBytes(cls, arg, base);
+                return parsePInt(cls, toString(arg), base);
             } catch (NumberFormatException e) {
+                errorProfile.enter();
                 throw raise(ValueError, "invalid literal for int() with base %s: %s", base, arg);
             }
         }
 
+        @Specialization(guards = "isNoValue(base)")
+        Object parseBytesError(PythonClass cls, PIBytesLike arg, @SuppressWarnings("unused") PNone base,
+                        @Cached("create()") BranchProfile errorProfile) {
+            return parseBytesError(cls, arg, 10, errorProfile);
+        }
+
         @Specialization(guards = "isPrimitiveInt(cls)", rewriteOn = NumberFormatException.class)
         int parseInt(Object cls, PString arg, int keywordArg) throws NumberFormatException {
             return parseInt(cls, arg.getValue(), keywordArg);
@@ -1020,6 +1043,11 @@ Object parsePInt(PythonClass cls, PString arg, int keywordArg) {
             return parsePInt(cls, arg.getValue(), keywordArg);
         }
 
+        @Specialization(guards = "isNoValue(base)")
+        Object parsePInt(PythonClass cls, PString arg, PNone base) {
+            return createInt(cls, arg.getValue(), base);
+        }
+
         @Specialization(guards = "isPrimitiveInt(cls)", rewriteOn = NumberFormatException.class)
         @TruffleBoundary
         int parseInt(@SuppressWarnings("unused") Object cls, String arg, int keywordArg) throws NumberFormatException {
@@ -1078,7 +1106,7 @@ Object fail(PythonClass cls, Object arg, Object keywordArg) {
             throw raise(TypeError, "int() can't convert non-string with explicit base");
         }
 
-        @Specialization(guards = {"isNoValue(keywordArg)", "!isNoValue(obj)"})
+        @Specialization(guards = {"isNoValue(keywordArg)", "!isNoValue(obj)", "!isHandledType(obj)"})
         public Object createInt(PythonClass cls, Object obj, PNone keywordArg,
                         @Cached("create(__INT__)") LookupAndCallUnaryNode callIntNode,
                         @Cached("create(__TRUNC__)") LookupAndCallUnaryNode callTruncNode,
@@ -1113,12 +1141,21 @@ public Object createInt(PythonClass cls, Object obj, PNone keywordArg,
             }
         }
 
-        private byte[] getByteArray(PIBytesLike pByteArray) {
+        protected static boolean isHandledType(Object obj) {
+            return PGuards.isInteger(obj) || obj instanceof Double || obj instanceof Boolean || PGuards.isString(obj) || PGuards.isBytes(obj);
+        }
+
+        private String toString(PIBytesLike pByteArray) {
             if (toByteArrayNode == null) {
                 CompilerDirectives.transferToInterpreterAndInvalidate();
-                toByteArrayNode = insert(SequenceStorageNodes.ToByteArrayNode.create());
+                toByteArrayNode = insert(BytesNodes.ToBytesNode.create());
             }
-            return toByteArrayNode.execute(pByteArray.getSequenceStorage());
+            return toString(toByteArrayNode.execute(pByteArray));
+        }
+
+        @TruffleBoundary
+        private static String toString(byte[] barr) {
+            return new String(barr);
         }
 
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

@@ -83,6 +83,7 @@
 import com.oracle.graal.python.builtins.objects.PNotImplemented;
 import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
+import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
 import com.oracle.graal.python.builtins.objects.cell.PCell;
 import com.oracle.graal.python.builtins.objects.code.PCode;
 import com.oracle.graal.python.builtins.objects.common.HashingCollectionNodes;
@@ -456,6 +457,7 @@ private static BigInteger[] divideAndRemainder(PInt a, PInt b) {
     public abstract static class EvalNode extends PythonBuiltinNode {
         @Child private GetItemNode getNameNode = GetItemNode.create();
         @Child private ReadCallerFrameNode readCallerFrameNode = ReadCallerFrameNode.create();
+        @Child private SequenceStorageNodes.ToByteArrayNode toByteArrayNode;
 
         @Specialization
         public Object eval(VirtualFrame frame, String expression, @SuppressWarnings("unused") PNone globals, @SuppressWarnings("unused") PNone locals) {
@@ -515,6 +517,39 @@ public Object eval(VirtualFrame frame, PCode code, @SuppressWarnings("unused") P
             return evalExpression(code, callerGlobals, locals, callerClosure);
         }
 
+        @Specialization
+        public Object eval(VirtualFrame frame, PBytes expression, PNone globals, PNone locals) {
+            return eval(frame, getString(expression), globals, locals);
+        }
+
+        @Specialization
+        public Object eval(VirtualFrame frame, PBytes expression, PythonObject globals, PNone locals) {
+            return eval(frame, getString(expression), globals, locals);
+        }
+
+        @Specialization
+        public Object eval(VirtualFrame frame, PBytes expression, PythonObject globals, PythonObject locals) {
+            return eval(frame, getString(expression), globals, locals);
+        }
+
+        @Specialization
+        public Object eval(VirtualFrame frame, PBytes expression, PNone globals, PythonObject locals) {
+            return eval(frame, getString(expression), globals, locals);
+        }
+
+        @TruffleBoundary
+        private String getString(PBytes expression) {
+            return new String(getByteArray(expression));
+        }
+
+        private byte[] getByteArray(PIBytesLike pByteArray) {
+            if (toByteArrayNode == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                toByteArrayNode = insert(SequenceStorageNodes.ToByteArrayNode.create());
+            }
+            return toByteArrayNode.execute(pByteArray.getSequenceStorage());
+        }
+
         @TruffleBoundary
         private static Object evalExpression(PCode code, PythonObject globals, PythonObject locals, PCell[] closure) {
             RootNode root = code.getRootNode();