[GR-32014] Use intrinsified StructSequence for C API.

PullRequest: graalpython/1848

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/structseq.c

@@ -43,6 +43,8 @@
 #define REAL_SIZE_TP(tp) PyLong_AsSsize_t(PyDict_GetItem((tp)->tp_dict, polyglot_from_string("n_fields", SRC_CS)))
 #define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op))
 
+char *PyStructSequence_UnnamedField = "unnamed field";
+
 static void
 structseq_dealloc(PyStructSequence *obj)
 {
@@ -55,87 +57,104 @@ structseq_dealloc(PyStructSequence *obj)
     PyObject_GC_Del(obj);
 }
 
-/* StructSequences a.k.a. 'namedtuple' */
-UPCALL_ID(PyStructSequence_New);
+typedef PyObject *(*structseq_new_fun_t)(PyTypeObject *);
+UPCALL_TYPED_ID(PyStructSequence_New, structseq_new_fun_t);
 PyObject* PyStructSequence_New(PyTypeObject* o) {
-    return UPCALL_CEXT_O(_jls_PyStructSequence_New, native_type_to_java(o));
+    return _jls_PyStructSequence_New(native_type_to_java(o));
 }
 
-UPCALL_ID(PyStructSequence_InitType2);
-int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) {
-    Py_ssize_t n_members = desc->n_in_sequence;
+static Py_ssize_t
+count_members(PyStructSequence_Desc *desc, Py_ssize_t *n_unnamed_members) {
     Py_ssize_t i;
 
-    memset(type, 0, sizeof(PyTypeObject));
-
-    // put field names and doc strings into two tuples
-    PyObject* field_names = PyTuple_New(n_members);
-    PyObject* field_docs = PyTuple_New(n_members);
-    PyStructSequence_Field* fields = desc->fields;
-    for (i = 0; i < n_members; i++) {
-        PyTuple_SetItem(field_names, i, polyglot_from_string(fields[i].name, SRC_CS));
-        PyTuple_SetItem(field_docs, i, polyglot_from_string(fields[i].doc, SRC_CS));
+    *n_unnamed_members = 0;
+    for (i = 0; desc->fields[i].name != NULL; ++i) {
+        if (desc->fields[i].name == PyStructSequence_UnnamedField) {
+            (*n_unnamed_members)++;
+        }
     }
+    return i;
+}
 
-    // we create the new type managed
-    PyTypeObject* newType = (PyTypeObject*) UPCALL_CEXT_O(_jls_PyStructSequence_InitType2,
-            polyglot_from_string(desc->name, SRC_CS),
-            polyglot_from_string(desc->doc, SRC_CS),
-            native_to_java(field_names),
-            native_to_java(field_docs));
+typedef int (*structseq_init_fun_t)(void *, void *, void *, int);
+UPCALL_TYPED_ID(PyStructSequence_InitType2, structseq_init_fun_t);
+int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) {
+    Py_ssize_t n_members, n_unnamed_members, n_named_members;
+    Py_ssize_t i;
+
+    memset(type, 0, sizeof(PyTypeObject));
 
-    if (newType == NULL) {
-        Py_DECREF(field_names);
-        Py_DECREF(field_docs);
-        return -1;
-    }
 
     // copy generic fields (CPython mem-copies a template)
+    type->tp_name = desc->name;
     type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *);
     type->tp_itemsize = sizeof(PyObject *);
-    type->tp_repr = newType->tp_repr;
     type->tp_flags = Py_TPFLAGS_DEFAULT;
     type->tp_members = NULL;
-    type->tp_new = newType->tp_new;
+    type->tp_doc = desc->doc;
     type->tp_base = &PyTuple_Type;
-    type->tp_alloc = newType->tp_alloc;
     type->tp_dealloc = (destructor)structseq_dealloc;
 
-    // now copy specific fields
-    type->tp_name = newType->tp_name;
-    type->tp_doc = newType->tp_doc;
-    type->tp_dict = newType->tp_dict;
-
-    Py_DECREF(newType);
-
     // now initialize the type
     if (PyType_Ready(type) < 0)
         return -1;
     Py_INCREF(type);
 
-    return 0;
+    n_members = count_members(desc, &n_unnamed_members);
+    n_named_members = n_members - n_unnamed_members;
+    // put field names and doc strings into two lists
+    void** field_names = (void **) truffle_managed_malloc(n_named_members * sizeof(void *));
+    void** field_docs = (void **) truffle_managed_malloc(n_named_members * sizeof(void *));
+    PyStructSequence_Field* fields = desc->fields;
+    int j = 0;
+    for (i = 0; i < n_members; i++) {
+        if (fields[i].name != PyStructSequence_UnnamedField) {
+            field_names[j] = polyglot_from_string(fields[i].name, SRC_CS);
+            field_docs[j] = polyglot_from_string(fields[i].doc, SRC_CS);
+            j++;
+        }
+    }
+
+    // this initializes the type dict (adds attributes)
+    return _jls_PyStructSequence_InitType2(
+            native_type_to_java(type),
+            /* TODO(fa): use polyglot_from_VoidPtr_array once this is visible */
+            polyglot_from_PyObjectPtr_array((PyObjectPtr *) field_names, (uint64_t) n_members),
+            polyglot_from_PyObjectPtr_array((PyObjectPtr *) field_docs, (uint64_t) n_members),
+            desc->n_in_sequence
+    );
 }
 
+typedef PyTypeObject *(*structseq_newtype_fun_t)(void *, void *, void *, void *, int);
+UPCALL_TYPED_ID(PyStructSequence_NewType, structseq_newtype_fun_t);
 PyTypeObject* PyStructSequence_NewType(PyStructSequence_Desc *desc) {
-    Py_ssize_t n_members = desc->n_in_sequence;
+    Py_ssize_t n_members, n_unnamed_members, n_named_members;
     Py_ssize_t i;
 
-    // put field names and doc strings into two tuples
-    PyObject* field_names = PyTuple_New(n_members);
-    PyObject* field_docs = PyTuple_New(n_members);
+    n_members = count_members(desc, &n_unnamed_members);
+    n_named_members = n_members - n_unnamed_members;
+    // put field names and doc strings into two lists
+    void** field_names = (void **) truffle_managed_malloc(n_named_members * sizeof(void *));
+    void** field_docs = (void **) truffle_managed_malloc(n_named_members * sizeof(void *));
     PyStructSequence_Field* fields = desc->fields;
+    int j = 0;
     for (i = 0; i < n_members; i++) {
-    	PyTuple_SetItem(field_names, i, polyglot_from_string(fields[i].name, SRC_CS));
-    	PyTuple_SetItem(field_docs, i, polyglot_from_string(fields[i].doc, SRC_CS));
+        if (fields[i].name != PyStructSequence_UnnamedField) {
+            field_names[j] = polyglot_from_string(fields[i].name, SRC_CS);
+            field_docs[j] = polyglot_from_string(fields[i].doc, SRC_CS);
+            j++;
+        }
     }
 
     // we create the new type managed
-    PyTypeObject* newType = (PyTypeObject*) UPCALL_CEXT_O(_jls_PyStructSequence_InitType2,
-    		polyglot_from_string(desc->name, SRC_CS),
-			polyglot_from_string(desc->doc, SRC_CS),
-			native_to_java(field_names),
-			native_to_java(field_docs));
-    return newType;
+    return _jls_PyStructSequence_NewType(
+            polyglot_from_string(desc->name, SRC_CS),
+            polyglot_from_string(desc->doc, SRC_CS),
+            /* TODO(fa): use polyglot_from_VoidPtr_array once this is visible */
+            polyglot_from_PyObjectPtr_array((PyObjectPtr *) field_names, (uint64_t) n_members),
+            polyglot_from_PyObjectPtr_array((PyObjectPtr *) field_docs, (uint64_t) n_members),
+            desc->n_in_sequence
+    );
 }
 
 // taken from CPython "Objects/structseq.c"

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -408,6 +408,7 @@ static void add_method_or_slot(PyTypeObject* cls, PyObject* type_dict, char* nam
 
 UPCALL_ID(PyTruffle_Get_Inherited_Native_Slots);
 UPCALL_ID(PyTruffle_Compute_Mro);
+UPCALL_ID(PyTruffle_NewTypeDict);
 int PyType_Ready(PyTypeObject* cls) {
 #define RETURN_ERROR(__type__) \
     do { \
@@ -492,7 +493,7 @@ int PyType_Ready(PyTypeObject* cls) {
     /* Initialize tp_dict */
     PyObject* dict = cls->tp_dict;
     if (dict == NULL) {
-        dict = PyDict_New();
+        dict = UPCALL_CEXT_O(_jls_PyTruffle_NewTypeDict, native_type_to_java(cls));
         if (dict == NULL) {
         	RETURN_ERROR(cls);
         }

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -574,16 +574,15 @@ def CPyExtType(name, code, **kwargs):
     PyMODINIT_FUNC
     PyInit_{name}(void)
     {{
-        PyObject* m;
+        PyObject* m = PyModule_Create(&{name}module);
+        if (m == NULL)
+            return NULL;
 
         {ready_code}
         if (PyType_Ready(&{name}Type) < 0)
             return NULL;
         {post_ready_code}
 
-        m = PyModule_Create(&{name}module);
-        if (m == NULL)
-            return NULL;
 
         Py_INCREF(&{name}Type);
         PyModule_AddObject(m, "{name}", (PyObject *)&{name}Type);

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_structseq.py

@@ -39,64 +39,104 @@
 
 import sys
 from . import CPyExtType, CPyExtTestCase, CPyExtFunction, CPyExtFunctionOutVars, unhandled_error_compare, GRAALPYTHON
+
 __dir__ = __file__.rpartition("/")[0]
 
-# 
-# class TestPyStructSequenceTypes(object):
-#     def test_properties(self):
-#         TestPyStructSequence = CPyExtType("TestPyStructSequence",
-#                                           """
-#                                static PyStructSequence_Field typeinfo_fields[] = {
-#                                    {"element0",      "The first element."},
-#                                    {"element1",      "The second element."},
-#                                    {NULL, NULL,}
-#                                };
-#            
-#                                static PyStructSequence_Desc typeinfo_desc = { 
-#                                    "TestPyStructSequenceTypes.TestPyStructSequence",
-#                                    "Information about some custom struct type",
-#                                    typeinfo_fields,
-#                                    2,
-#                                };
-#                                 """,
-#                                 ready_code="""if(PyStructSequence_InitType2(&TestPyStructSequenceType, &typeinfo_desc) < 0) {
-#                                  return NULL;
-#                              }
-#                              Py_INCREF(&TestPyStructSequenceType);
-#                              """,
-#         )
-#         assert TestPyStructSequence.__doc__ == "Information about some custom struct type"
-# 
-#         tester = TestPyStructSequence()
-#         assert hasattr(tester.element0)
-#         assert hasattr(tester.element1)
+
+class TestPyStructSequenceTypes(object):
+    def test_properties(self):
+        TestPyStructSequence = CPyExtType("TestPyStructSequence",
+                                          """
+                               static PyStructSequence_Field typeinfo_fields[] = {
+                                   {"element0",      "The first element."},
+                                   {"element1",      "The second element."},
+                                   {NULL, NULL,}
+                               };
+
+                               static PyStructSequence_Desc typeinfo_desc = { 
+                                   "TestPyStructSequenceTypes.TestPyStructSequence",
+                                   "Information about some custom struct type",
+                                   typeinfo_fields,
+                                   2,
+                               };
+                                """,
+                                          ready_code="""
+                               /* our template initializes refcnt to 1; so reset */
+                               Py_REFCNT(&TestPyStructSequenceType) = 0;
+                               if(PyStructSequence_InitType2(&TestPyStructSequenceType, &typeinfo_desc) < 0) {
+                                 return NULL;
+                               }
+                               Py_INCREF(&TestPyStructSequenceType);
+                               """,
+                                          )
+        assert TestPyStructSequence.__doc__ == "Information about some custom struct type"
+
+        tester = TestPyStructSequence(("hello", "world"))
+        assert hasattr(tester, "element0")
+        assert hasattr(tester, "element1")
+        assert tester.element0 == "hello"
+        assert tester.element1 == "world"
+
+    def test_structseq_newtype(self):
+        TestPyStructSequenceNewType = CPyExtType("TestPyStructSequenceNewType", """
+                               static PyStructSequence_Field typeinfo_fields[] = {
+                                   {"element0",      "The first element."},
+                                   {"element1",      "The second element."},
+                                   {"element2",      "The third element."},
+                                   {NULL, NULL,}
+                               };
+
+                               static PyStructSequence_Desc typeinfo_desc = {
+                                   "TestPyStructSequenceNewType.TestPyStructSequenceNewType",
+                                   "Information about some new struct type",
+                                   typeinfo_fields,
+                                   3,
+                               };
+                                """,
+                                post_ready_code="""
+                                PyTypeObject *new_type = PyStructSequence_NewType(&typeinfo_desc);
+                                if (new_type == NULL)
+                                     return NULL;
+                                PyModule_AddObject(m, "TestPyStructSequenceNewType", (PyObject *)new_type);
+                                return m;
+                             """,
+        )
+        assert TestPyStructSequenceNewType.__doc__ == "Information about some new struct type"
+
+        tester = TestPyStructSequenceNewType(("hello", "beautiful", "world"))
+        assert hasattr(tester, "element0")
+        assert hasattr(tester, "element1")
+        assert hasattr(tester, "element2")
+        assert tester.element0 == "hello"
+        assert tester.element1 == "beautiful"
+        assert tester.element2 == "world"
 
 
 class TestPyStructSequence(CPyExtTestCase):
- 
+
     def compile_module(self, name):
         type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
         super(TestPyStructSequence, self).compile_module(name)
- 
+
     test_PyStructSequence_InitType2 = CPyExtFunction(
         lambda args: 0,
-        lambda: ( (0,), ),
+        lambda: ((0,),),
         code='''
         static PyTypeObject CustomStructSeqType;
-          
+
         static PyStructSequence_Field typeinfo_fields[] = {
             {"element0",      "The first element."},
             {"element1",      "The second element."},
             {NULL, NULL,}
         };
-          
+
         static PyStructSequence_Desc typeinfo_desc = { 
             "custom.named.tuple",
             "Information about some custom struct type",
             typeinfo_fields,
             2,
         };
-          
+
         int wrap_PyStructSequence_InitType2(int n) {
             return PyStructSequence_InitType2(&CustomStructSeqType, &typeinfo_desc);
         }
@@ -110,12 +150,8 @@ def compile_module(self, name):
 
     test_PyStructSequence_Usage = CPyExtFunction(
         lambda args: args[0],
-        lambda: ( 
+        lambda: (
             (("hello", "world",),),
-#             ("john", "doe",), 
-#             (1, "doe",), 
-#             ("john", False,), 
-#             ("john", None,), 
         ),
         code='''
         static PyTypeObject CustomStructSeqType;
@@ -125,14 +161,14 @@ def compile_module(self, name):
             {"element1",      "The second element."},
             {NULL, NULL,}
         };
-         
-        static PyStructSequence_Desc typeinfo_desc = { 
+
+        static PyStructSequence_Desc typeinfo_desc = {
             "custom.named.tuple",
             "Information about some custom struct type",
             typeinfo_fields,
             2,
         };
-         
+
         PyObject* wrap_PyStructSequence_Usage(PyObject* elements) {
             Py_ssize_t i = 0;
             PyObject* elem = NULL;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -32,6 +32,7 @@
 import java.util.concurrent.Semaphore;
 import java.util.logging.Level;
 
+import com.oracle.graal.python.builtins.objects.type.PythonAbstractClass;
 import org.graalvm.options.OptionDescriptors;
 import org.graalvm.options.OptionKey;
 import org.graalvm.options.OptionValues;
@@ -811,7 +812,7 @@ public Shape getEmptyShape() {
         return emptyShape;
     }
 
-    public Shape getShapeForClass(PythonManagedClass klass) {
+    public Shape getShapeForClass(PythonAbstractClass klass) {
         if (singleContextAssumption.isValid()) {
             return Shape.newBuilder(getEmptyShape()).addConstantProperty(HiddenAttributes.CLASS, klass, 0).build();
         } else {