[GR-34614][GR-44689] More fixes for PyTorch - native tuples, destructors

PullRequest: graalpython/2671

Author: msimacek

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -98,6 +98,7 @@ static void object_dealloc(PyObject *self) {
 
 /* prototype */
 PyObject* PyTruffle_Tuple_Alloc(PyTypeObject* cls, Py_ssize_t nitems);
+void PyTruffle_Tuple_Dealloc(PyTupleObject* tuple);
 
 PyAPI_DATA(PyTypeObject) _PyExc_BaseException;
 PyAPI_DATA(PyTypeObject) _PyExc_StopIteration;
@@ -145,7 +146,7 @@ PyTypeObject PySlice_Type = 				PY_TRUFFLE_TYPE("slice", &PyType_Type, Py_TPFLAG
 PyTypeObject PyStaticMethod_Type = 			PY_TRUFFLE_TYPE("staticmethod", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, sizeof(PyType_Type));
 PyTypeObject PySuper_Type = 				PY_TRUFFLE_TYPE("super", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, sizeof(superobject));
 PyTypeObject PyTraceBack_Type = 			PY_TRUFFLE_TYPE("traceback", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyTypeObject));
-PyTypeObject PyTuple_Type = 				PY_TRUFFLE_TYPE_GENERIC("tuple", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TUPLE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, sizeof(PyTupleObject) - sizeof(PyObject *), sizeof(PyObject *), PyTruffle_Tuple_Alloc, 0, 0, 0);
+PyTypeObject PyTuple_Type = 				PY_TRUFFLE_TYPE_GENERIC("tuple", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TUPLE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_SEQUENCE, sizeof(PyTupleObject) - sizeof(PyObject *), sizeof(PyObject *), PyTruffle_Tuple_Alloc, (destructor)PyTruffle_Tuple_Dealloc, 0, 0);
 PyTypeObject PyType_Type = 					PY_TRUFFLE_TYPE_WITH_ITEMSIZE("type", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS, sizeof(PyHeapTypeObject), sizeof(PyMemberDef));
 PyTypeObject PyUnicode_Type = 				PY_TRUFFLE_TYPE("str", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_UNICODE_SUBCLASS | _Py_TPFLAGS_MATCH_SELF, sizeof(PyUnicodeObject));
 /* NOTE: we use the same Python type (namely 'PBuiltinFunction') for 'wrapper_descriptor' as for 'method_descriptor'; so the flags must be the same! */
@@ -906,6 +907,10 @@ void PyTruffle_ObjectArrayFree(PyObject** array, int32_t size) {
     free(array);
 }
 
+void PyTruffle_SetStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
+    Py_XSETREF(ptr[index], newitem);
+}
+
 PyAPI_FUNC(Py_ssize_t) PyTruffle_Object_Size(PyObject *op) {
     return ((PyVarObject*)op)->ob_size;
 }

graalpython/com.oracle.graal.python.cext/src/pylifecycle.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -40,18 +40,15 @@
  */
 #include "capi.h"
 
+int Py_IsInitialized(void) {
+    return 1;
+}
+
+#ifndef COMPILING_NATIVE_CAPI
 void _Py_NO_RETURN  _Py_FatalErrorFunc(const char *func, const char *msg) {
 	GraalPyTruffle_FatalErrorFunc(func != NULL? truffleString(func): NULL, truffleString(msg), -1);
 	/* If the above upcall returns, then we just fall through to the 'abort' call. */
 	abort();
 }
+#endif // COMPILING_NATIVE_CAPI
 
-PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) {
-	PyErr_SetString(PyExc_SystemError, "'PyOS_setsig' not implemented");
-	return NULL;
-}
-
-int PyOS_InterruptOccurred(void) {
-	PyErr_SetString(PyExc_SystemError, "'PyOS_InterruptOccurred' not implemented");
-	return -1;
-}

graalpython/com.oracle.graal.python.cext/src/tupleobject.c

@@ -79,7 +79,6 @@ PyObject * tuple_subtype_new(PyTypeObject *type, PyObject *iterable) {
     if (newobj == NULL) {
         return NULL;
     }
-    newobj->ob_item = (PyObject **) ((char *)newobj + offsetof(PyTupleObject, ob_item) + sizeof(PyObject **));
 
     // This polyglot type cast is important such that we can directly read and
     // write members of the pointer from Java code.
@@ -95,15 +94,6 @@ PyObject * tuple_subtype_new(PyTypeObject *type, PyObject *iterable) {
     return (PyObject*) newobj;
 }
 
-int PyTruffle_Tuple_SetItem(PyObject* tuple, Py_ssize_t position, PyObject* item) {
-    PyTuple_SET_ITEM(tuple, position, item);
-    return 0;
-}
-
-PyObject* PyTruffle_Tuple_GetItem(PyObject* tuple, Py_ssize_t position) {
-    return PyTuple_GET_ITEM(tuple, position);
-}
-
 PyObject* PyTruffle_Tuple_Alloc(PyTypeObject* cls, Py_ssize_t nitems) {
 	/*
 	 * TODO(fa): For 'PyVarObjects' (i.e. 'nitems > 0') we increase the size by 'sizeof(void *)'
@@ -118,6 +108,22 @@ PyObject* PyTruffle_Tuple_Alloc(PyTypeObject* cls, Py_ssize_t nitems) {
     	*((PyObject **) ((char *)newObj + cls->tp_dictoffset)) = NULL;
     }
     PyObject_INIT_VAR(newObj, cls, nitems);
+    ((PyTupleObject*)newObj)->ob_item = (PyObject **) ((char *)newObj + offsetof(PyTupleObject, ob_item) + sizeof(PyObject **));
     return newObj;
 }
 
+void PyTruffle_Tuple_Dealloc(PyTupleObject* self) {
+    Py_ssize_t len =  PyTuple_GET_SIZE(self);
+    if (len > 0) {
+        Py_ssize_t i = len;
+        while (--i >= 0) {
+            Py_XDECREF(self->ob_item[i]);
+        }
+    }
+    Py_TYPE(self)->tp_free((PyObject *)self);
+}
+
+void* PyTruffle_NativeTupleItems(PyTupleObject* tuple) {
+    return polyglot_from_PyObjectPtr_array(tuple->ob_item, tuple->ob_base.ob_size);
+}
+

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -306,29 +306,31 @@ static void inherit_slots(PyTypeObject *type, PyTypeObject *base) {
         COPYSLOT(tp_iter);
         COPYSLOT(tp_iternext);
     }
-
-    if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_FINALIZE) &&
-        (PyTypeObject_tp_flags(base) & Py_TPFLAGS_HAVE_FINALIZE)) {
-        COPYSLOT(tp_finalize);
-    }
-    if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_GC) ==
-        (PyTypeObject_tp_flags(base) & Py_TPFLAGS_HAVE_GC)) {
-        /* They agree about gc. */
-        COPYSLOT(tp_free);
-    }
-    else if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_GC) &&
-    		PyTypeObject_tp_free(type) == NULL &&
-			PyTypeObject_tp_free(base) == PyObject_Free) {
-        /* A bit of magic to plug in the correct default
-         * tp_free function when a derived class adds gc,
-         * didn't define tp_free, and the base uses the
-         * default non-gc tp_free.
+    {
+        COPYSLOT(tp_alloc);
+        if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_FINALIZE) &&
+            (PyTypeObject_tp_flags(base) & Py_TPFLAGS_HAVE_FINALIZE)) {
+            COPYSLOT(tp_finalize);
+        }
+        if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_GC) ==
+            (PyTypeObject_tp_flags(base) & Py_TPFLAGS_HAVE_GC)) {
+            /* They agree about gc. */
+            COPYSLOT(tp_free);
+        }
+        else if ((PyTypeObject_tp_flags(type) & Py_TPFLAGS_HAVE_GC) &&
+                PyTypeObject_tp_free(type) == NULL &&
+                PyTypeObject_tp_free(base) == PyObject_Free) {
+            /* A bit of magic to plug in the correct default
+             * tp_free function when a derived class adds gc,
+             * didn't define tp_free, and the base uses the
+             * default non-gc tp_free.
+             */
+            set_PyTypeObject_tp_free(type, PyObject_GC_Del);
+        }
+        /* else they didn't agree about gc, and there isn't something
+         * obvious to be done -- the type is on its own.
          */
-    	set_PyTypeObject_tp_free(type, PyObject_GC_Del);
     }
-    /* else they didn't agree about gc, and there isn't something
-     * obvious to be done -- the type is on its own.
-     */
 }
 
 static int add_member(PyTypeObject* cls, PyObject* type_dict, const char* mname, int mtype, Py_ssize_t moffset, int mflags, char* mdoc) {

graalpython/com.oracle.graal.python.jni/src/capi_forwards.h

@@ -3354,10 +3354,8 @@ PyAPI_FUNC(PyObject*) PyOS_FSPath(PyObject* a) {
     PyObject* result = (PyObject*) __target__PyOS_FSPath(a);
     return result;
 }
-int (*__target__PyOS_InterruptOccurred)() = NULL;
 PyAPI_FUNC(int) PyOS_InterruptOccurred() {
-    int result = (int) __target__PyOS_InterruptOccurred();
-    return result;
+    unimplemented("PyOS_InterruptOccurred"); exit(-1);
 }
 PyAPI_FUNC(char*) PyOS_Readline(FILE* a, FILE* b, const char* c) {
     unimplemented("PyOS_Readline"); exit(-1);
@@ -3376,10 +3374,8 @@ PyAPI_FUNC(int) PyOS_mystricmp(const char* a, const char* b) {
 PyAPI_FUNC(int) PyOS_mystrnicmp(const char* a, const char* b, Py_ssize_t c) {
     unimplemented("PyOS_mystrnicmp"); exit(-1);
 }
-PyOS_sighandler_t (*__target__PyOS_setsig)(int, PyOS_sighandler_t) = NULL;
 PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int a, PyOS_sighandler_t b) {
-    PyOS_sighandler_t result = (PyOS_sighandler_t) __target__PyOS_setsig(a, b);
-    return result;
+    unimplemented("PyOS_setsig"); exit(-1);
 }
 double (*__target__PyOS_string_to_double)(const char*, char**, PyObject*) = NULL;
 PyAPI_FUNC(double) PyOS_string_to_double(const char* a, char** b, PyObject* c) {
@@ -4957,9 +4953,6 @@ PyAPI_FUNC(int) Py_IsFalse(PyObject* a) {
     int result = (int) __target__Py_IsFalse(a);
     return result;
 }
-PyAPI_FUNC(int) Py_IsInitialized() {
-    unimplemented("Py_IsInitialized"); exit(-1);
-}
 int (*__target__Py_IsNone)(PyObject*) = NULL;
 PyAPI_FUNC(int) Py_IsNone(PyObject* a) {
     int result = (int) __target__Py_IsNone(a);
@@ -6879,9 +6872,7 @@ void initializeCAPIForwards(void* (*getAPI)(const char*)) {
     __target__PyNumber_TrueDivide = getAPI("PyNumber_TrueDivide");
     __target__PyNumber_Xor = getAPI("PyNumber_Xor");
     __target__PyOS_FSPath = getAPI("PyOS_FSPath");
-    __target__PyOS_InterruptOccurred = getAPI("PyOS_InterruptOccurred");
     __target__PyOS_double_to_string = getAPI("PyOS_double_to_string");
-    __target__PyOS_setsig = getAPI("PyOS_setsig");
     __target__PyOS_string_to_double = getAPI("PyOS_string_to_double");
     __target__PyOS_strtol = getAPI("PyOS_strtol");
     __target__PyOS_strtoul = getAPI("PyOS_strtoul");

graalpython/com.oracle.graal.python.jni/src/capi_native.c

@@ -422,6 +422,16 @@ void Py_IncRef(PyObject *a) {
 }
 */
 
+/*
+This is a workaround for C++ modules, namely PyTorch, that declare global/static variables with destructors that call
+_Py_DECREF. The destructors get called by libc during exit during which we cannot make upcalls as that would segfault.
+So we rebind them to no-ops when exitting.
+*/
+static void nop_Py_DecRef(PyObject* obj) {}
+void finalizeCAPI() {
+    __target___Py_DecRef = nop_Py_DecRef;
+    __target__Py_DecRef = nop_Py_DecRef;
+}
 
 PyObject* PyTuple_Pack(Py_ssize_t n, ...) {
     va_list vargs;
@@ -557,11 +567,13 @@ char _PyByteArray_empty_string[] = "";
  * The following source files contain code that can be compiled directly and does not need to be called via stubs in Sulong:
  */
 
+#define COMPILING_NATIVE_CAPI
 #include "_warnings.c"
 #include "boolobject.c"
 #include "complexobject.c"
 #include "dictobject.c"
 #include "modsupport_shared.c"
+#include "pylifecycle.c"
 
 /*
  * This mirrors the definition in capi.c that we us on Sulong, and needs to be

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -490,7 +490,7 @@ def CPyExtType(name, code, **kwargs):
     {includes}
 
     typedef struct {{
-        PyObject_HEAD;
+        {struct_base};
         {cmembers}
     }} {name}Object;
 
@@ -636,6 +636,7 @@ def CPyExtType(name, code, **kwargs):
     kwargs.setdefault("tp_free", "PyObject_Del")
     kwargs.setdefault("cmembers", "")
     kwargs.setdefault("includes", "")
+    kwargs.setdefault("struct_base", "PyObject_HEAD")
     c_source = UnseenFormatter().format(template, **kwargs)
 
     source_file = "%s/%s.c" % (__dir__, name)