Skip to content

Commit 1b33592

Browse files
fangererfangerer
committed
[GR-26683] Support multi-phase extension module initialization.
PullRequest: graalpython/1816
2 parents 7f652d2 + edd6371 commit 1b33592

File tree

25 files changed

+1123
-484
lines changed

25 files changed

+1123
-484
lines changed

‎graalpython/com.oracle.graal.python.cext/modules/_testmultiphase.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
/* Testing module for multi-phase initialization of extension modules (PEP 489)
77
*/
88

9+
/* For the tests, we need doc strings. */
10+
#define WITH_DOC_STRINGS 1
11+
912
#include "Python.h"
1013

1114
/* Example objects */

‎graalpython/com.oracle.graal.python.cext/src/capi.c‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,40 @@ unsigned long get_tp_flags(PyTypeObject* obj) {
374374
return obj->tp_flags;
375375
}
376376

377+
POLYGLOT_DECLARE_TYPE(PyMethodDef);
378+
/**
379+
* To be used from Java code only. Reads native 'PyModuleDef.m_methods' field and
380+
* returns a typed pointer that can be used as interop array.
381+
*/
382+
PyMethodDef* get_PyModuleDef_m_methods(PyModuleDef* moduleDef) {
383+
PyMethodDef* members = moduleDef->m_methods;
384+
if (members) {
385+
uint64_t i = 0;
386+
while (members[i].ml_name != NULL) {
387+
i++;
388+
}
389+
return polyglot_from_PyMethodDef_array(members, i);
390+
}
391+
return NULL;
392+
}
393+
394+
POLYGLOT_DECLARE_TYPE(PyModuleDef_Slot);
395+
/**
396+
* To be used from Java code only. Reads native 'PyModuleDef.m_slots' field and
397+
* returns a typed pointer that can be used as interop array.
398+
*/
399+
PyModuleDef_Slot* get_PyModuleDef_m_slots(PyModuleDef* moduleDef) {
400+
PyModuleDef_Slot* slots = moduleDef->m_slots;
401+
if (slots) {
402+
uint64_t i = 0;
403+
while (slots[i].slot != 0) {
404+
i++;
405+
}
406+
return polyglot_from_PyModuleDef_Slot_array(slots, i);
407+
}
408+
return NULL;
409+
}
410+
377411
/** to be used from Java code only; returns the type ID for a byte array */
378412
polyglot_typeid get_byte_array_typeid(uint64_t len) {
379413
return polyglot_array_typeid(polyglot_i8_typeid(), len);
@@ -467,7 +501,7 @@ void PyTruffle_Free(void* obj) {
467501
if(points_to_handle_space(obj) && is_handle(obj)) {
468502
release_handle(obj);
469503
} else {
470-
free(obj);
504+
PyMem_RawFree(obj);
471505
}
472506
}
473507

‎graalpython/com.oracle.graal.python.cext/src/capi.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,9 @@ PyObject* native_to_java_stealing(PyObject* obj) {
262262

263263
MUST_INLINE
264264
PyTypeObject* native_type_to_java(PyTypeObject* type) {
265+
if (type == NULL) {
266+
return NULL;
267+
}
265268
if (points_to_handle_space(type)) {
266269
return (PyTypeObject *)resolve_handle(type);
267270
}

‎graalpython/com.oracle.graal.python.cext/src/moduleobject.c‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,14 @@
4141
#include "capi.h"
4242

4343
PyTypeObject PyModule_Type = PY_TRUFFLE_TYPE("module", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, sizeof(PyModuleObject));
44-
45-
PyTypeObject PyModuleDef_Type = {
46-
PyVarObject_HEAD_INIT(&PyType_Type, 0)
47-
"moduledef", /* tp_name */
48-
sizeof(struct PyModuleDef), /* tp_basicsize */
49-
0, /* tp_itemsize */
50-
};
44+
PyTypeObject PyModuleDef_Type = PY_TRUFFLE_TYPE("moduledef", &PyType_Type, 0, sizeof(struct PyModuleDef));
5145

5246

5347
UPCALL_ID(_PyModule_GetAndIncMaxModuleNumber);
5448
// partially taken from CPython "Objects/moduleobject.c"
5549
PyObject*
5650
PyModuleDef_Init(struct PyModuleDef* def)
5751
{
58-
if (PyType_Ready(&PyModuleDef_Type) < 0)
59-
return NULL;
6052
if (def->m_base.m_index == 0) {
6153
Py_REFCNT(def) = 1;
6254
Py_TYPE(def) = &PyModuleDef_Type;
@@ -154,6 +146,18 @@ PyObject* PyModule_NewObject(PyObject* name) {
154146
return UPCALL_CEXT_O(_jls_PyModule_NewObject, native_to_java(name));
155147
}
156148

149+
PyObject* PyModule_New(const char *name) {
150+
return UPCALL_CEXT_O(_jls_PyModule_NewObject, polyglot_from_string(name, SRC_CS));
151+
}
152+
153+
PyModuleDef* PyModule_GetDef(PyObject* m) {
154+
if (!PyModule_Check(m)) {
155+
PyErr_BadArgument();
156+
return NULL;
157+
}
158+
return ((PyModuleObject *)m)->md_def;
159+
}
160+
157161
void* PyModule_GetState(PyObject *m) {
158162
if (!PyModule_Check(m)) {
159163
PyErr_BadArgument();

‎graalpython/com.oracle.graal.python.cext/src/pystate.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,36 @@ UPCALL_ID(PyState_FindModule)
6262
PyObject* PyState_FindModule(struct PyModuleDef* module) {
6363
return UPCALL_CEXT_O(_jls_PyState_FindModule, polyglot_from_string(module->m_name, SRC_CS));
6464
}
65+
66+
int PyState_AddModule(PyObject* module, struct PyModuleDef* def) {
67+
Py_ssize_t index;
68+
if (!def) {
69+
Py_FatalError("PyState_AddModule: Module Definition is NULL");
70+
return -1;
71+
}
72+
// TODO(fa): check if module was already added
73+
74+
if (def->m_slots) {
75+
PyErr_SetString(PyExc_SystemError,
76+
"PyState_AddModule called on module with slots");
77+
return -1;
78+
}
79+
80+
// TODO(fa): implement
81+
return 0;
82+
}
83+
84+
int PyState_RemoveModule(struct PyModuleDef* def) {
85+
Py_ssize_t index = def->m_base.m_index;
86+
if (def->m_slots) {
87+
PyErr_SetString(PyExc_SystemError,
88+
"PyState_RemoveModule called on module with slots");
89+
return -1;
90+
}
91+
if (index == 0) {
92+
Py_FatalError("PyState_RemoveModule: Module index invalid.");
93+
return -1;
94+
}
95+
// TODO(fa): implement
96+
return 0;
97+
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java‎

Lines changed: 57 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
*/
4141
package com.oracle.graal.python.builtins.modules;
4242

43-
import static com.oracle.graal.python.nodes.SpecialAttributeNames.__FILE__;
4443
import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError;
4544

4645
import java.io.IOException;
@@ -50,27 +49,31 @@
5049
import com.oracle.graal.python.PythonLanguage;
5150
import com.oracle.graal.python.builtins.Builtin;
5251
import com.oracle.graal.python.builtins.CoreFunctions;
52+
import com.oracle.graal.python.builtins.Python3Core;
53+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
5354
import com.oracle.graal.python.builtins.PythonBuiltins;
54-
import com.oracle.graal.python.builtins.modules.PythonCextBuiltinsFactory.DefaultCheckFunctionResultNodeGen;
5555
import com.oracle.graal.python.builtins.objects.PNone;
5656
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
5757
import com.oracle.graal.python.builtins.objects.bytes.PBytesLike;
58+
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.ExecModuleNode;
59+
import com.oracle.graal.python.builtins.objects.cext.capi.DynamicObjectNativeWrapper;
60+
import com.oracle.graal.python.builtins.objects.cext.capi.ExternalFunctionNodesFactory.DefaultCheckFunctionResultNodeGen;
61+
import com.oracle.graal.python.builtins.objects.cext.capi.NativeMember;
5862
import com.oracle.graal.python.builtins.objects.cext.common.CExtCommonNodes.CheckFunctionResultNode;
5963
import com.oracle.graal.python.builtins.objects.cext.common.CExtContext;
64+
import com.oracle.graal.python.builtins.objects.cext.common.CExtContext.ModuleSpec;
6065
import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ApiInitException;
6166
import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ImportException;
6267
import com.oracle.graal.python.builtins.objects.cext.hpy.HPyExternalFunctionNodes.HPyCheckFunctionResultNode;
6368
import com.oracle.graal.python.builtins.objects.cext.hpy.HPyExternalFunctionNodesFactory.HPyCheckHandleResultNodeGen;
6469
import com.oracle.graal.python.builtins.objects.code.PCode;
65-
import com.oracle.graal.python.builtins.objects.dict.PDict;
70+
import com.oracle.graal.python.builtins.objects.common.HashingStorageLibrary;
6671
import com.oracle.graal.python.builtins.objects.ints.IntBuiltins;
6772
import com.oracle.graal.python.builtins.objects.ints.PInt;
6873
import com.oracle.graal.python.builtins.objects.module.PythonModule;
6974
import com.oracle.graal.python.builtins.objects.object.PythonObject;
7075
import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
7176
import com.oracle.graal.python.builtins.objects.str.PString;
72-
import com.oracle.graal.python.nodes.ErrorMessages;
73-
import com.oracle.graal.python.nodes.PRaiseNode;
7477
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromDynamicObjectNode;
7578
import com.oracle.graal.python.nodes.attributes.SetAttributeNode;
7679
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -81,14 +84,13 @@
8184
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
8285
import com.oracle.graal.python.runtime.GilNode;
8386
import com.oracle.graal.python.runtime.PythonContext;
84-
import com.oracle.graal.python.builtins.Python3Core;
8587
import com.oracle.graal.python.runtime.PythonOptions;
86-
import com.oracle.graal.python.runtime.exception.PythonErrorType;
8788
import com.oracle.truffle.api.CompilerDirectives;
8889
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
8990
import com.oracle.truffle.api.dsl.Cached;
9091
import com.oracle.truffle.api.dsl.CachedContext;
9192
import com.oracle.truffle.api.dsl.CachedLanguage;
93+
import com.oracle.truffle.api.dsl.Fallback;
9294
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
9395
import com.oracle.truffle.api.dsl.NodeFactory;
9496
import com.oracle.truffle.api.dsl.Specialization;
@@ -212,7 +214,7 @@ Object run(VirtualFrame frame, PythonObject moduleSpec, @SuppressWarnings("unuse
212214
PythonContext context = getContext();
213215
Object state = IndirectCallContext.enter(frame, language, context, this);
214216
try {
215-
return run(context, name, path);
217+
return run(context, new ModuleSpec(name, path, moduleSpec));
216218
} catch (ApiInitException ie) {
217219
throw ie.reraise(getConstructAndRaiseNode(), frame);
218220
} catch (ImportException ie) {
@@ -225,29 +227,16 @@ Object run(VirtualFrame frame, PythonObject moduleSpec, @SuppressWarnings("unuse
225227
}
226228

227229
@TruffleBoundary
228-
private Object run(PythonContext context, String name, String path) throws IOException, ApiInitException, ImportException {
229-
230-
Object existingModule = findExtensionObject(name, path);
230+
private Object run(PythonContext context, ModuleSpec spec) throws IOException, ApiInitException, ImportException {
231+
Object existingModule = findExtensionObject(spec);
231232
if (existingModule != null) {
232233
return existingModule;
233234
}
234-
235-
Object result = CExtContext.loadCExtModule(this, context, name, path, getCheckResultNode(), getCheckHPyResultNode());
236-
if (!(result instanceof PythonModule)) {
237-
// PyModuleDef_Init(pyModuleDef)
238-
// TODO: PyModule_FromDefAndSpec((PyModuleDef*)m, spec);
239-
throw PRaiseNode.raiseUncached(this, PythonErrorType.NotImplementedError, ErrorMessages.MULTI_PHASE_INIT_OF_EXTENSION_MODULE_S, name);
240-
} else {
241-
((PythonModule) result).setAttribute(__FILE__, path);
242-
// TODO: _PyImport_FixupExtensionObject(result, name, path, sys.modules)
243-
PDict sysModules = context.getSysModules();
244-
sysModules.setItem(name, result);
245-
return result;
246-
}
235+
return CExtContext.loadCExtModule(this, context, spec, getCheckResultNode(), getCheckHPyResultNode());
247236
}
248237

249238
@SuppressWarnings({"static-method", "unused"})
250-
private Object findExtensionObject(String name, String path) {
239+
private Object findExtensionObject(ModuleSpec spec) {
251240
// TODO: to avoid initializing an extension module twice, keep an internal dict
252241
// and possibly return from there, i.e., _PyImport_FindExtensionObject(name, path)
253242
return null;
@@ -270,13 +259,52 @@ private HPyCheckFunctionResultNode getCheckHPyResultNode() {
270259
}
271260
}
272261

273-
@Builtin(name = "exec_dynamic", minNumOfPositionalArgs = 1)
262+
@Builtin(name = "exec_dynamic", minNumOfPositionalArgs = 1, doc = "exec_dynamic($module, mod, /)\n--\n\nInitialize an extension module.")
274263
@GenerateNodeFactory
275264
public abstract static class ExecDynamicNode extends PythonBuiltinNode {
276265
@Specialization
277-
public Object run(PythonModule extensionModule) {
278-
// TODO: implement PyModule_ExecDef
279-
return extensionModule;
266+
int doPythonModule(VirtualFrame frame, PythonModule extensionModule,
267+
@CachedLanguage PythonLanguage language,
268+
@CachedLibrary(limit = "1") HashingStorageLibrary lib,
269+
@Cached ExecModuleNode execModuleNode) {
270+
Object nativeModuleDef = extensionModule.getNativeModuleDef();
271+
if (nativeModuleDef == null) {
272+
return 0;
273+
}
274+
275+
/*
276+
* Check if module is already initialized. CPython does that by testing if 'md_state !=
277+
* NULL'. So, we do the same. Currently, we store this in the generic storage of the
278+
* native wrapper.
279+
*/
280+
DynamicObjectNativeWrapper nativeWrapper = extensionModule.getNativeWrapper();
281+
if (nativeWrapper != null && nativeWrapper.getNativeMemberStore() != null) {
282+
Object item = lib.getItem(nativeWrapper.getNativeMemberStore(), NativeMember.MD_STATE.getMemberName());
283+
if (item != PNone.NO_VALUE) {
284+
return 0;
285+
}
286+
}
287+
288+
PythonContext context = getContext();
289+
if (!context.hasCApiContext()) {
290+
throw raise(PythonBuiltinClassType.SystemError, "C API not yet initialized");
291+
}
292+
293+
/*
294+
* ExecModuleNode will run the module definition's exec function which may run arbitrary
295+
* C code. So we need to setup an indirect call.
296+
*/
297+
Object state = IndirectCallContext.enter(frame, language, context, this);
298+
try {
299+
return execModuleNode.execute(context.getCApiContext(), extensionModule, nativeModuleDef);
300+
} finally {
301+
IndirectCallContext.exit(frame, language, context, state);
302+
}
303+
}
304+
305+
@Fallback
306+
static int doOther(@SuppressWarnings("unused") Object extensionModule) {
307+
return 0;
280308
}
281309
}
282310

0 commit comments

Comments
 (0)