Skip to content

Commit 74bceee

Browse files
fangererfangerer
committed
Support native buffer protocol for 'mmap'.
1 parent 21bcbba commit 74bceee

File tree

9 files changed

+331
-119
lines changed

9 files changed

+331
-119
lines changed

‎graalpython/com.oracle.graal.python.cext/modules/_mmap.c‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/* Copyright (c) 2019, Oracle and/or its affiliates.
2+
* Copyright (C) 1996-2017 Python Software Foundation
3+
*
4+
* Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5+
*/
6+
7+
#include "../src/capi.h"
8+
9+
typedef enum
10+
{
11+
ACCESS_DEFAULT,
12+
ACCESS_READ,
13+
ACCESS_WRITE,
14+
ACCESS_COPY
15+
} access_mode;
16+
17+
typedef struct {
18+
PyObject_HEAD
19+
char * data;
20+
Py_ssize_t size;
21+
Py_ssize_t pos; /* relative to offset */
22+
#ifdef MS_WINDOWS
23+
long long offset;
24+
#else
25+
off_t offset;
26+
#endif
27+
int exports;
28+
29+
#ifdef MS_WINDOWS
30+
HANDLE map_handle;
31+
HANDLE file_handle;
32+
char * tagname;
33+
#endif
34+
35+
#ifdef UNIX
36+
int fd;
37+
#endif
38+
39+
PyObject *weakreflist;
40+
access_mode access;
41+
} mmap_object;
42+
43+
44+
POLYGLOT_DECLARE_TYPE(mmap_object);
45+
static PyTypeObject mmap_object_type = PY_TRUFFLE_TYPE("mmap.mmap", NULL, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, sizeof(mmap_object));
46+
47+
int mmap_getbuffer(PyObject *self, Py_buffer *view, int flags) {
48+
// TODO(fa) readonly flag
49+
return PyBuffer_FillInfo(view, (PyObject*)self, ((mmap_object *)self)->data, PyObject_Size((PyObject *)self), 0, flags);
50+
}
51+
52+
static PyObject* mmap_init_bufferprotocol(PyObject* self, PyObject* mmap_type) {
53+
assert(PyType_Check(mmap_type));
54+
initialize_type_structure(&mmap_object_type, mmap_type, polyglot_mmap_object_typeid());
55+
56+
polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java(mmap_type), (getbufferproc) mmap_getbuffer, (releasebufferproc) NULL);
57+
return Py_None;
58+
}
59+
60+
static struct PyMethodDef module_functions[] = {
61+
{"init_bufferprotocol", mmap_init_bufferprotocol, METH_O, NULL},
62+
{NULL, NULL} /* sentinel */
63+
};
64+
65+
static struct PyModuleDef mmapmodule = {
66+
PyModuleDef_HEAD_INIT,
67+
"_mmap",
68+
NULL,
69+
-1,
70+
module_functions,
71+
NULL,
72+
NULL,
73+
NULL,
74+
NULL
75+
};
76+
77+
PyMODINIT_FUNC
78+
PyInit__mmap(void)
79+
{
80+
PyObject *dict, *module;
81+
82+
module = PyModule_Create(&mmapmodule);
83+
if (module == NULL) {
84+
return NULL;
85+
}
86+
dict = PyModule_GetDict(module);
87+
if (!dict) {
88+
return NULL;
89+
}
90+
91+
return module;
92+
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ public boolean run() {
145145
public abstract static class CreateDynamic extends PythonBuiltinNode {
146146
protected static final String INITIALIZE_CAPI = "initialize_capi";
147147
protected static final String IMPORT_NATIVE_MEMORYVIEW = "import_native_memoryview";
148+
protected static final String RUN_CAPI_LOADED_HOOKS = "run_capi_loaded_hooks";
148149
private static final String LLVM_LANGUAGE = "llvm";
149150
@Child private SetItemNode setItemNode;
150151
@Child private CheckFunctionResultNode checkResultNode;
@@ -246,6 +247,7 @@ private void ensureCapiWasLoaded() {
246247
CallUnaryMethodNode callNode = insert(CallUnaryMethodNode.create());
247248
callNode.executeObject(readNode.execute(ctxt.getCore().lookupBuiltinModule("python_cext"), INITIALIZE_CAPI), capi);
248249
ctxt.setCapiWasLoaded(capi);
250+
callNode.executeObject(readNode.execute(ctxt.getCore().lookupBuiltinModule("python_cext"), RUN_CAPI_LOADED_HOOKS), capi);
249251

250252
// initialization needs to be finished already but load memoryview implemenation
251253
// immediately

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/NativeMemberNames.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ public final class NativeMemberNames {
127127
public static final String M_ML = "m_ml";
128128
public static final String DATETIME_DATA = "data";
129129
public static final String SET_USED = "used";
130+
public static final String MMAP_DATA = "data";
130131

131132
@CompilationFinal(dimensions = 1) public static final String[] values;
132133
static {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PySequenceArrayWrapperMR.java‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,14 @@
6060
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.ListGeneralizationNode;
6161
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.NormalizeIndexNode;
6262
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.StorageToNativeNode;
63+
import com.oracle.graal.python.builtins.objects.ints.PInt;
6364
import com.oracle.graal.python.builtins.objects.list.ListBuiltins;
6465
import com.oracle.graal.python.builtins.objects.list.ListBuiltinsFactory;
6566
import com.oracle.graal.python.builtins.objects.list.PList;
67+
import com.oracle.graal.python.builtins.objects.memoryview.PMemoryView;
68+
import com.oracle.graal.python.builtins.objects.mmap.MMapBuiltins;
69+
import com.oracle.graal.python.builtins.objects.mmap.PMMap;
70+
import com.oracle.graal.python.builtins.objects.slice.PSlice;
6671
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
6772
import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltins;
6873
import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltinsFactory;
@@ -82,6 +87,7 @@
8287
import com.oracle.truffle.api.dsl.ImportStatic;
8388
import com.oracle.truffle.api.dsl.Specialization;
8489
import com.oracle.truffle.api.dsl.TypeSystemReference;
90+
import com.oracle.truffle.api.frame.VirtualFrame;
8591
import com.oracle.truffle.api.interop.ForeignAccess;
8692
import com.oracle.truffle.api.interop.Message;
8793
import com.oracle.truffle.api.interop.MessageResolution;
@@ -111,8 +117,8 @@ public Object access(PySequenceArrayWrapper object) {
111117
abstract static class ReadNode extends Node {
112118
@Child private ReadArrayItemNode readArrayItemNode;
113119

114-
public Object access(PySequenceArrayWrapper object, Object key) {
115-
return getReadArrayItemNode().execute(object.getDelegate(), key);
120+
public Object access(VirtualFrame frame, PySequenceArrayWrapper object, Object key) {
121+
return getReadArrayItemNode().execute(frame, object.getDelegate(), key);
116122
}
117123

118124
private ReadArrayItemNode getReadArrayItemNode() {
@@ -144,20 +150,20 @@ public Object access(PySequenceArrayWrapper object, Object key, Object value) {
144150

145151
@ImportStatic(SpecialMethodNames.class)
146152
@TypeSystemReference(PythonTypes.class)
147-
abstract static class ReadArrayItemNode extends Node {
153+
abstract static class ReadArrayItemNode extends PNodeWithContext {
148154

149155
@Child private ToSulongNode toSulongNode;
150156

151-
public abstract Object execute(Object arrayObject, Object idx);
157+
public abstract Object execute(VirtualFrame frame, Object arrayObject, Object idx);
152158

153159
@Specialization
154-
Object doTuple(PTuple tuple, long idx,
160+
Object doTuple(@SuppressWarnings("unused") VirtualFrame frame, PTuple tuple, long idx,
155161
@Cached("createTupleGetItem()") TupleBuiltins.GetItemNode getItemNode) {
156162
return getToSulongNode().execute(getItemNode.execute(tuple, idx));
157163
}
158164

159165
@Specialization
160-
Object doTuple(PList list, long idx,
166+
Object doTuple(@SuppressWarnings("unused") VirtualFrame frame, PList list, long idx,
161167
@Cached("createListGetItem()") ListBuiltins.GetItemNode getItemNode) {
162168
return getToSulongNode().execute(getItemNode.execute(list, idx));
163169
}
@@ -168,7 +174,7 @@ Object doTuple(PList list, long idx,
168174
* {@code uint64_t} since we do not know how many bytes are requested.
169175
*/
170176
@Specialization
171-
long doBytesI64(PIBytesLike bytesLike, long byteIdx,
177+
long doBytesI64(@SuppressWarnings("unused") VirtualFrame frame, PIBytesLike bytesLike, long byteIdx,
172178
@Cached("createClassProfile()") ValueProfile profile,
173179
@Cached("create()") SequenceStorageNodes.LenNode lenNode,
174180
@Cached("create()") SequenceStorageNodes.GetItemNode getItemNode) {
@@ -179,9 +185,13 @@ long doBytesI64(PIBytesLike bytesLike, long byteIdx,
179185
return 0L;
180186
}
181187
int i = (int) byteIdx;
182-
long result = 0;
183188
SequenceStorage store = profiled.getSequenceStorage();
184-
result |= getItemNode.executeInt(store, i);
189+
return packLong(store, getItemNode, len, i);
190+
}
191+
192+
private static long packLong(SequenceStorage store, SequenceStorageNodes.GetItemNode getItemNode, int len, int i) {
193+
long result = 0;
194+
result |= getItemNode.executeInt(store, i) & 0xFFL;
185195
if (i + 1 < len)
186196
result |= ((long) getItemNode.executeInt(store, i + 1) << 8L) & 0xFF00L;
187197
if (i + 2 < len)
@@ -199,8 +209,28 @@ long doBytesI64(PIBytesLike bytesLike, long byteIdx,
199209
return result;
200210
}
201211

212+
@Specialization
213+
long doMMap(VirtualFrame frame, PMMap list, long idx,
214+
@Cached("create()") MMapBuiltins.GetItemNode getItemNode,
215+
@Cached("create()") SequenceNodes.GetSequenceStorageNode getStorageNode,
216+
@Cached("create()") SequenceStorageNodes.LenNode lenNode,
217+
@Cached("create()") SequenceStorageNodes.GetItemNode getStoreItemNode) {
218+
try {
219+
int i = PInt.intValueExact(idx);
220+
PSlice sl = factory().createSlice(i, i + 8, 1);
221+
Object bytes = getItemNode.executeObject(frame, list, sl);
222+
SequenceStorage storage = getStorageNode.execute(bytes);
223+
int storageLen = lenNode.execute(storage);
224+
return packLong(storage, getStoreItemNode, storageLen, 0);
225+
} catch (ArithmeticException e) {
226+
// TODO raise native exception
227+
CompilerDirectives.transferToInterpreter();
228+
throw new IllegalArgumentException();
229+
}
230+
}
231+
202232
@Specialization(guards = {"!isTuple(object)", "!isList(object)"})
203-
Object doGeneric(Object object, long idx,
233+
Object doGeneric(@SuppressWarnings("unused") VirtualFrame frame, Object object, long idx,
204234
@Cached("create(__GETITEM__)") LookupAndCallBinaryNode getItemNode) {
205235
return getToSulongNode().execute(getItemNode.executeObject(object, idx));
206236
}
@@ -515,7 +545,7 @@ Object doPtrArrayMultiCtx(@SuppressWarnings("unused") PSequence object) {
515545
}
516546

517547
protected static boolean hasByteArrayContent(Object object) {
518-
return object instanceof PBytes || object instanceof PByteArray;
548+
return object instanceof PBytes || object instanceof PByteArray || object instanceof PMemoryView || object instanceof PMMap;
519549
}
520550

521551
public static GetTypeIDNode create() {

0 commit comments

Comments
 (0)