Obtain latest master

Author: olyagpl

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "ea08890c99fa458eedbe654f6d413cfbd995f8a8" }
+{ "overlay": "a821a41d4b56097241ba670508db188ead7e8b58" }

docs/contributor/IMPLEMENTATION_DETAILS.md

@@ -248,3 +248,28 @@ we are always running single threaded or always own the GIL already when we get
 to a specific message we never emit a boundary call to lock and unlock the
 GIL. This implies that we may deopt in some places if, after compiling some
 code, we later start a second thread.
+
+## Threading
+
+As explained above, we use a GIL to prevent parallel execution of Python code. A
+timer is running the interrupts threads periodically to relinquish the GIL and
+give other threads a chance to run. This preemption is prohibited in most C
+extension code, however, since the assumption in C extensions written for
+CPython is that the GIL will not be relinquished while executing the C code and
+many C extensions are not written to reentrant.
+
+So, commonly at any given time there is only one Graal Python thread executing
+and all the other threads are waiting to acquire the GIL. If the Python
+interpreter shuts down, there are two sets of threads we need to deal with:
+"well-behaved" threads created using the `threading` module are interrupted and
+joined using the `threading` module's `shutdown` function. Daemon threads or
+threads created using the internal Python `_thread` module cannot be joined in
+this way. For those threads, we invalidate their `PythonThreadState` (a
+thread-local data structure) and use the Java `Thread#interrupt` method to
+interrupt their waiting on the GIL. This exception is handled by checking if the
+thread state has been invalidated and if so, just exit the thread gracefully.
+
+For embedders, it may be important to be able to interrupt Python threads by
+other means. We use the TruffleSafepoint mechanism to mark our threads waiting
+to acquire the GIL as blocked for the purpose of safepoints. The Truffle
+safepoint action mechanism can thus be used to kill threads waiting on the GIL.

docs/user/Packages.md

@@ -41,5 +41,5 @@ Other extensions might currently work, but they are not actively tested.
 Note that to try extensions on GraalVM's Python runtime, you have to download, build, and install them manually for now.
 
 ### Using `pip`
-The `pip` package installer is available and working in a `venv`, but there is no support for SSL yet.
-This means you can install packages from PyPI if you use an HTTP mirror, and you can install local packages.
+
+The `pip` package installer is available and working when using a `venv`.

graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/ParserBenchRunner.java

@@ -46,7 +46,7 @@
 import com.oracle.graal.python.parser.PythonParserImpl;
 import com.oracle.graal.python.parser.PythonSSTNodeFactory;
 import com.oracle.graal.python.runtime.PythonContext;
-import com.oracle.graal.python.runtime.PythonCore;
+import com.oracle.graal.python.builtins.Python3Core;
 import com.oracle.graal.python.runtime.PythonParser;
 import com.oracle.truffle.api.TruffleFile;
 import com.oracle.truffle.api.source.Source;
@@ -98,7 +98,7 @@ public class ParserBenchRunner {
 
     protected final PythonContext pyContext;
     protected final PythonParserImpl parser;
-    protected final PythonCore core;
+    protected final Python3Core core;
     private List<Source> sources;
 
     public ParserBenchRunner() {

graalpython/com.oracle.graal.python.cext/hpy/hpy.c

@@ -48,7 +48,6 @@
 #define UNWRAP(_h) ((_h)._i)
 #define WRAP(_ptr) ((HPy){(_ptr)})
 
-
 typedef HPyDef* HPyDefPtr;
 
 POLYGLOT_DECLARE_TYPE(HPy);
@@ -91,6 +90,10 @@ void* graal_hpy_calloc(size_t count, size_t eltsize) {
 	return calloc(count, eltsize);
 }
 
+void graal_hpy_free(void *ptr) {
+	free(ptr);
+}
+
 void* graal_hpy_from_HPy_array(void *arr, uint64_t len) {
 	return polyglot_from_HPy_array(arr, len);
 }
@@ -571,3 +574,9 @@ int graal_hpy_bulk_free(void* ptrArray[], int64_t len) {
 	}
     return 0;
 }
+
+POLYGLOT_DECLARE_TYPE(HPy_buffer);
+HPy_buffer *graal_hpy_allocate_buffer() {
+	return polyglot_from_HPy_buffer((HPy_buffer *) malloc(sizeof(HPy_buffer)));
+}
+

graalpython/com.oracle.graal.python.cext/include/common/autogen_hpyfunc_declare.h

@@ -63,6 +63,8 @@
 #define _HPyFunc_DECLARE_HPyFunc_GETTER(SYM) static HPy SYM(HPyContext ctx, HPy, void *)
 #define _HPyFunc_DECLARE_HPyFunc_SETTER(SYM) static int SYM(HPyContext ctx, HPy, HPy, void *)
 #define _HPyFunc_DECLARE_HPyFunc_OBJOBJPROC(SYM) static int SYM(HPyContext ctx, HPy, HPy)
+#define _HPyFunc_DECLARE_HPyFunc_GETBUFFERPROC(SYM) static int SYM(HPyContext, HPy, HPy_buffer *, int)
+#define _HPyFunc_DECLARE_HPyFunc_RELEASEBUFFERPROC(SYM) static void SYM(HPyContext, HPy, HPy_buffer *)
 #define _HPyFunc_DECLARE_HPyFunc_DESTROYFUNC(SYM) static void SYM(void *)
 
 typedef HPy (*HPyFunc_noargs)(HPyContext ctx, HPy self);
@@ -95,4 +97,6 @@ typedef int (*HPyFunc_initproc)(HPyContext ctx, HPy self, HPy *args, HPy_ssize_t
 typedef HPy (*HPyFunc_getter)(HPyContext ctx, HPy, void *);
 typedef int (*HPyFunc_setter)(HPyContext ctx, HPy, HPy, void *);
 typedef int (*HPyFunc_objobjproc)(HPyContext ctx, HPy, HPy);
+typedef int (*HPyFunc_getbufferproc)(HPyContext, HPy, HPy_buffer *, int);
+typedef void (*HPyFunc_releasebufferproc)(HPyContext, HPy, HPy_buffer *);
 typedef void (*HPyFunc_destroyfunc)(void *);

graalpython/com.oracle.graal.python.cext/include/common/autogen_hpyslot.h

@@ -34,6 +34,8 @@
 */
 
 typedef enum {
+    HPy_bf_getbuffer = 1,
+    HPy_bf_releasebuffer = 2,
     HPy_nb_absolute = 6,
     HPy_nb_add = 7,
     HPy_nb_and = 8,
@@ -84,6 +86,8 @@ typedef enum {
     HPy_tp_destroy = 1000,
 } HPySlot_Slot;
 
+#define _HPySlot_SIG__HPy_bf_getbuffer HPyFunc_GETBUFFERPROC
+#define _HPySlot_SIG__HPy_bf_releasebuffer HPyFunc_RELEASEBUFFERPROC
 #define _HPySlot_SIG__HPy_nb_absolute HPyFunc_UNARYFUNC
 #define _HPySlot_SIG__HPy_nb_add HPyFunc_BINARYFUNC
 #define _HPySlot_SIG__HPy_nb_and HPyFunc_BINARYFUNC

graalpython/com.oracle.graal.python.cext/include/common/autogen_impl.h

@@ -113,6 +113,11 @@ HPyAPI_STORAGE double _HPy_IMPL_NAME(Float_AsDouble)(HPyContext ctx, HPy h)
     return PyFloat_AsDouble(_h2py(h));
 }
 
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Bool_FromLong)(HPyContext ctx, long v)
+{
+    return _py2h(PyBool_FromLong(v));
+}
+
 HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME_NOPREFIX(Length)(HPyContext ctx, HPy h)
 {
     return PyObject_Length(_h2py(h));
@@ -363,6 +368,11 @@ HPyAPI_STORAGE int _HPy_IMPL_NAME_NOPREFIX(SetItem)(HPyContext ctx, HPy obj, HPy
     return PyObject_SetItem(_h2py(obj), _h2py(key), _h2py(value));
 }
 
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME_NOPREFIX(Type)(HPyContext ctx, HPy obj)
+{
+    return _py2h(PyObject_Type(_h2py(obj)));
+}
+
 HPyAPI_STORAGE HPy _HPy_IMPL_NAME_NOPREFIX(Repr)(HPyContext ctx, HPy obj)
 {
     return _py2h(PyObject_Repr(_h2py(obj)));

graalpython/com.oracle.graal.python.cext/include/common/cpy_types.h

@@ -38,6 +38,7 @@ typedef struct PyMethodDef cpy_PyMethodDef;
 typedef PyType_Slot cpy_PyTypeSlot;
 typedef PyGetSetDef cpy_PyGetSetDef;
 typedef PyMemberDef cpy_PyMemberDef;
+typedef struct bufferinfo cpy_Py_buffer;
 
 
 #endif /* HPY_UNIVERSAL_CPY_TYPES_H */

graalpython/com.oracle.graal.python.cext/include/common/hpyfunc.h

@@ -33,6 +33,8 @@ typedef enum {
 
     HPyFunc_DESTROYFUNC,
 
+    HPyFunc_GETBUFFERPROC,
+    HPyFunc_RELEASEBUFFERPROC,
     HPyFunc_UNARYFUNC,
     HPyFunc_BINARYFUNC,
     HPyFunc_TERNARYFUNC,
@@ -94,6 +96,19 @@ typedef enum {
     enum { SYM##_sig = SIG };              \
     _HPyFunc_TRAMPOLINE_##SIG(SYM, IMPL)
 
+typedef struct {
+    void *buf;
+    HPy obj;        /* owned reference */
+    HPy_ssize_t len;
+    HPy_ssize_t itemsize;
+    int readonly;
+    int ndim;
+    char *format;
+    HPy_ssize_t *shape;
+    HPy_ssize_t *strides;
+    HPy_ssize_t *suboffsets;
+    void *internal;
+} HPy_buffer;
 
 #include "autogen_hpyfunc_declare.h"