[GR-10357] [GR-11082] Introduce native sequence storage.

PullRequest: graalpython/158

Author: fangerer

graalpython/com.oracle.graal.python.cext/include/polyglot.h

@@ -539,6 +539,9 @@ polyglot_typeid polyglot_array_typeid(polyglot_typeid base, uint64_t len);
 /**
  * Converts a polyglot value to a dynamic struct or array pointer.
  *
+ * The typeid passed to this function must refer to a struct or array type.
+ * Passing a primitive typeid is not valid.
+ *
  * @see polyglot_as_MyStruct
  * @see polyglot_as_MyStruct_array
  *
@@ -551,6 +554,9 @@ void *polyglot_as_typed(void *value, polyglot_typeid typeId);
 /**
  * Create a polyglot value from a native pointer to a struct or array.
  *
+ * The typeid passed to this function must refer to a struct or array type.
+ * Passing a primitive typeid is not valid.
+ *
  * @see polyglot_from_MyStruct
  * @see polyglot_from_MyStruct_array
  *
@@ -564,31 +570,55 @@ void *polyglot_from_typed(void *ptr, polyglot_typeid typeId);
  * Internal function. Do not use directly.
  *
  * @see POLYGLOT_DECLARE_STRUCT
+ * @see POLYGLOT_DECLARE_TYPE
  */
 polyglot_typeid __polyglot_as_typeid(void *ptr);
 
-#define POLYGLOT_DECLARE_GENERIC_TYPE(typedecl, typename)                                                                                            \
-  __attribute__((always_inline)) static void *polyglot_##typename##_typeid() {                                                                       \
+/**
+ * Internal macro. Do not use directly.
+ *
+ * @see POLYGLOT_DECLARE_STRUCT
+ * @see POLYGLOT_DECLARE_TYPE
+ */
+#define __POLYGLOT_DECLARE_GENERIC_ARRAY(typedecl, typename)                                                                                         \
+  __attribute__((always_inline)) static inline polyglot_typeid polyglot_##typename##_typeid() {                                                      \
     static typedecl __polyglot_typeid_##typename[0];                                                                                                 \
     return __polyglot_as_typeid(__polyglot_typeid_##typename);                                                                                       \
   }                                                                                                                                                  \
                                                                                                                                                      \
-  __attribute__((always_inline)) static inline typedecl *polyglot_as_##typename(void *p) {                                                           \
-    void *ret = polyglot_as_typed(p, polyglot_##typename##_typeid());                                                                                \
-    return (typedecl *)ret;                                                                                                                          \
-  }                                                                                                                                                  \
-                                                                                                                                                     \
   __attribute__((always_inline)) static inline typedecl *polyglot_as_##typename##_array(void *p) {                                                   \
     void *ret = polyglot_as_typed(p, polyglot_array_typeid(polyglot_##typename##_typeid(), 0));                                                      \
     return (typedecl *)ret;                                                                                                                          \
   }                                                                                                                                                  \
                                                                                                                                                      \
-  __attribute__((always_inline)) static void *polyglot_from_##typename(typedecl * s) {                                                               \
-    return polyglot_from_typed(s, polyglot_##typename##_typeid());                                                                                   \
+  __attribute__((always_inline)) static inline void *polyglot_from_##typename##_array(typedecl *arr, uint64_t len) {                                 \
+    return polyglot_from_typed(arr, polyglot_array_typeid(polyglot_##typename##_typeid(), len));                                                     \
+  }
+
+__POLYGLOT_DECLARE_GENERIC_ARRAY(bool, boolean)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(int8_t, i8)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(int16_t, i16)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(int32_t, i32)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(int64_t, i64)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(float, float)
+__POLYGLOT_DECLARE_GENERIC_ARRAY(double, double)
+
+/**
+ * Internal macro. Do not use directly.
+ *
+ * @see POLYGLOT_DECLARE_STRUCT
+ * @see POLYGLOT_DECLARE_TYPE
+ */
+#define __POLYGLOT_DECLARE_GENERIC_TYPE(typedecl, typename)                                                                                          \
+  __POLYGLOT_DECLARE_GENERIC_ARRAY(typedecl, typename)                                                                                               \
+                                                                                                                                                     \
+  __attribute__((always_inline)) static inline typedecl *polyglot_as_##typename(void *p) {                                                           \
+    void *ret = polyglot_as_typed(p, polyglot_##typename##_typeid());                                                                                \
+    return (typedecl *)ret;                                                                                                                          \
   }                                                                                                                                                  \
                                                                                                                                                      \
-  __attribute__((always_inline)) static void *polyglot_from_##typename##_array(typedecl *arr, uint64_t len) {                                        \
-    return polyglot_from_typed(arr, polyglot_array_typeid(polyglot_##typename##_typeid(), len));                                                     \
+  __attribute__((always_inline)) static inline void *polyglot_from_##typename(typedecl * s) {                                                        \
+    return polyglot_from_typed(s, polyglot_##typename##_typeid());                                                                                   \
   }
 
 /**
@@ -614,7 +644,7 @@ polyglot_typeid __polyglot_as_typeid(void *ptr);
  * void *polyglot_from_MyStruct_array(struct MyStruct *arr, uint64_t len);
  * \endcode
  */
-#define POLYGLOT_DECLARE_STRUCT(type) POLYGLOT_DECLARE_GENERIC_TYPE(struct type, type)
+#define POLYGLOT_DECLARE_STRUCT(type) __POLYGLOT_DECLARE_GENERIC_TYPE(struct type, type)
 
 /**
  * Declare polyglot conversion functions for a user-defined anonymous struct type.
@@ -639,9 +669,194 @@ polyglot_typeid __polyglot_as_typeid(void *ptr);
  * void *polyglot_from_MyType_array(MyType *arr, uint64_t len);
  * \endcode
  */
-#define POLYGLOT_DECLARE_TYPE(type) POLYGLOT_DECLARE_GENERIC_TYPE(type, type)
+#define POLYGLOT_DECLARE_TYPE(type) __POLYGLOT_DECLARE_GENERIC_TYPE(type, type)
 
 #ifdef DOXYGEN // documentation only
+
+/**
+ * Get a polyglot typeid for the primitive bool type.
+ */
+static void *polyglot_boolean_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive int8_t type.
+ */
+static void *polyglot_int8_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive int16_t type.
+ */
+static void *polyglot_int16_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive int32_t type.
+ */
+static void *polyglot_int32_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive int64_t type.
+ */
+static void *polyglot_int64_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive float type.
+ */
+static void *polyglot_float_typeid();
+
+/**
+ * Get a polyglot typeid for the primitive double type.
+ */
+static void *polyglot_double_typeid();
+
+/**
+ * Converts a polyglot value to an integer array.
+ *
+ * For example, this code snippet:
+ *
+ * \code
+ * int32_t *arr = polyglot_as_i32_array(arrayValue);
+ * for (int i = 0; i < polyglot_get_array_size(arr); i++) {
+ *   sum += arr[i];
+ * }
+ * \endcode
+ *
+ * is equivalent to
+ *
+ * \code
+ * for (int i = 0; i < polyglot_get_array_size(arrayValue); i++) {
+ *   void *elem = polyglot_get_array_element(arrayValue, i);
+ *   sum += polyglot_as_i32(elem);
+ * }
+ * \endcode
+ *
+ * The returned pointer is a view of the original value, and does not need to be
+ * freed separately.
+ *
+ * \param value a polyglot array value
+ * \return array view of the polyglot value
+ */
+static int32_t *polyglot_as_i32_array(void *value);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive integer array.
+ * The resulting polyglot value can be passed to other languages and accessed
+ * from there.
+ *
+ * For example, given this code snippet:
+ *
+ * \code
+ * int32_t *s = calloc(len, sizeof(*s));
+ * s[idx] = ...;
+ * void *value = polyglot_from_i32_array(s, len);
+ * someJSFunction(value);
+ * \endcode
+ *
+ * The following JavaScript code can access the native pointer as if it were a
+ * JavaScript array:
+ *
+ * \code
+ * function someJSFunction(value) {
+ *   ...
+ *   result = value[idx];
+ *   ...
+ * }
+ * \endcode
+ *
+ * The array access will be bounds checked with the given array length.
+ *
+ * The returned pointer will be semantically equal to the original pointer. In
+ * particular, if one of them is freed, the other will become invalid.
+ *
+ * \param arr a pointer to a primitive integer array
+ * \param len the length of the array
+ * \return a polyglot value representing arr
+ */
+static void *polyglot_from_i32_array(int32_t *arr, uint64_t len);
+
+/**
+ * Converts a polyglot value to a bool array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static bool *polyglot_as_boolean_array(void *value);
+
+/**
+ * Converts a polyglot value to an integer array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static int8_t *polyglot_as_i8_array(void *value);
+
+/**
+ * Converts a polyglot value to an integer array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static int16_t *polyglot_as_i16_array(void *value);
+
+/**
+ * Converts a polyglot value to an integer array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static int64_t *polyglot_as_i64_array(void *value);
+
+/**
+ * Converts a polyglot value to a float array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static float *polyglot_as_float_array(void *value);
+
+/**
+ * Converts a polyglot value to a double array.
+ *
+ * \see polyglot_as_i32_array
+ */
+static double *polyglot_as_double_array(void *value);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive bool array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_boolean_array(bool *arr, uint64_t len);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive integer array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_i8_array(int8_t *arr, uint64_t len);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive integer array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_i16_array(int16_t *arr, uint64_t len);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive integer array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_i64_array(int64_t *arr, uint64_t len);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive float array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_float_array(float *arr, uint64_t len);
+
+/**
+ * Create a polyglot value from a native pointer to a primitive double array.
+ *
+ * \see polyglot_from_i32_array
+ */
+static void *polyglot_from_double_array(double *arr, uint64_t len);
+
 struct MyStruct;
 
 /**

graalpython/com.oracle.graal.python.cext/src/bytesobject.c

@@ -52,13 +52,13 @@ PyObject* PyBytes_FromStringAndSize(const char* str, Py_ssize_t sz) {
     setlocale(LC_ALL, NULL);
     const char* encoding = nl_langinfo(CODESET);
     void *jstr = str != NULL ? polyglot_from_string_n(str, sz, SRC_CS) : to_java(NULL);
-	return UPCALL_CEXT_O("PyBytes_FromStringAndSize", jstr, polyglot_from_string(encoding, SRC_CS));
+	return UPCALL_CEXT_O("PyBytes_FromStringAndSize", jstr, sz, polyglot_from_string(encoding, SRC_CS));
 }
 
 PyObject * PyBytes_FromString(const char *str) {
 	setlocale(LC_ALL, NULL);
 	const char* encoding = nl_langinfo(CODESET);
-	return UPCALL_CEXT_O("PyBytes_FromStringAndSize", polyglot_from_string(str, SRC_CS), polyglot_from_string(encoding, SRC_CS));
+	return UPCALL_CEXT_O("PyBytes_FromStringAndSize", polyglot_from_string(str, SRC_CS), 0, polyglot_from_string(encoding, SRC_CS));
 }
 
 char* PyBytes_AsString(PyObject *obj) {

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -119,11 +119,8 @@ initialize_type(_PyNotImplemented_Type, NotImplementedType, _object);
 initialize_type(PyDictProxy_Type, mappingproxy, _object);
 initialize_type(PyEllipsis_Type, ellipsis, _object);
 
-typedef uint8_t ByteArray[0];
-POLYGLOT_DECLARE_TYPE(ByteArray);
-
-typedef PyObject* PtrArray[0];
-POLYGLOT_DECLARE_TYPE(PtrArray);
+typedef PyObject* PyObjectPtr;
+POLYGLOT_DECLARE_TYPE(PyObjectPtr);
 
 static void initialize_globals() {
     // None
@@ -210,12 +207,12 @@ void* get_ob_type(PyObject* obj) {
 
 /** to be used from Java code only; returns the type ID for a byte array */
 polyglot_typeid get_byte_array_typeid(uint64_t len) {
-    return polyglot_ByteArray_typeid();
+    return polyglot_array_typeid(polyglot_i8_typeid(), len);
 }
 
 /** to be used from Java code only; returns the type ID for a 'PyObject*' array */
 polyglot_typeid get_ptr_array_typeid(uint64_t len) {
-    return polyglot_PtrArray_typeid();
+    return polyglot_array_typeid(polyglot_PyObjectPtr_typeid(), len);
 }
 
 typedef struct PyObjectHandle {
@@ -264,16 +261,23 @@ const char* PyTruffle_StringToCstr(void* o, int32_t strLen) {
     return str;
 }
 
-const char* PyTruffle_ByteArrayToNative(const void* jbyteArray, int len) {
-    int i;
-    int size = len != 0 ? len : 1;
-    char* barr = (const char*) malloc(size * sizeof(char));
-    barr[0] = '\0';
-    for(i=0; i < len; i++) {
-        barr[i] = (char) polyglot_get_array_element(jbyteArray, i);
-    }
-    return (const char*) barr;
-}
+#define PRIMITIVE_ARRAY_TO_NATIVE(__jtype__, __ctype__, __polyglot_type__, __element_cast__) \
+    void* PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
+        int64_t i; \
+        int64_t size = len + 1; \
+        __ctype__* carr = (__ctype__*) malloc(size * sizeof(__ctype__)); \
+        carr[len] = (__ctype__)0; \
+        for (i=0; i < len; i++) { \
+            carr[i] = __element_cast__(polyglot_get_array_element(jarray, i)); \
+        } \
+        return polyglot_from_##__polyglot_type__##_array(carr, len); \
+    } \
+
+PRIMITIVE_ARRAY_TO_NATIVE(Byte, int8_t, i8, polyglot_as_i8);
+PRIMITIVE_ARRAY_TO_NATIVE(Int, int32_t, i32, polyglot_as_i32);
+PRIMITIVE_ARRAY_TO_NATIVE(Long, int64_t, i64, polyglot_as_i64);
+PRIMITIVE_ARRAY_TO_NATIVE(Double, double, double, polyglot_as_double);
+PRIMITIVE_ARRAY_TO_NATIVE(Object, PyObjectPtr, PyObjectPtr, (PyObjectPtr));
 
 #define ReadMember(object, offset, T) ((T*)(((char*)object) + PyLong_AsSsize_t(offset)))[0]