[GR-28428] [GR-28429] Mmap uses PosixSupportLibrary abstraction.

PullRequest: graalpython/1641

Author: steve-s

graalpython/com.oracle.graal.python.annotations/src/com/oracle/graal/python/annotations/ArgumentClinic.java

@@ -66,8 +66,9 @@
 
     /**
      * The string should contain valid Java constant value expression, for example, {@code true}, or
-     * {@code \"some string\"}. You may have to update the annotation processor to include import of
-     * necessary packages or use fully qualified names.
+     * {@code \"some string\"}. Another supported value is an identifier of a static field inside
+     * the annotated class. For anything else, you may have to update the annotation processor to
+     * include import of necessary packages or use fully qualified names.
      */
     String defaultValue() default "";
 
@@ -122,10 +123,21 @@ enum ClinicConversion {
          */
         Int,
         /**
-         * Corresponds to CPython's {@code Py_ssize_t} converter. Supports {@link #defaultValue()},
-         * and {@link #useDefaultForNone()}.
+         * Corresponds to CPython's {@code long} converter ("L"/"l" for old style conversions).
+         * Supports {@link #defaultValue()}, and {@link #useDefaultForNone()}.
+         */
+        Long,
+        /**
+         * Corresponds to CPython's {@code Py_ssize_t} converter, except that it converts the result
+         * into Java integer. Supports {@link #defaultValue()}, and {@link #useDefaultForNone()}.
          */
         Index,
+        /**
+         * Roughly corresponds to CPython's legacy "n" converter: calls the __index__ and then
+         * converts it to Java long. Supports {@link #defaultValue()}, and
+         * {@link #useDefaultForNone()}.
+         */
+        LongIndex,
         /**
          * Corresponds to CPython's {@code slice_index} converter. Supports {@link #defaultValue()},
          * and {@link #useDefaultForNone()}.

graalpython/com.oracle.graal.python.cext/posix/posix.c

@@ -62,6 +62,7 @@
 #include <sys/utsname.h>
 #include <sys/wait.h>
 #include <sys/file.h>
+#include <sys/mman.h>
 #include <unistd.h>
 
 
@@ -505,6 +506,33 @@ int32_t call_system(const char *pathname) {
     return system(pathname);
 }
 
+void *call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset) {
+    void *result = mmap(NULL, length, prot, flags, fd, offset);
+    return result == MAP_FAILED ? NULL : result;
+}
+
+int32_t call_munmap(void* address, int64_t length) {
+    return munmap(address, length);
+}
+
+void call_msync(void* address, int64_t offset, int64_t length) {
+    // TODO: can be generalized to also accept different flags,
+    // but MS_SYNC and such seem to be defined to different values across systems
+    msync(address + offset, length, MS_SYNC);
+}
+
+int8_t read_byte(int8_t *address, int64_t index) {
+    return address[index];
+}
+
+void write_bytes(int8_t *address, int8_t* buffer, int64_t index, int32_t length) {
+    memcpy(address + index, buffer, length);
+}
+
+void read_bytes(int8_t *address, int8_t* buffer, int64_t index, int32_t length) {
+    memcpy(buffer, address + index, length);
+}
+
 int32_t get_errno() {
     return errno;
 }

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/ArgumentClinicModel.java

@@ -44,9 +44,13 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
+import javax.lang.model.element.Element;
+import javax.lang.model.element.ElementKind;
 import javax.lang.model.element.TypeElement;
 
 import com.oracle.graal.python.annotations.ArgumentClinic;
@@ -173,7 +177,15 @@ public static ArgumentClinicData create(ArgumentClinic annotation, TypeElement t
                                 // factory method
                                 continue factoryLoop;
                             }
-                            args[i] = annotation.defaultValue();
+                            String defaultValue = annotation.defaultValue();
+                            Stream<? extends Element> eclosedElements = type.getEnclosedElements().stream();
+                            eclosedElements = eclosedElements.filter(x -> x.getKind() == ElementKind.FIELD && x.getSimpleName().toString().equals(defaultValue));
+                            Optional<? extends Element> typeElement = eclosedElements.findFirst();
+                            if (typeElement.isPresent()) {
+                                args[i] = type.getQualifiedName() + "." + typeElement.get().getSimpleName();
+                            } else {
+                                args[i] = defaultValue;
+                            }
                             break;
                         case UseDefaultForNone:
                             args[i] = String.valueOf(annotation.useDefaultForNone());

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/ConverterFactory.java

@@ -94,9 +94,11 @@ public enum Param {
     private static ConverterFactory[] BuiltinString;
     private static ConverterFactory[] BuiltinStringWithDefaultValue;
     private static ConverterFactory[] BuiltinInt;
+    private static ConverterFactory[] BuiltinLong;
     private static ConverterFactory[] BuiltinCodePoint;
     private static ConverterFactory[] BuiltinBuffer;
     private static ConverterFactory[] BuiltinIndex;
+    private static ConverterFactory[] BuiltinLongIndex;
     private static ConverterFactory[] BuiltinSliceIndex;
     private static ConverterFactory[] BuiltinNone;
     private static ConverterFactory[] BuiltinDouble;
@@ -127,6 +129,8 @@ public static ConverterFactory[] getBuiltin(ArgumentClinic annotation) {
                 return annotation.defaultValue().isEmpty() ? BuiltinString : BuiltinStringWithDefaultValue;
             case Int:
                 return BuiltinInt;
+            case Long:
+                return BuiltinLong;
             case Double:
                 return BuiltinDouble;
             case CodePoint:
@@ -135,6 +139,8 @@ public static ConverterFactory[] getBuiltin(ArgumentClinic annotation) {
                 return BuiltinBuffer;
             case Index:
                 return BuiltinIndex;
+            case LongIndex:
+                return BuiltinLongIndex;
             case SliceIndex:
                 return BuiltinSliceIndex;
             case None:
@@ -209,10 +215,12 @@ public static void initBuiltins(Elements elementUtils) throws ProcessingError {
         BuiltinString = forBuiltin(elementUtils, "JavaStringConverterNode");
         BuiltinStringWithDefaultValue = forBuiltin(elementUtils, "JavaStringConverterWithDefaultValueNode");
         BuiltinInt = forBuiltin(elementUtils, "JavaIntConversionNode");
+        BuiltinLong = forBuiltin(elementUtils, "JavaLongConversionNode");
         BuiltinDouble = forBuiltin(elementUtils, "JavaDoubleConversionNode");
         BuiltinCodePoint = forBuiltin(elementUtils, "CodePointConversionNode");
         BuiltinBuffer = forBuiltin(elementUtils, "BufferConversionNode");
         BuiltinIndex = forBuiltin(elementUtils, "IndexConversionNode");
+        BuiltinLongIndex = forBuiltin(elementUtils, "LongIndexConverterNode");
         BuiltinSliceIndex = forBuiltin(elementUtils, "SliceIndexConversionNode");
         BuiltinNone = forBuiltin(elementUtils, "DefaultValueNode");
     }

graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py

@@ -2,15 +2,10 @@
 # Copyright (C) 1996-2017 Python Software Foundation
 #
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
-from test.support import (TESTFN, run_unittest, import_module, unlink,
-                          requires, _2G, _4G, gc_collect, cpython_only)
-import unittest
 import os
 import re
-import itertools
-import socket
-import sys
-import weakref
+import unittest
+from test.support import (TESTFN, run_unittest, import_module)
 
 # Skip test if we can't import mmap.
 mmap = import_module('mmap')
@@ -179,6 +174,53 @@ def test_context_manager_exception(self):
         self.assertTrue(m.closed, "context manager failed")
 
 
+FIND_BUFFER_SIZE = 1024 # keep in sync with FindNode#BUFFER_SIZE
+def test_find():
+    cases = [
+        # (size, needle_pos)
+        (FIND_BUFFER_SIZE * 3 + 1, FIND_BUFFER_SIZE * 3 - 2),
+        (FIND_BUFFER_SIZE * 3 + 3, FIND_BUFFER_SIZE * 3),
+        (FIND_BUFFER_SIZE * 2, FIND_BUFFER_SIZE),
+        (FIND_BUFFER_SIZE * 2, FIND_BUFFER_SIZE - 1),
+        (11, 1),
+    ]
+    for (size, needle_pos) in cases:
+        m = mmap.mmap(-1, size)
+        m[needle_pos] = b'a'[0]
+        m[needle_pos + 1] = b'b'[0]
+        m[needle_pos + 2] = b'c'[0]
+        assert m.find(b'abc') == needle_pos
+        assert m.find(b'abc', 0, needle_pos) == -1
+        assert m.find(b'abc', 0, needle_pos+2) == -1
+        assert m.find(b'abc', 0, needle_pos+3) == needle_pos
+        assert m.find(b'abc', needle_pos) == needle_pos
+        assert m.find(b'abc', needle_pos+1) == -1
+        assert m.find(b'abc', needle_pos-1) == needle_pos
+        m.close()
+
+
+def test_getitem():
+    m = mmap.mmap(-1, 12)
+    for i in range(0, 12):
+        m[i] = i
+    assert m[slice(-10, 100)] == b'\x02\x03\x04\x05\x06\x07\x08\t\n\x0b'
+
+
+def test_readline():
+    m = mmap.mmap(-1, 9)
+    for i in range(0, 9):
+        m[i] = i
+    m[4] = b'\n'[0]
+    assert m.readline() == b'\x00\x01\x02\x03\n'
+    assert m.readline() == b'\x05\x06\x07\x08'
+
+    m = mmap.mmap(-1, 1024 + 3)
+    m[1024] = b'\n'[0]
+    m[1025] = b'a'[0]
+    m[1026] = b'b'[0]
+    assert m.readline() == (b'\x00' * 1024) + b'\n'
+    assert m.readline() == b'ab'
+
 
 def test_main():
     #run_unittest(MmapTests, LargeMmapTests)

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_mmap.txt

@@ -1,20 +1,28 @@
 *graalpython.lib-python.3.test.test_mmap.LargeMmapTests.test_around_2GB
 *graalpython.lib-python.3.test.test_mmap.LargeMmapTests.test_around_4GB
+*graalpython.lib-python.3.test.test_mmap.LargeMmapTests.test_large_offset
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_anonymous
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_bad_file_desc
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_basic
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_concat_repeat_exception
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_context_manager
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_context_manager_exception
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_crasher_on_windows
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_double_close
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_empty_file
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_entire_file
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_find_end
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_invalid_descriptor
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_length_0_large_offset
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_length_0_offset
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_madvise
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_prot_readonly
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_read_all
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_read_invalid_arg
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_resize_past_pos
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_sizeof
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_subclass
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_tagname
+*graalpython.lib-python.3.test.test_mmap.MmapTests.test_tougher_find
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_weakref
 *graalpython.lib-python.3.test.test_mmap.MmapTests.test_write_returning_the_number_of_bytes_written