Skip to content

Commit 484ee94

Browse files
otethalotethal
committed
[GR-28423] Use the NFI posix backend by default in the launcher
PullRequest: graalpython/1649
2 parents 629b612 + 7505ecc commit 484ee94

File tree

27 files changed

+976
-117
lines changed

27 files changed

+976
-117
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
66
## Version 21.1.0
77

88
* Added subclassing of Java classes in JVM mode
9+
* Use native posix functions in the GraalPython Launcher
910

1011
## Version 21.0.0
1112

‎graalpython/com.oracle.graal.python.cext/posix/posix.c‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,28 @@ int32_t set_inheritable(int32_t fd, int32_t inheritable) {
107107
int32_t call_openat(int32_t dirFd, const char *pathname, int32_t flags, int32_t mode) {
108108
int fixedFlags = flags;
109109
// TODO remove this once we properly synchronize constants between Java and C
110-
if (flags & 64) {
111-
fixedFlags &= ~64;
110+
111+
#define WRONG_O_CLOEXEC 524288
112+
#define WRONG_O_APPEND 1024
113+
#define WRONG_O_TRUNC 512
114+
#define WRONG_O_EXCL 128
115+
#define WRONG_O_CREAT 64
116+
117+
fixedFlags &= ~(WRONG_O_CLOEXEC | WRONG_O_APPEND | WRONG_O_TRUNC | WRONG_O_EXCL | WRONG_O_CREAT);
118+
119+
if (flags & WRONG_O_CREAT) {
112120
fixedFlags |= O_CREAT;
113121
}
114-
if (flags & 524288) {
115-
fixedFlags &= ~524288;
122+
if (flags & WRONG_O_EXCL) {
123+
fixedFlags |= O_EXCL;
124+
}
125+
if (flags & WRONG_O_TRUNC) {
126+
fixedFlags |= O_TRUNC;
127+
}
128+
if (flags & WRONG_O_APPEND) {
129+
fixedFlags |= O_APPEND;
130+
}
131+
if (flags & WRONG_O_CLOEXEC) {
116132
fixedFlags |= O_CLOEXEC;
117133
}
118134
return openat(fixDirFd(dirFd), pathname, fixedFlags, mode);
@@ -199,7 +215,7 @@ int32_t call_select(int32_t nfds, int32_t* readfds, int32_t readfdsLen,
199215

200216
struct timeval timeout = {timeoutSec, timeoutUsec};
201217

202-
int result = select(nfds, &readfdsSet, &writefdsSet, &errfdsSet, timeoutSec > 0 ? &timeout : NULL);
218+
int result = select(nfds, &readfdsSet, &writefdsSet, &errfdsSet, timeoutSec >= 0 ? &timeout : NULL);
203219

204220
// fill in the output parameter
205221
fill_select_result(readfds, readfdsLen, &readfdsSet, selected, 0);
@@ -322,7 +338,7 @@ int32_t call_symlinkat(const char *target, int32_t dirFd, const char *linkpath)
322338
}
323339

324340
int32_t call_mkdirat(int32_t dirFd, const char *pathname, int32_t mode) {
325-
return mkdirat(dirFd, pathname, mode);
341+
return mkdirat(fixDirFd(dirFd), pathname, mode);
326342
}
327343

328344
int32_t call_getcwd(char *buf, uint64_t size) {
@@ -403,7 +419,7 @@ int32_t call_futimens(int32_t fd, int64_t *timespec) {
403419
}
404420

405421
int32_t call_renameat(int32_t oldDirFd, const char *oldPath, int32_t newDirFd, const char *newPath) {
406-
return renameat(oldDirFd, oldPath, newDirFd, newPath);
422+
return renameat(fixDirFd(oldDirFd), oldPath, fixDirFd(newDirFd), newPath);
407423
}
408424

409425
int32_t call_faccessat(int32_t dirFd, const char *path, int32_t mode, int32_t effectiveIds, int32_t followSymlinks) {
@@ -426,7 +442,7 @@ int32_t call_fchmod(int32_t fd, int32_t mode) {
426442
}
427443

428444
int64_t call_readlinkat(int32_t dirFd, const char *path, char *buf, uint64_t size) {
429-
return readlinkat(dirFd, path, buf, size);
445+
return readlinkat(fixDirFd(dirFd), path, buf, size);
430446
}
431447

432448
int64_t call_waitpid(int64_t pid, int32_t *status, int32_t options) {
@@ -506,31 +522,19 @@ int32_t call_system(const char *pathname) {
506522
return system(pathname);
507523
}
508524

509-
void *call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset) {
525+
int64_t call_mmap(int64_t length, int32_t prot, int32_t flags, int32_t fd, int64_t offset) {
510526
void *result = mmap(NULL, length, prot, flags, fd, offset);
511-
return result == MAP_FAILED ? NULL : result;
527+
return result == MAP_FAILED ? 0 : (int64_t) result;
512528
}
513529

514-
int32_t call_munmap(void* address, int64_t length) {
515-
return munmap(address, length);
530+
int32_t call_munmap(int64_t address, int64_t length) {
531+
return munmap((void *) address, length);
516532
}
517533

518-
void call_msync(void* address, int64_t offset, int64_t length) {
534+
void call_msync(int64_t address, int64_t offset, int64_t length) {
519535
// TODO: can be generalized to also accept different flags,
520536
// but MS_SYNC and such seem to be defined to different values across systems
521-
msync(address + offset, length, MS_SYNC);
522-
}
523-
524-
int8_t read_byte(int8_t *address, int64_t index) {
525-
return address[index];
526-
}
527-
528-
void write_bytes(int8_t *address, int8_t* buffer, int64_t index, int32_t length) {
529-
memcpy(address + index, buffer, length);
530-
}
531-
532-
void read_bytes(int8_t *address, int8_t* buffer, int64_t index, int32_t length) {
533-
memcpy(buffer, address + index, length);
537+
msync(((int8_t *) address) + offset, length, MS_SYNC);
534538
}
535539

536540
int32_t get_errno() {

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
116116
List<String> arguments = new ArrayList<>(inputArgs);
117117
List<String> subprocessArgs = new ArrayList<>();
118118
programArgs = new ArrayList<>();
119+
boolean posixBackendSpecified = false;
119120
for (int i = 0; i < arguments.size(); i++) {
120121
String arg = arguments.get(i);
121122
switch (arg) {
@@ -285,12 +286,15 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
285286
}
286287
} else {
287288
if (arg.startsWith("--llvm.") ||
288-
arg.startsWith("--python.CoreHome") ||
289-
arg.startsWith("--python.StdLibHome") ||
290-
arg.startsWith("--python.CAPI") ||
291-
arg.startsWith("--PosixModuleBackend")) {
289+
matchesPythonOption(arg, "CoreHome") ||
290+
matchesPythonOption(arg, "StdLibHome") ||
291+
matchesPythonOption(arg, "CAPI") ||
292+
matchesPythonOption(arg, "PosixModuleBackend")) {
292293
addRelaunchArg(arg);
293294
}
295+
if (matchesPythonOption(arg, "PosixModuleBackend")) {
296+
posixBackendSpecified = true;
297+
}
294298
// possibly a polyglot argument
295299
unrecognized.add(arg);
296300
}
@@ -313,7 +317,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
313317
if (!subprocessArgs.isEmpty()) {
314318
subExec(inputArgs, subprocessArgs);
315319
}
316-
320+
if (!posixBackendSpecified) {
321+
polyglotOptions.put("python.PosixModuleBackend", "native");
322+
}
317323
return unrecognized;
318324
}
319325

@@ -524,7 +530,7 @@ protected void launch(Builder contextBuilder) {
524530
if (warnOptions == null || warnOptions.isEmpty()) {
525531
warnOptions = "";
526532
}
527-
String executable = getContextOptionIfSetViaCommandLine("python.Executable");
533+
String executable = getContextOptionIfSetViaCommandLine("Executable");
528534
if (executable != null) {
529535
contextBuilder.option("python.ExecutableList", executable);
530536
} else {
@@ -562,6 +568,7 @@ protected void launch(Builder contextBuilder) {
562568
contextBuilder.option("python.TerminalHeight", Integer.toString(consoleHandler.getTerminalHeight()));
563569

564570
contextBuilder.option("python.CheckHashPycsMode", checkHashPycsMode);
571+
contextBuilder.option("python.RunViaLauncher", "true");
565572

566573
if (multiContext) {
567574
contextBuilder.engine(Engine.newBuilder().allowExperimentalOptions(true).options(enginePolyglotOptions).build());
@@ -610,12 +617,17 @@ protected void launch(Builder contextBuilder) {
610617
System.exit(rc);
611618
}
612619

620+
private static boolean matchesPythonOption(String arg, String key) {
621+
assert !key.startsWith("python.");
622+
return arg.startsWith("--python." + key) || arg.startsWith("--" + key);
623+
}
624+
613625
private String getContextOptionIfSetViaCommandLine(String key) {
614-
if (System.getProperty("polyglot." + key) != null) {
615-
return System.getProperty("polyglot." + key);
626+
if (System.getProperty("polyglot.python." + key) != null) {
627+
return System.getProperty("polyglot.python." + key);
616628
}
617629
for (String f : givenArguments) {
618-
if (f.startsWith("--" + key)) {
630+
if (matchesPythonOption(f, key)) {
619631
String[] splits = f.split("=", 2);
620632
if (splits.length > 1) {
621633
return splits[1];

‎graalpython/com.oracle.graal.python.test/src/tests/test_fcntl.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ def posix_module_backend(self):
5151
import tempfile
5252
import time
5353
import unittest
54+
import sys
5455

5556
PREFIX = 'select_graalpython_test'
5657
TEMP_DIR = tempfile.gettempdir()
@@ -82,18 +83,22 @@ def python_flock_blocks_sh_flock(python_flock_type, sh_flock_type):
8283

8384
class FcntlTests(unittest.TestCase):
8485
@unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)')
86+
@unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility')
8587
def test_flock_x_and_x(self):
8688
python_flock_blocks_sh_flock(fcntl.LOCK_EX, 'x')
8789

8890
@unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)')
91+
@unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility')
8992
def test_flock_x_and_s(self):
9093
python_flock_blocks_sh_flock(fcntl.LOCK_EX, 's')
9194

9295
@unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)')
96+
@unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility')
9397
def test_flock_s_and_x(self):
9498
python_flock_blocks_sh_flock(fcntl.LOCK_SH, 'x')
9599

96100
@unittest.skipUnless(__graalpython__.posix_module_backend() != 'java', 'No support in Truffle API (GR-28740)')
101+
@unittest.skipUnless(sys.platform != 'darwin', 'MacOSX does not have flock utility')
97102
def test_flock_s_and_s(self):
98103
os.close(os.open(TEST_FILENAME_FULL_PATH, os.O_WRONLY | os.O_CREAT))
99104
file = os.open(TEST_FILENAME_FULL_PATH, os.O_WRONLY)
@@ -104,4 +109,4 @@ def test_flock_s_and_s(self):
104109
assert p.poll() == 42
105110
finally:
106111
fcntl.flock(file, fcntl.LOCK_UN)
107-
os.close(file)
112+
os.close(file)

‎graalpython/com.oracle.graal.python.test/src/tests/test_interop.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,7 @@ def test_isinstance02():
522522
assert isinstance(h, HashMap)
523523
assert isinstance(h, Map)
524524

525+
@skipIf(is_native, "not supported in native mode")
525526
def test_is_type():
526527
import java
527528
from java.util.logging import Handler
@@ -538,7 +539,8 @@ def test_is_type():
538539
assert not java.is_type(lr)
539540
assert not java.is_type(Level.ALL)
540541
assert not java.is_type("ahoj")
541-
542+
543+
@skipIf(is_native, "not supported in native mode")
542544
def test_extend_java_class_01():
543545
from java.util.logging import Handler
544546
from java.util.logging import LogRecord
@@ -585,7 +587,8 @@ def sayHello(self):
585587
assert h2.isLoggable(lr)
586588
assert h2.this.counter == 1
587589
assert h.this.counter == 3
588-
590+
591+
@skipIf(is_native, "not supported in native mode")
589592
def test_extend_java_class_02():
590593
from java.math import BigDecimal
591594
try:
@@ -595,7 +598,8 @@ class MyDecimal(BigDecimal):
595598
assert True
596599
else:
597600
assert False
598-
601+
602+
@skipIf(is_native, "not supported in native mode")
599603
def test_extend_java_class_03():
600604
#test of java constructor
601605
from java.util.logging import LogRecord

‎graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ def test_os_pipe():
1010
r,w = os.pipe()
1111
written = os.write(w, b"hello")
1212
assert os.read(r, written) == b"hello"
13+
os.close(r)
14+
os.close(w)
1315

1416

1517
class TestSubprocess(unittest.TestCase):

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_repl.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
1-
*graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_close_stdin
21
*graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_multiline_string_parsing
32
*graalpython.lib-python.3.test.test_repl.TestInteractiveInterpreter.test_no_memory

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
5656
import com.oracle.graal.python.builtins.objects.bytes.BytesUtils;
5757
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
58-
import com.oracle.graal.python.builtins.objects.bytes.PBytesLike;
5958
import com.oracle.graal.python.builtins.objects.common.SequenceNodes.LenNode;
6059
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
6160
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.GetItemNode;
@@ -2387,7 +2386,7 @@ PosixFileHandle doUnicode(PString value,
23872386
}
23882387

23892388
@Specialization
2390-
PosixFileHandle doBytes(PBytesLike value,
2389+
PosixFileHandle doBytes(PBytes value,
23912390
@Cached BytesNodes.ToBytesNode toByteArrayNode,
23922391
@CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib) {
23932392
return new PosixPath(value, checkPath(posixLib.createPathFromBytes(getPosixSupport(), toByteArrayNode.execute(value))), true);
@@ -2443,7 +2442,7 @@ PosixFileHandle doGeneric(VirtualFrame frame, Object value,
24432442
}
24442443

24452444
protected boolean isHandled(Object value) {
2446-
return PGuards.isPNone(value) && nullable || PGuards.canBeInteger(value) && allowFd || PGuards.isString(value) || PGuards.isBytes(value);
2445+
return PGuards.isPNone(value) && nullable || PGuards.canBeInteger(value) && allowFd || PGuards.isString(value) || PGuards.isPBytes(value);
24472446
}
24482447

24492448
private String getAllowedTypes() {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SelectModuleBuiltins.java‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242

4343
import static com.oracle.graal.python.nodes.ErrorMessages.INVALID_VALUE_NAN;
4444
import static com.oracle.graal.python.nodes.ErrorMessages.TOO_LARGE_TO_CONVERT_TO;
45+
import static com.oracle.graal.python.runtime.PosixSupportLibrary.FD_SETSIZE;
46+
import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
4547

4648
import java.util.List;
4749

@@ -63,6 +65,8 @@
6365
import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
6466
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
6567
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
68+
import com.oracle.graal.python.runtime.EmulatedPosixSupport;
69+
import com.oracle.graal.python.runtime.PosixResources;
6670
import com.oracle.graal.python.runtime.PosixSupportLibrary;
6771
import com.oracle.graal.python.runtime.PosixSupportLibrary.ChannelNotSelectableException;
6872
import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException;
@@ -126,10 +130,10 @@ PTuple doGeneric(VirtualFrame frame, Object rlist, Object wlist, Object xlist, O
126130
@Cached PyTimeFromObjectNode pyTimeFromObjectNode,
127131
@CachedLibrary(limit = "3") PythonObjectLibrary itemLib,
128132
@Cached BranchProfile notSelectableBranch) {
129-
130-
ObjAndFDList readFDs = seq2set(frame, rlist, rlistLibrary, itemLib, callGetItemNode, constructListNode);
131-
ObjAndFDList writeFDs = seq2set(frame, wlist, wlistLibrary, itemLib, callGetItemNode, constructListNode);
132-
ObjAndFDList xFDs = seq2set(frame, xlist, xlistLibrary, itemLib, callGetItemNode, constructListNode);
133+
EmulatedPosixSupport emulatedPosixSupport = getContext().getResources();
134+
ObjAndFDList readFDs = seq2set(frame, rlist, rlistLibrary, itemLib, callGetItemNode, constructListNode, emulatedPosixSupport);
135+
ObjAndFDList writeFDs = seq2set(frame, wlist, wlistLibrary, itemLib, callGetItemNode, constructListNode, emulatedPosixSupport);
136+
ObjAndFDList xFDs = seq2set(frame, xlist, xlistLibrary, itemLib, callGetItemNode, constructListNode, emulatedPosixSupport);
133137

134138
Timeval timeoutval = null;
135139
if (!PGuards.isPNone(timeout)) {
@@ -141,7 +145,12 @@ PTuple doGeneric(VirtualFrame frame, Object rlist, Object wlist, Object xlist, O
141145

142146
SelectResult result;
143147
try {
144-
result = posixLib.select(getPosixSupport(), readFDs.fds, writeFDs.fds, xFDs.fds, timeoutval);
148+
if (readFDs.containsSocket || writeFDs.containsSocket || xFDs.containsSocket) {
149+
// TODO remove this once native sockets are supported
150+
result = PosixSupportLibrary.getUncached().select(emulatedPosixSupport, readFDs.fds, writeFDs.fds, xFDs.fds, timeoutval);
151+
} else {
152+
result = posixLib.select(getPosixSupport(), readFDs.fds, writeFDs.fds, xFDs.fds, timeoutval);
153+
}
145154
} catch (PosixException e) {
146155
throw raiseOSErrorFromPosixException(frame, e);
147156
} catch (ChannelNotSelectableException e) {
@@ -170,31 +179,39 @@ private PList toList(boolean[] result, ObjAndFDList fds) {
170179
return factory().createList(PythonUtils.arrayCopyOf(resultObjs, resultObjsIdx));
171180
}
172181

173-
private static ObjAndFDList seq2set(VirtualFrame frame, Object sequence, PythonObjectLibrary sequenceLib, PythonObjectLibrary itemLib, LookupAndCallBinaryNode callGetItemNode,
174-
FastConstructListNode constructListNode) {
182+
private ObjAndFDList seq2set(VirtualFrame frame, Object sequence, PythonObjectLibrary sequenceLib, PythonObjectLibrary itemLib, LookupAndCallBinaryNode callGetItemNode,
183+
FastConstructListNode constructListNode, PosixResources resources) {
175184
PArguments.ThreadState threadState = PArguments.getThreadState(frame);
176185
// We cannot assume any size of those two arrays, because the sequence may change as a
177186
// side effect of the invocation of fileno. We also need to call lengthWithState
178187
// repeatedly in the loop condition
179188
ArrayBuilder<Object> objects = new ArrayBuilder<>();
180189
IntArrayBuilder fds = new IntArrayBuilder();
181190
PSequence pSequence = constructListNode.execute(sequence);
191+
boolean containsSocket = false;
182192
for (int i = 0; i < sequenceLib.lengthWithState(sequence, threadState); i++) {
183193
Object pythonObject = callGetItemNode.executeObject(frame, pSequence, i);
184194
objects.add(pythonObject);
185-
fds.add(itemLib.asFileDescriptorWithState(pythonObject, threadState));
195+
int fd = itemLib.asFileDescriptorWithState(pythonObject, threadState);
196+
if (fd >= FD_SETSIZE) {
197+
throw raise(ValueError, ErrorMessages.FILE_DESCRIPTOR_OUT_OF_RANGE_IN_SELECT);
198+
}
199+
fds.add(fd);
200+
containsSocket |= resources.isSocket(fd);
186201
}
187-
return new ObjAndFDList(objects.toArray(new Object[0]), fds.toArray());
202+
return new ObjAndFDList(objects.toArray(new Object[0]), fds.toArray(), containsSocket);
188203
}
189204

190205
@ValueType
191206
private static final class ObjAndFDList {
192207
private final Object[] objects;
193208
private final int[] fds;
209+
private final boolean containsSocket; // TODO remove when native sockets are supported
194210

195-
private ObjAndFDList(Object[] objects, int[] fds) {
211+
private ObjAndFDList(Object[] objects, int[] fds, boolean containsSocket) {
196212
this.objects = objects;
197213
this.fds = fds;
214+
this.containsSocket = containsSocket;
198215
}
199216
}
200217

0 commit comments

Comments
 (0)