Skip to content

Commit 2fd3d5c

Browse files
timfeltimfel
committed
[GR-12956] [GR-12617] [GR-12942] Fixes for Graal compilation during numpy setup
PullRequest: graalpython/324
2 parents 035852b + 131ae49 commit 2fd3d5c

File tree

24 files changed

+936
-117
lines changed

24 files changed

+936
-117
lines changed

‎graalpython/com.oracle.graal.python.cext/src/dictobject.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ PyObject* PyDict_GetItem(PyObject* d, PyObject* k) {
5757
return UPCALL_CEXT_O(_jls_PyDict_GetItem, native_to_java(d), native_to_java(k));
5858
}
5959

60+
PyObject* _PyDict_GetItemId(PyObject* d, _Py_Identifier* id) {
61+
return PyDict_GetItemString(d, id->string);
62+
}
63+
6064
UPCALL_ID(PyDict_DelItem);
6165
int PyDict_DelItem(PyObject *d, PyObject *k) {
6266
return UPCALL_CEXT_I(_jls_PyDict_DelItem, native_to_java(d), native_to_java(k));

‎graalpython/com.oracle.graal.python.cext/src/exceptions.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ PyObject * PyExc_AssertionError = NULL;
8181
PyObject * PyExc_UnboundLocalError = NULL;
8282
PyObject * PyExc_NotImplementedError = NULL;
8383
PyObject * PyExc_RecursionError = NULL;
84+
PyObject * PyExc_UnicodeEncodeError = NULL;
8485

8586
void initialize_exceptions() {
8687
PyExc_AttributeError = PY_EXCEPTION("AttributeError");
@@ -118,6 +119,7 @@ void initialize_exceptions() {
118119
PyExc_NotImplementedError = PY_EXCEPTION("NotImplementedError");
119120
PyExc_RecursionError = PY_EXCEPTION("RecursionError");
120121
PyExc_NotImplementedError = PY_EXCEPTION("NotImplementedError");
122+
PyExc_UnicodeEncodeError = PY_EXCEPTION("UnicodeEncodeError");
121123
}
122124

123125

‎graalpython/com.oracle.graal.python.cext/src/object.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,10 @@ PyObject* _PyObject_GC_New(PyTypeObject *tp) {
388388
return _PyObject_New(tp);
389389
}
390390

391+
PyVarObject* _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) {
392+
return _PyObject_NewVar(tp, nitems);
393+
}
394+
391395
PyVarObject *
392396
_PyObject_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
393397
{

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonLD.java‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,15 @@
5050
import java.util.ArrayList;
5151
import java.util.Arrays;
5252
import java.util.Collection;
53+
import java.util.HashSet;
5354
import java.util.List;
55+
import java.util.Set;
5456

5557
import jline.internal.InputStreamReader;
5658

5759
public class GraalPythonLD extends GraalPythonCompiler {
60+
private static final String LLVM_IR_BITCODE = "llvm-ir-bitcode";
61+
private static final String LLVM_NM = "llvm-nm";
5862
private static List<String> linkPrefix = Arrays.asList(new String[]{
5963
"llvm-link",
6064
"-o",
@@ -68,6 +72,9 @@ static void main(String[] args) {
6872
private List<String> fileInputs;
6973
private List<String> ldArgs;
7074

75+
private Set<String> definedSymbols = new HashSet<>();
76+
private Set<String> undefinedSymbols = new HashSet<>();
77+
7178
private void run(String[] args) {
7279
parseOptions(args);
7380
launchLD();
@@ -94,16 +101,16 @@ private void parseOptions(String[] args) {
94101
break;
95102
default:
96103
if (arg.endsWith(".o") || arg.endsWith(".bc")) {
97-
fileInputs.add(arg);
104+
addFile(arg);
98105
} else if (arg.startsWith("-L")) {
99106
libraryDirs.add(arg.substring(2));
100107
} else if (arg.startsWith("-l")) {
101108
List<String> bcFiles = searchLib(libraryDirs, arg.substring(2));
102109
for (String bcFile : bcFiles) {
103110
try {
104-
if (Files.probeContentType(Paths.get(bcFile)).contains("llvm-ir-bitcode")) {
111+
if (Files.probeContentType(Paths.get(bcFile)).contains(LLVM_IR_BITCODE)) {
105112
logV("library input:", bcFile);
106-
fileInputs.add(bcFile);
113+
addFile(bcFile);
107114
} else {
108115
droppedArgs.add(bcFile + "(dropped as library input)");
109116
}
@@ -122,6 +129,55 @@ private void parseOptions(String[] args) {
122129
logV("Dropped args: ", droppedArgs);
123130
}
124131

132+
void addFile(String f) {
133+
fileInputs.add(f);
134+
135+
try {
136+
// symbols defined up to here
137+
ProcessBuilder nm = new ProcessBuilder();
138+
nm.command(LLVM_NM, "-g", "--defined-only", f);
139+
nm.redirectInput(Redirect.INHERIT);
140+
nm.redirectError(Redirect.INHERIT);
141+
nm.redirectOutput(Redirect.PIPE);
142+
Process nmProc = nm.start();
143+
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(nmProc.getInputStream()))) {
144+
String line = null;
145+
while ((line = buffer.readLine()) != null) {
146+
String[] symboldef = line.split(" ");
147+
if (symboldef.length >= 2) {
148+
definedSymbols.add(symboldef[symboldef.length - 1]);
149+
}
150+
}
151+
}
152+
nmProc.waitFor();
153+
154+
// remove now resolved symbols
155+
undefinedSymbols.removeAll(definedSymbols);
156+
157+
// add symbols undefined now
158+
nm.command(LLVM_NM, "-u", f);
159+
nm.redirectInput(Redirect.INHERIT);
160+
nm.redirectError(Redirect.INHERIT);
161+
nm.redirectOutput(Redirect.PIPE);
162+
nmProc = nm.start();
163+
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(nmProc.getInputStream()))) {
164+
String line = null;
165+
while ((line = buffer.readLine()) != null) {
166+
String[] symboldef = line.split(" ");
167+
if (symboldef.length >= 2) {
168+
String sym = symboldef[symboldef.length - 1];
169+
if (!definedSymbols.contains(sym)) {
170+
undefinedSymbols.add(sym);
171+
}
172+
}
173+
}
174+
}
175+
nmProc.waitFor();
176+
} catch (InterruptedException | IOException e) {
177+
throw new RuntimeException(e);
178+
}
179+
}
180+
125181
private List<String> searchLib(List<String> libraryDirs, String lib) {
126182
List<String> bcFiles = new ArrayList<>();
127183
String[] suffixes = new String[]{".bc", ".o", ".a", ".so"};
@@ -179,6 +235,52 @@ private Collection<? extends String> arMembers(String path) throws IOException,
179235
extractAr.redirectOutput(Redirect.INHERIT);
180236
extractAr.command("ar", "xv", path);
181237
extractAr.start().waitFor();
238+
239+
// ar has special semantics w.r.t ordering of included symbols. we try to emulate the smarts
240+
// of GNU ld by listing all undefined symbols until here, extracting only those that we are
241+
// still missing, and adding them to a bitcode file that will only add these to the linked
242+
// product.
243+
// According to some emscripten documentation and ML discussions, this is actually an error
244+
// in the build process, because such a smart linker should not be assumed for POSIX, but it
245+
// seems ok to emulate this at least for the very common case of ar archives with symbol
246+
// definitions that overlap what's defined in explicitly include .o files
247+
for (String f : members) {
248+
if (Files.probeContentType(Paths.get(f)).contains(LLVM_IR_BITCODE)) {
249+
HashSet<String> definedHere = new HashSet<>();
250+
ProcessBuilder nm = new ProcessBuilder();
251+
nm.command(LLVM_NM, "-g", "--defined-only", f);
252+
nm.redirectInput(Redirect.INHERIT);
253+
nm.redirectError(Redirect.INHERIT);
254+
nm.redirectOutput(Redirect.PIPE);
255+
Process nmProc = nm.start();
256+
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(nmProc.getInputStream()))) {
257+
String line = null;
258+
while ((line = buffer.readLine()) != null) {
259+
String[] symboldef = line.split(" ");
260+
if (symboldef.length >= 2) {
261+
definedHere.add(symboldef[symboldef.length - 1]);
262+
}
263+
}
264+
}
265+
nmProc.waitFor();
266+
267+
ArrayList<String> extractCmd = new ArrayList<>();
268+
extractCmd.add("llvm-extract");
269+
for (String def : definedHere) {
270+
if (undefinedSymbols.contains(def)) {
271+
definedSymbols.add(def);
272+
undefinedSymbols.remove(def);
273+
extractCmd.add("-func");
274+
extractCmd.add(def);
275+
}
276+
}
277+
extractCmd.add(f);
278+
extractCmd.add("-o");
279+
extractCmd.add(f);
280+
exec(extractCmd);
281+
}
282+
}
283+
182284
return members;
183285
}
184286

‎graalpython/com.oracle.graal.python.test/src/tests/test_with.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
# Qunaibit 02/05/2014
2525
# With Statement
2626

27+
import sys
28+
2729
a = 5
2830

2931
LOG = []
@@ -171,3 +173,90 @@ def test_with_restore_raise():
171173
, "type: <class 'TypeError'>"
172174
, "value: "
173175
, "except exit"], "was: %s" % log
176+
177+
178+
def test_with_in_generator():
179+
enter = 0
180+
exit = 0
181+
itr = 0
182+
nxt = 0
183+
r = []
184+
185+
class Gen():
186+
def __init__(self):
187+
self.l = iter([1,2,3])
188+
189+
def __enter__(self):
190+
nonlocal enter
191+
enter += 1
192+
return self
193+
194+
def __exit__(self, *args):
195+
nonlocal exit
196+
exit += 1
197+
198+
def __iter__(self):
199+
nonlocal itr
200+
itr += 1
201+
return self
202+
203+
def __next__(self):
204+
nonlocal nxt
205+
nxt += 1
206+
return next(self.l)
207+
208+
def gen():
209+
with Gen() as g:
210+
for i in g:
211+
yield i
212+
213+
e = None
214+
try:
215+
try:
216+
raise OverflowError
217+
except OverflowError as e1:
218+
e = e1
219+
for i in gen():
220+
r.append(i)
221+
raise
222+
except OverflowError as e2:
223+
assert e2 is e
224+
else:
225+
assert False
226+
227+
assert r == [1,2,3], r
228+
assert enter == 1, enter
229+
assert exit == 1, exit
230+
assert itr == 1, itr
231+
assert nxt == 4, nxt
232+
233+
234+
def test_with_non_inherited():
235+
class X():
236+
pass
237+
238+
x = X()
239+
x.__enter__ = lambda s: s
240+
241+
try:
242+
with x as l:
243+
pass
244+
except AttributeError as e:
245+
if sys.version_info.minor > 5:
246+
assert "__enter__" in str(e)
247+
else:
248+
assert "__exit__" in str(e)
249+
250+
251+
y_enter_called = 0
252+
class Y():
253+
def __enter__(self):
254+
nonlocal y_enter_called
255+
y_enter_called += 1
256+
257+
try:
258+
with Y() as y:
259+
pass
260+
except AttributeError as e:
261+
assert "__exit__" in str(e)
262+
assert y_enter_called == 0

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,8 @@
135135
import com.oracle.graal.python.builtins.objects.module.PythonModule;
136136
import com.oracle.graal.python.builtins.objects.object.ObjectBuiltins;
137137
import com.oracle.graal.python.builtins.objects.object.PythonObject;
138+
import com.oracle.graal.python.builtins.objects.posix.DirEntryBuiltins;
139+
import com.oracle.graal.python.builtins.objects.posix.ScandirIteratorBuiltins;
138140
import com.oracle.graal.python.builtins.objects.random.RandomBuiltins;
139141
import com.oracle.graal.python.builtins.objects.range.RangeBuiltins;
140142
import com.oracle.graal.python.builtins.objects.referencetype.ReferenceTypeBuiltins;
@@ -276,6 +278,8 @@ private static final PythonBuiltins[] initializeBuiltins() {
276278
new GetSetDescriptorTypeBuiltins(),
277279
new BaseExceptionBuiltins(),
278280
new PosixModuleBuiltins(),
281+
new ScandirIteratorBuiltins(),
282+
new DirEntryBuiltins(),
279283
new ImpModuleBuiltins(),
280284
new ArrayModuleBuiltins(),
281285
new ArrayBuiltins(),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ public enum PythonBuiltinClassType implements LazyPythonClass {
9393
PSocket("socket", "_socket"),
9494
PStaticmethod("staticmethod", "builtins"),
9595
PClassmethod("classmethod", "builtins"),
96+
PScandirIterator("ScandirIterator", "posix"),
97+
PDirEntry("DirEntry", "posix"),
9698

9799
// Errors and exceptions:
98100

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import java.nio.channels.ReadableByteChannel;
4646
import java.nio.channels.SeekableByteChannel;
4747
import java.nio.channels.WritableByteChannel;
48+
import java.nio.file.InvalidPathException;
4849
import java.nio.file.LinkOption;
4950
import java.nio.file.StandardCopyOption;
5051
import java.nio.file.StandardOpenOption;
@@ -85,11 +86,13 @@
8586
import com.oracle.graal.python.builtins.objects.ints.PInt;
8687
import com.oracle.graal.python.builtins.objects.str.PString;
8788
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
89+
import com.oracle.graal.python.builtins.objects.type.PythonClass;
8890
import com.oracle.graal.python.nodes.PNodeWithContext;
8991
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
9092
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
9193
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
9294
import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
95+
import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
9396
import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
9497
import com.oracle.graal.python.nodes.util.CastToIndexNode;
9598
import com.oracle.graal.python.runtime.PosixResources;
@@ -529,7 +532,7 @@ private static int posixPermissionsToMode(int inputMode, final Set<PosixFilePerm
529532
return mode;
530533
}
531534

532-
protected static StatNode create() {
535+
public static StatNode create() {
533536
return StatNodeFactory.create();
534537
}
535538
}
@@ -565,6 +568,43 @@ private static Object[] listToArray(Collection<TruffleFile> listFiles) {
565568
}
566569
}
567570

571+
@Builtin(name = "ScandirIterator", fixedNumOfPositionalArgs = 2, constructsClass = PythonBuiltinClassType.PScandirIterator, isPublic = true)
572+
@GenerateNodeFactory
573+
@TypeSystemReference(PythonArithmeticTypes.class)
574+
public abstract static class ScandirIterNode extends PythonBinaryBuiltinNode {
575+
private final BranchProfile gotException = BranchProfile.create();
576+
577+
@Specialization
578+
Object doit(PythonClass cls, String path) {
579+
try {
580+
TruffleFile file = getContext().getEnv().getTruffleFile(path);
581+
return factory().createScandirIterator(cls, path, file.newDirectoryStream());
582+
} catch (SecurityException | IOException e) {
583+
gotException.enter();
584+
throw raise(OSError, path);
585+
}
586+
}
587+
}
588+
589+
@Builtin(name = "DirEntry", fixedNumOfPositionalArgs = 3, constructsClass = PythonBuiltinClassType.PDirEntry, isPublic = true)
590+
@GenerateNodeFactory
591+
@TypeSystemReference(PythonArithmeticTypes.class)
592+
public abstract static class DirEntryNode extends PythonTernaryBuiltinNode {
593+
private final BranchProfile gotException = BranchProfile.create();
594+
595+
@Specialization
596+
Object doit(PythonClass cls, String name, String path) {
597+
try {
598+
TruffleFile dir = getContext().getEnv().getTruffleFile(path);
599+
TruffleFile file = dir.resolve(name);
600+
return factory().createDirEntry(cls, name, file);
601+
} catch (SecurityException | InvalidPathException e) {
602+
gotException.enter();
603+
throw raise(OSError, path);
604+
}
605+
}
606+
}
607+
568608
@Builtin(name = "dup", fixedNumOfPositionalArgs = 1)
569609
@GenerateNodeFactory
570610
@TypeSystemReference(PythonArithmeticTypes.class)

0 commit comments

Comments
 (0)