Skip to content

Commit a1aef80

Browse files
msimacekmsimacek
committed
Fix handling non-ascii characters in CStringWrapper
1 parent 7d72437 commit a1aef80

File tree

9 files changed

+64
-58
lines changed

9 files changed

+64
-58
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_misc.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -367,3 +367,21 @@ def test_graalpy_version():
367367
expected_num <<= 8
368368
expected_num |= parts[i]
369369
assert tester.get_version_num() == expected_num
370+
371+
372+
def test_unicode_docstring():
373+
class ClassWithDoc:
374+
"""This class has a doc 🙂"""
375+
376+
tester = CPyExtType(
377+
"DocstringTester",
378+
code='''
379+
static PyObject* get_doc(PyObject* unused, PyObject* type) {
380+
return PyUnicode_FromString(((PyTypeObject*)type)->tp_doc);
381+
}
382+
''',
383+
tp_methods='''
384+
{"get_doc", (PyCFunction)get_doc, METH_O | METH_STATIC, ""}
385+
''',
386+
)
387+
assert tester.get_doc(ClassWithDoc)

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextAbstractBuiltins.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -943,10 +943,11 @@ static int set(GetSetDescriptor obj, Object value,
943943
static int set(PythonAbstractNativeObject type, Object value,
944944
@Bind("this") Node inliningTarget,
945945
@SuppressWarnings("unused") @Cached IsTypeNode isType,
946+
@Cached TruffleString.SwitchEncodingNode switchEncoding,
946947
@Cached CStructAccess.WritePointerNode writePointerNode) {
947948
Object cValue;
948949
if (value instanceof TruffleString stringValue) {
949-
cValue = new CStringWrapper(stringValue);
950+
cValue = new CStringWrapper(switchEncoding.execute(stringValue, TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
950951
} else {
951952
cValue = PythonContext.get(inliningTarget).getNativeNull();
952953
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PyMethodDefHelper.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -156,7 +156,7 @@ Object allocate() {
156156
assert name != null;
157157
CStringWrapper nameWrapper;
158158
try {
159-
nameWrapper = new CStringWrapper(name);
159+
nameWrapper = new CStringWrapper(name.switchEncodingUncached(TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
160160
} catch (CannotCastException e) {
161161
throw CompilerDirectives.shouldNotReachHere(e);
162162
}
@@ -166,7 +166,7 @@ Object allocate() {
166166
docWrapper = PythonContext.get(null).getNativeNull();
167167
} else {
168168
try {
169-
docWrapper = new CStringWrapper(doc);
169+
docWrapper = new CStringWrapper(doc.switchEncodingUncached(TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
170170
} catch (CannotCastException e) {
171171
throw CompilerDirectives.shouldNotReachHere(e);
172172
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PythonClassNativeWrapper.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -76,20 +76,20 @@
7676
public final class PythonClassNativeWrapper extends PythonAbstractObjectNativeWrapper {
7777
private final CStringWrapper nameWrapper;
7878

79-
private PythonClassNativeWrapper(PythonManagedClass object, TruffleString name) {
79+
private PythonClassNativeWrapper(PythonManagedClass object, TruffleString name, TruffleString.SwitchEncodingNode switchEncoding) {
8080
super(object, true);
81-
this.nameWrapper = new CStringWrapper(name);
81+
this.nameWrapper = new CStringWrapper(switchEncoding.execute(name, TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
8282
}
8383

8484
public CStringWrapper getNameWrapper() {
8585
return nameWrapper;
8686
}
8787

88-
public static PythonClassNativeWrapper wrap(PythonManagedClass obj, TruffleString name) {
88+
public static PythonClassNativeWrapper wrap(PythonManagedClass obj, TruffleString name, TruffleString.SwitchEncodingNode switchEncoding) {
8989
// important: native wrappers are cached
9090
PythonClassNativeWrapper nativeWrapper = obj.getClassNativeWrapper();
9191
if (nativeWrapper == null) {
92-
nativeWrapper = new PythonClassNativeWrapper(obj, name);
92+
nativeWrapper = new PythonClassNativeWrapper(obj, name, switchEncoding);
9393
obj.setNativeWrapper(nativeWrapper);
9494
}
9595
return nativeWrapper;
@@ -108,7 +108,7 @@ public static void wrapNative(PythonManagedClass clazz, TruffleString name, Obje
108108
throw CompilerDirectives.shouldNotReachHere();
109109
}
110110

111-
PythonClassNativeWrapper wrapper = new PythonClassNativeWrapper(clazz, name);
111+
PythonClassNativeWrapper wrapper = new PythonClassNativeWrapper(clazz, name, TruffleString.SwitchEncodingNode.getUncached());
112112
clazz.setNativeWrapper(wrapper);
113113

114114
CStructAccess.ReadI64Node readI64 = CStructAccess.ReadI64Node.getUncached();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ToNativeTypeNode.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@
103103
import com.oracle.graal.python.runtime.PythonContext;
104104
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
105105
import com.oracle.truffle.api.CompilerAsserts;
106+
import com.oracle.truffle.api.strings.TruffleString;
106107

107108
public abstract class ToNativeTypeNode {
108109

@@ -279,7 +280,7 @@ static void initializeType(PythonClassNativeWrapper obj, Object mem, boolean hea
279280
// return a C string wrapper that really allocates 'char*' on TO_NATIVE
280281
Object docObj = clazz.getAttribute(SpecialAttributeNames.T___DOC__);
281282
try {
282-
docObj = new CStringWrapper(CastToTruffleStringNode.executeUncached(docObj));
283+
docObj = new CStringWrapper(CastToTruffleStringNode.executeUncached(docObj).switchEncodingUncached(TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
283284
} catch (CannotCastException e) {
284285
// if not directly a string, give up (we don't call descriptors here)
285286
docObj = ctx.getNativeNull();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/GetNativeWrapperNode.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -53,6 +53,7 @@
5353
import com.oracle.graal.python.builtins.objects.cext.capi.TruffleObjectNativeWrapper;
5454
import com.oracle.graal.python.builtins.objects.floats.PFloat;
5555
import com.oracle.graal.python.builtins.objects.ints.PInt;
56+
import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass;
5657
import com.oracle.graal.python.builtins.objects.type.PythonManagedClass;
5758
import com.oracle.graal.python.builtins.objects.type.TypeNodes;
5859
import com.oracle.graal.python.builtins.objects.type.TypeNodes.IsTypeNode;
@@ -65,6 +66,7 @@
6566
import com.oracle.truffle.api.dsl.Bind;
6667
import com.oracle.truffle.api.dsl.Cached;
6768
import com.oracle.truffle.api.dsl.Cached.Exclusive;
69+
import com.oracle.truffle.api.dsl.Cached.Shared;
6870
import com.oracle.truffle.api.dsl.GenerateUncached;
6971
import com.oracle.truffle.api.dsl.ImportStatic;
7072
import com.oracle.truffle.api.dsl.Specialization;
@@ -172,15 +174,17 @@ static PythonNativeWrapper doSingleton(PythonAbstractObject object,
172174
@Specialization
173175
static PythonNativeWrapper doPythonClassUncached(PythonManagedClass object,
174176
@Bind("this") Node inliningTarget,
175-
@Exclusive @Cached TypeNodes.GetNameNode getNameNode) {
176-
return PythonClassNativeWrapper.wrap(object, getNameNode.execute(inliningTarget, object));
177+
@Cached TypeNodes.GetNameNode getNameNode,
178+
@Shared @Cached TruffleString.SwitchEncodingNode switchEncoding) {
179+
return PythonClassNativeWrapper.wrap(object, getNameNode.execute(inliningTarget, object), switchEncoding);
177180
}
178181

179182
@Specialization
180183
static PythonNativeWrapper doPythonTypeUncached(PythonBuiltinClassType object,
181184
@Bind("this") Node inliningTarget,
182-
@Exclusive @Cached TypeNodes.GetNameNode getNameNode) {
183-
return PythonClassNativeWrapper.wrap(PythonContext.get(getNameNode).lookupType(object), getNameNode.execute(inliningTarget, object));
185+
@Shared @Cached TruffleString.SwitchEncodingNode switchEncoding) {
186+
PythonBuiltinClass type = PythonContext.get(inliningTarget).lookupType(object);
187+
return PythonClassNativeWrapper.wrap(type, type.getName(), switchEncoding);
184188
}
185189

186190
@Specialization(guards = {"!isClass(inliningTarget, object, isTypeNode)", "!isNativeObject(object)", "!isSpecialSingleton(object)"}, limit = "1")

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/common/CArrayWrappers.java‎

Lines changed: 19 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -40,7 +40,6 @@
4040
*/
4141
package com.oracle.graal.python.builtins.objects.cext.common;
4242

43-
import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;
4443
import static com.oracle.graal.python.util.PythonUtils.byteArraySupport;
4544

4645
import java.nio.ByteOrder;
@@ -58,7 +57,6 @@
5857
import com.oracle.truffle.api.dsl.Bind;
5958
import com.oracle.truffle.api.dsl.Cached;
6059
import com.oracle.truffle.api.dsl.Cached.Exclusive;
61-
import com.oracle.truffle.api.dsl.Cached.Shared;
6260
import com.oracle.truffle.api.interop.InteropLibrary;
6361
import com.oracle.truffle.api.interop.InvalidArrayIndexException;
6462
import com.oracle.truffle.api.interop.InvalidBufferOffsetException;
@@ -175,25 +173,22 @@ public void free() {
175173
* wrapper let's a TruffleString look like a {@code char*}.
176174
*/
177175
@ExportLibrary(InteropLibrary.class)
178-
@SuppressWarnings("truffle-abstract-export")
179176
public static final class CStringWrapper extends CArrayWrapper {
177+
private TruffleString.Encoding encoding;
180178

181-
public CStringWrapper(TruffleString delegate) {
179+
public CStringWrapper(TruffleString delegate, TruffleString.Encoding encoding) {
182180
super(delegate);
181+
this.encoding = encoding;
182+
assert delegate.isValidUncached(encoding);
183183
}
184184

185185
public TruffleString getString() {
186-
TruffleString s = (TruffleString) getDelegate();
187-
// TODO GR-37217: use sys.getdefaultencoding if the string contains non-latin1
188-
// codepoints
189-
assert s.getCodeRangeUncached(TS_ENCODING) == TruffleString.CodeRange.ASCII;
190-
return s;
186+
return (TruffleString) getDelegate();
191187
}
192188

193189
@ExportMessage
194-
long getArraySize(
195-
@Shared @Cached TruffleString.CodePointLengthNode codePointLengthNode) {
196-
return codePointLengthNode.execute(getString(), TS_ENCODING) + 1;
190+
long getArraySize() {
191+
return getString().byteLength(encoding) + 1;
197192
}
198193

199194
@ExportMessage
@@ -204,28 +199,17 @@ boolean hasArrayElements() {
204199

205200
@ExportMessage
206201
byte readArrayElement(long index,
207-
@Shared @Cached TruffleString.CodePointLengthNode codePointLengthNode,
208-
@Shared @Cached TruffleString.CodePointAtIndexNode codePointAtIndexNode) throws InvalidArrayIndexException {
202+
@CachedLibrary("this") InteropLibrary thisLib) throws InvalidArrayIndexException, UnsupportedMessageException {
209203
try {
210-
int idx = PInt.intValueExact(index);
211-
TruffleString s = getString();
212-
int len = codePointLengthNode.execute(s, TS_ENCODING);
213-
if (idx >= 0 && idx < len) {
214-
return (byte) codePointAtIndexNode.execute(s, idx, TS_ENCODING);
215-
} else if (idx == len) {
216-
return 0;
217-
}
218-
} catch (OverflowException e) {
219-
// fall through
204+
return thisLib.readBufferByte(this, index);
205+
} catch (InvalidBufferOffsetException e) {
206+
throw InvalidArrayIndexException.create(index);
220207
}
221-
CompilerDirectives.transferToInterpreterAndInvalidate();
222-
throw InvalidArrayIndexException.create(index);
223208
}
224209

225210
@ExportMessage
226-
boolean isArrayElementReadable(long identifier,
227-
@Shared @Cached TruffleString.CodePointLengthNode codePointLengthNode) {
228-
return 0 <= identifier && identifier < getArraySize(codePointLengthNode);
211+
boolean isArrayElementReadable(long index) {
212+
return 0 <= index && index <= getString().byteLength(encoding);
229213
}
230214

231215
@ExportMessage
@@ -248,19 +232,17 @@ boolean hasBufferElements() {
248232
}
249233

250234
@ExportMessage
251-
long getBufferSize(
252-
@Shared @Cached TruffleString.CodePointLengthNode codePointLengthNode) {
253-
return codePointLengthNode.execute(getString(), TS_ENCODING) + 1;
235+
long getBufferSize() {
236+
return getString().byteLength(encoding) + 1;
254237
}
255238

256239
@ExportMessage
257240
byte readBufferByte(long byteOffset,
258-
@Shared @Cached TruffleString.CodePointLengthNode codePointLengthNode,
259-
@Shared @Cached TruffleString.CodePointAtIndexNode codePointAtIndexNode) throws InvalidBufferOffsetException {
241+
@Cached TruffleString.ReadByteNode readByteNode) throws InvalidBufferOffsetException {
260242
TruffleString s = getString();
261-
int len = codePointLengthNode.execute(s, TS_ENCODING);
243+
int len = s.byteLength(encoding);
262244
if (byteOffset >= 0 && byteOffset < len) {
263-
return (byte) codePointAtIndexNode.execute(s, (int) byteOffset, TS_ENCODING);
245+
return (byte) readByteNode.execute(s, (int) byteOffset, encoding);
264246
} else if (byteOffset == len) {
265247
return 0;
266248
} else {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyBuffer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -157,7 +157,7 @@ Object readMember(String member,
157157
case J_MEMBER_NDIM:
158158
return buffer.getDims();
159159
case J_MEMBER_FORMAT:
160-
return buffer.getFormat() != null ? new CStringWrapper(buffer.getFormat()) : toNativeNode.execute(PNone.NO_VALUE);
160+
return buffer.getFormat() != null ? new CStringWrapper(buffer.getFormat().switchEncodingUncached(UTF_8), UTF_8) : toNativeNode.execute(PNone.NO_VALUE);
161161
case J_MEMBER_SHAPE:
162162
return toCArray(toNativeNode, buffer.getShape());
163163
case J_MEMBER_STRIDES:

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/llvm/GraalHPyLLVMContext.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -777,7 +777,7 @@ private static HPyExecuteWrapper createContextFunction(HPyContextMember member)
777777
private Object[] createMembers(TruffleString name) {
778778
Object[] members = new Object[HPyContextMember.VALUES.length];
779779

780-
members[HPyContextMember.NAME.ordinal()] = new CStringWrapper(name);
780+
members[HPyContextMember.NAME.ordinal()] = new CStringWrapper(name.switchEncodingUncached(TruffleString.Encoding.UTF_8), TruffleString.Encoding.UTF_8);
781781
// TODO(fa): we should use the value of macro HPY_ABI_VERSION here
782782
members[HPyContextMember.ABI_VERSION.ordinal()] = 0;
783783

0 commit comments

Comments
 (0)