Normalize unicode identifiers

Author: msimacek

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -52,6 +52,8 @@
 import java.util.List;
 import java.util.function.Supplier;
 
+import org.graalvm.shadowed.com.ibm.icu.text.Normalizer2;
+
 import com.oracle.graal.python.pegparser.sst.ArgTy;
 import com.oracle.graal.python.pegparser.sst.CmpOpTy;
 import com.oracle.graal.python.pegparser.sst.ComprehensionTy;
@@ -350,7 +352,7 @@ public Token getLastNonWhitespaceToken() {
     public ExprTy.Name name_token() {
         Token t = expect(Token.Kind.NAME);
         if (t != null) {
-            return factory.createVariable(getText(t), t.sourceRange);
+            return name_from_token(t);
         } else {
             return null;
         }
@@ -504,6 +506,13 @@ public ExprTy.Name name_from_token(Token t) {
             return null;
         }
         String id = getText(t);
+        for (int i = 0; i < id.length(); i++) {
+            if (id.charAt(i) > 0xff) {
+                // If the identifier is not ASCII, normalize it according to PEP 3131
+                id = Normalizer2.getNFKCInstance().normalize(id);
+                break;
+            }
+        }
         return factory.createVariable(id, t.sourceRange);
     }
 

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/tokenizer/Tokenizer.java

@@ -1,4 +1,4 @@
-/* Copyright (c) 2021, 2024, Oracle and/or its affiliates.
+/* Copyright (c) 2021, 2025, Oracle and/or its affiliates.
  * Copyright (C) 1996-2021 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2

graalpython/com.oracle.graal.python.test/src/tests/test_ast.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -181,6 +181,9 @@ def test_unparse_bytes_constant_kind(self):
         exec(compile(tree, '<string>', 'exec'), vars)
         self.assertEqual("u'abc'", vars['f'].__annotations__['x'])
 
+    def test_parse_unicode(self):
+        self.assertEqual(ast.parse("𝕦𝕟𝕚𝕔𝕠𝕕𝕖").body[0].value.id, 'unicode')
+
 
 if __name__ == '__main__':
     unittest.main()