[GR-53373] Update Cython patch to recythonize files after unpatched Cython

PullRequest: graalpython/3326

Author: msimacek

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ResourceModuleBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -126,6 +126,9 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
     @Override
     public void initialize(Python3Core core) {
         super.initialize(core);
+
+        addBuiltinConstant("error", PythonBuiltinClassType.OSError);
+
         addBuiltinConstant("RUSAGE_CHILDREN", RUSAGE_CHILDREN);
         addBuiltinConstant("RUSAGE_SELF", RUSAGE_SELF);
         addBuiltinConstant("RUSAGE_THREAD", RUSAGE_THREAD);

graalpython/lib-graalpython/patches/Cython/Cython-0.29.32.patch

@@ -1,8 +1,21 @@
 diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py
-index 7eb55e2..76d4857 100644
+index 7eb55e2..1fbab30 100644
 --- a/Cython/Build/Dependencies.py
 +++ b/Cython/Build/Dependencies.py
-@@ -1094,6 +1094,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+@@ -1060,6 +1060,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+                 else:
+                     dep_timestamp, dep = deps.newest_dependency(source)
+                     priority = 2 - (dep in deps.immediate_dependencies(source))
++                # GraalPy change: force recythonize if not cythonized by our patched version
++                if os.path.exists(c_file):
++                    with open(c_file, 'rb') as f:
++                        # We modify the watermark to include graalpy
++                        if not b'graalpy' in f.read(100):
++                            c_timestamp = -1
+                 if force or c_timestamp < dep_timestamp:
+                     if not quiet and not force:
+                         if source == dep:
+@@ -1094,6 +1100,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
          progress = progress_fmt.format(i+1, N)
          to_compile[i] = to_compile[i][1:] + (progress,)
 
@@ -101,6 +114,17 @@ index 7361a55..6c4d9d9 100644
          entry.in_cinclude = self.in_cinclude
          entry.create_wrapper = create_wrapper
          if name:
+diff --git a/Cython/Compiler/Version.py b/Cython/Compiler/Version.py
+index dcb561f..c2c06a0 100644
+--- a/Cython/Compiler/Version.py
++++ b/Cython/Compiler/Version.py
+@@ -6,4 +6,5 @@ from .. import __version__ as version
+ 
+ # For 'generated by' header line in C files.
+ 
+-watermark = str(version)
++# GraalPy change: watermark our patched version
++watermark = f'{version}-graalpy'
 diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd
 index cd0f907..712cd92 100644
 --- a/Cython/Includes/cpython/datetime.pxd

graalpython/lib-graalpython/patches/Cython/Cython-0.29.37.patch

@@ -1,8 +1,21 @@
 diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py
-index 92fa96a..2885a4f 100644
+index 92fa96a..8864b78 100644
 --- a/Cython/Build/Dependencies.py
 +++ b/Cython/Build/Dependencies.py
-@@ -1082,6 +1082,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+@@ -1048,6 +1048,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+                 else:
+                     dep_timestamp, dep = deps.newest_dependency(source)
+                     priority = 2 - (dep in deps.immediate_dependencies(source))
++                # GraalPy change: force recythonize if not cythonized by our patched version
++                if os.path.exists(c_file):
++                    with open(c_file, 'rb') as f:
++                        # We modify the watermark to include graalpy
++                        if not b'graalpy' in f.read(100):
++                            c_timestamp = -1
+                 if force or c_timestamp < dep_timestamp:
+                     if not quiet and not force:
+                         if source == dep:
+@@ -1082,6 +1088,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
          progress = progress_fmt.format(i+1, N)
          to_compile[i] = to_compile[i][1:] + (progress,)
 
@@ -101,6 +114,17 @@ index 7361a55..6c4d9d9 100644
          entry.in_cinclude = self.in_cinclude
          entry.create_wrapper = create_wrapper
          if name:
+diff --git a/Cython/Compiler/Version.py b/Cython/Compiler/Version.py
+index dcb561f..c2c06a0 100644
+--- a/Cython/Compiler/Version.py
++++ b/Cython/Compiler/Version.py
+@@ -6,4 +6,5 @@ from .. import __version__ as version
+ 
+ # For 'generated by' header line in C files.
+ 
+-watermark = str(version)
++# GraalPy change: watermark our patched version
++watermark = f'{version}-graalpy'
 diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd
 index cd0f907..712cd92 100644
 --- a/Cython/Includes/cpython/datetime.pxd
@@ -154,7 +178,7 @@ index 099b3c4..0692df0 100644
          cval = ((PyComplexObject *)o)->cval;
      else
 diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c
-index aaa8a8e..152e9dc 100644
+index aaa8a8e..e60a5e0 100644
 --- a/Cython/Utility/Coroutine.c
 +++ b/Cython/Utility/Coroutine.c
 @@ -531,8 +531,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo
@@ -221,7 +245,7 @@ index 93f577f..5dac1d7 100644
  #endif
      else if (PyMethod_Check(method)) {
 diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
-index cfff606..4a6a6ea 100644
+index cfff606..90876c6 100644
 --- a/Cython/Utility/ModuleSetupCode.c
 +++ b/Cython/Utility/ModuleSetupCode.c
 @@ -186,6 +186,48 @@

graalpython/lib-graalpython/patches/Cython/Cython-3.0.10.patch

@@ -1,8 +1,21 @@
 diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py
-index 7de4065..e1589cd 100644
+index 7de4065..981011f 100644
 --- a/Cython/Build/Dependencies.py
 +++ b/Cython/Build/Dependencies.py
-@@ -1127,6 +1127,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+@@ -1090,6 +1090,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
+                 else:
+                     dep_timestamp, dep = deps.newest_dependency(source)
+                     priority = 2 - (dep in deps.immediate_dependencies(source))
++                # GraalPy change: force recythonize if not cythonized by our patched version
++                if os.path.exists(c_file):
++                    with open(c_file, 'rb') as f:
++                        # We modify the watermark to include graalpy
++                        if not b'graalpy' in f.read(100):
++                            c_timestamp = -1
+                 if force or c_timestamp < dep_timestamp:
+                     if not quiet and not force:
+                         if source == dep:
+@@ -1127,6 +1133,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
          progress = progress_fmt.format(i+1, N)
          to_compile[i] = to_compile[i][1:] + (progress,)
 
@@ -103,6 +116,17 @@ index 5f088dd..a258bdc 100644
          entry.in_cinclude = self.in_cinclude
          entry.create_wrapper = create_wrapper
          if name:
+diff --git a/Cython/Compiler/Version.py b/Cython/Compiler/Version.py
+index dcb561f..c2c06a0 100644
+--- a/Cython/Compiler/Version.py
++++ b/Cython/Compiler/Version.py
+@@ -6,4 +6,5 @@ from .. import __version__ as version
+ 
+ # For 'generated by' header line in C files.
+ 
+-watermark = str(version)
++# GraalPy change: watermark our patched version
++watermark = f'{version}-graalpy'
 diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd
 index 3dce395..77f8356 100644
 --- a/Cython/Includes/cpython/datetime.pxd

graalpython/lib-graalpython/patches/jaxlib/jaxlib-0.4.13.patch

@@ -1,9 +1,9 @@
 diff --git a/jaxlib_build_backend.py b/jaxlib_build_backend.py
 new file mode 100644
-index 000000000..5e6e553d2
+index 0000000..3727f7e
 --- /dev/null
 +++ b/jaxlib_build_backend.py
-@@ -0,0 +1,34 @@
+@@ -0,0 +1,39 @@
 +import os
 +import re
 +import sys
@@ -32,21 +32,26 @@ index 000000000..5e6e553d2
 +
 +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
 +    with tempfile.TemporaryDirectory() as d:
-+        subprocess.run([sys.executable, 'build/build.py', '--output_path', d], check=True)
++        cmd = [sys.executable, 'build/build.py', '--output_path', d]
++        if num_jobs := os.environ.get('CORE_COUNT'):
++            cmd += [f'--bazel_options=--jobs={num_jobs}']
++        if max_mem := os.environ.get('MAX_MEMORY_GB'):
++            cmd += [f'--bazel_options=--local_ram_resources={int(max_mem) * 1024}']
++        subprocess.run(cmd, check=True)
 +        wheels = list(Path(d).glob('*.whl'))
 +        assert len(wheels) == 1, f"Expected 1 wheel, found {len(wheels)}"
 +        wheel = wheels[0]
 +        shutil.copyfile(wheel, Path(wheel_directory) / wheel.name)
 +        return str(wheel.name)
 diff --git a/pyproject.toml b/pyproject.toml
-index eb3eb4c1f..f521d3f36 100644
+index eb3eb4c..5876c7b 100644
 --- a/pyproject.toml
 +++ b/pyproject.toml
 @@ -1,6 +1,7 @@
  [build-system]
 -requires = ["setuptools", "wheel"]
 -build-backend = "setuptools.build_meta"
-+requires = ["setuptools", "wheel", "numpy"]
++requires = ["setuptools", "wheel", "oldest-supported-numpy"]
 +build-backend = "jaxlib_build_backend"
 +backend-path = ["."]
 

graalpython/lib-graalpython/patches/tokenizers/tokenizers-0.13.3.patch

@@ -22,3 +22,29 @@ index 6282c31..47e6b12 100644
 
  [features]
  default = ["pyo3/extension-module"]
+diff --git a/tokenizers-lib/src/models/bpe/trainer.rs b/tokenizers-lib/src/models/bpe/trainer.rs
+index 43ab848..55f95f8 100644
+--- a/tokenizers-lib/src/models/bpe/trainer.rs
++++ b/tokenizers-lib/src/models/bpe/trainer.rs
+@@ -518,15 +518,16 @@ impl BpeTrainer {
+             let changes = top
+                 .pos
+                 .maybe_par_iter()
+-                .flat_map(|i| {
+-                    let w = &words[*i] as *const _ as *mut _;
++                .flat_map(|&i| {
++                    let word = &words[i] as *const _ as *mut Word;
+                     // We can merge each of these words in parallel here because each position
+                     // can be there only once (HashSet). So this is safe.
+                     unsafe {
+-                        let word: &mut Word = &mut (*w);
+-                        word.merge(top.pair.0, top.pair.1, new_token_id)
++                        // let word: &mut Word = &mut (*word);
++                        (*word)
++                            .merge(top.pair.0, top.pair.1, new_token_id)
+                             .into_iter()
+-                            .map(|c| (c, *i))
++                            .map(|c| (c, i))
+                             .collect::<Vec<_>>()
+                     }
+                 })