[GR-60191][GR-60192] Refactor coverage gates, run on multiple patforms

PullRequest: graalpython/3597

Author: msimacek

CHANGELOG.md

@@ -16,7 +16,9 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Rewrite wheelbuilder to be easier to use and contribute to. This version is now the same we run internally to build publishable wheels for some platforms we support, so the community can build the same wheels on their own hardware easily if desired.
 * `pip` is now able to fetch newer versions of GraalPy patches for third-party packages from `graalpython` GitHub repository, allowing us to add new patches to released versions.
   * The patch repository can be overridden using `PIP_GRAALPY_PATCHES_URL` environment variable, which can point to a local path or a URL. It can be disabled by setting it to an empty string.
-* Added `GRAALPY_VERSION` and `GRAALPY_VERSION_NUM` C macros
+* Added `GRAALPY_VERSION` and `GRAALPY_VERSION_NUM` C macros.
+* Remove `ginstall` module. It hasn't been necessary for several releases. Please, use `pip install`.
+* Remove experimental `SetupLLVMLibraryPaths` option. It was used to pre-set library path for LLVM toolchain's libc++. The path can still be set manually.
 
 ## Version 24.1.0
 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "cdd68c4c8eaada96094af74927c487b5b8d95ec8" }
+{ "overlay": "85b7a69ee24cdbfc05245c449347ef4feec038a7" }

docs/contributor/IMPLEMENTATION_DETAILS.md

@@ -85,12 +85,11 @@ and holding on to the last (if any) in a local variable.
 Some PyPI packages contain code that is not compatible with GraalPy.
 To overcome this limitation and support such packages, GraalPy contains
 patches for some popular packages. The patches are applied to packages
-installed via GraalPy specific utility `ginstall` and also to packages
 installed via `pip`. This is achieved by patching `pip` code.
 
 The patches are regular POSIX `patch` command compatible diffs located in
-`lib-graalpython/patches`. Check out the directory structure and metadate.toml
-files to get an idea of how rules are set for patches to be applied.
+`lib-graalpython/patches`. The directory has a `README.md` file describing
+how the patches are applied.
 
 ## The GIL
 

graalpy_virtualenv/graalpy_virtualenv/graalpy.py

@@ -43,13 +43,13 @@
 from subprocess import check_output as subprocess_check_output
 
 from virtualenv.create.creator import Creator
-from virtualenv.create.describe import PosixSupports
+from virtualenv.create.describe import PosixSupports, WindowsSupports
 from virtualenv.seed.embed.pip_invoke import PipInvoke
 from virtualenv.seed.wheels import get_wheel
 from virtualenv.seed.wheels.bundle import from_dir
 
 
-class GraalPyCreatorPosix(Creator, PosixSupports):
+class AbstractGraalPyCreator(Creator):
     """
     Describe and fake Creator service for GraalPy.
 
@@ -115,13 +115,21 @@ def create(self):
                            "or environment variable VIRTUALENV_CREATOR=venv")
 
 
+class GraalPyCreatorPosix(AbstractGraalPyCreator, PosixSupports):
+    pass
+
+
+class GraalPyCreatorWindows(AbstractGraalPyCreator, WindowsSupports):
+    pass
+
+
 @lru_cache()
 def get_ensurepip_path(exe):
     if exe.samefile(sys.executable):
         import ensurepip
         ensurepip_path = ensurepip.__path__[0]
     else:
-        cmd = [exe, "-u", "-c", 'import ensurepip; print(ensurepip.__path__[0])']
+        cmd = [str(exe), "-u", "-c", 'import ensurepip; print(ensurepip.__path__[0])']
         ensurepip_path = subprocess_check_output(cmd, universal_newlines=True).strip()
     return Path(ensurepip_path) / '_bundled'
 

graalpy_virtualenv/pyproject.toml

@@ -59,6 +59,7 @@ dependencies = [
 
 [project.entry-points."virtualenv.create"]
 graalpy-posix = "graalpy_virtualenv.graalpy:GraalPyCreatorPosix"
+graalpy-windows = "graalpy_virtualenv.graalpy:GraalPyCreatorWindows"
 
 [project.entry-points."virtualenv.seed"]
 graalpy = "graalpy_virtualenv:graalpy.GraalPySeeder"

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -238,7 +238,11 @@ static void initialize_builtin_types_and_structs() {
 
 int mmap_getbuffer(PyObject *self, Py_buffer *view, int flags) {
 	// TODO(fa) readonly flag
-    return PyBuffer_FillInfo(view, (PyObject*)self, GraalPyTruffle_GetMMapData(self), PyObject_Size((PyObject *)self), 0, flags);
+    char* data = GraalPyTruffle_GetMMapData(self);
+    if (!data) {
+        return -1;
+    }
+    return PyBuffer_FillInfo(view, (PyObject*)self, data, PyObject_Size((PyObject *)self), 0, flags);
 }
 
 PyAPI_FUNC(void) mmap_init_bufferprotocol(PyObject* mmap_type) {

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -119,7 +119,6 @@ def add_graalpython_core():
         "java",
         "pip_hook",
         "unicodedata",
-        "sulong_support",
     ]:
         modname = f"graalpy.{os.path.basename(name)}"
         modpath = os.path.join(lib_graalpython, f"{name}.py")

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -762,7 +762,6 @@ protected void launch(Builder contextBuilder) {
         if (!noSite) {
             contextBuilder.option("python.ForceImportSite", "true");
         }
-        contextBuilder.option("python.SetupLLVMLibraryPaths", "true");
         contextBuilder.option("python.IgnoreEnvironmentFlag", Boolean.toString(ignoreEnv));
         contextBuilder.option("python.UnbufferedIO", Boolean.toString(unbufferedIO));
 

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -727,6 +727,8 @@ def run_in_subprocess_and_watch(self):
             port = server.getsockname()[1]
             assert port
 
+            retries = 3
+
             while self.remaining_test_ids and not self.stop_event.is_set():
                 last_remaining_count = len(self.remaining_test_ids)
                 with open(tmp_dir / 'out', 'w+') as self.out_file:
@@ -744,8 +746,17 @@ def run_in_subprocess_and_watch(self):
                         cmd.append('--failfast')
                     self.process = subprocess.Popen(cmd, stdout=self.out_file, stderr=self.out_file)
 
-                    server.settimeout(600.0)
-                    with server.accept()[0] as sock:
+                    server.settimeout(180.0)
+                    try:
+                        sock = server.accept()[0]
+                    except TimeoutError:
+                        interrupt_process(self.process)
+                        retries -= 1
+                        if retries:
+                            continue
+                        sys.exit("Worker failed to connect to runner multiple times")
+
+                    with sock:
                         conn = Connection(sock)
 
                         conn.send([TestSpecifier(t.test_file, t.test_name) for t in self.remaining_test_ids])

graalpython/com.oracle.graal.python.test/src/tests/conftest.toml

@@ -12,41 +12,20 @@ partial_splits_individual_tests = true
 # on Windows, yet, add their files here.
 exclude_on = ['win32']
 selector = [
-    "test_code.py", # forward slash in path problem
-    "test_csv.py",
-    "test_imports.py", # import posix
-    "test_locale.py",
-    "test_math.py",
-    "test_memoryview.py",
-    "test_mmap.py", # sys.getwindowsversion
-    "test_multiprocessing.py", # import _winapi
+    "test_mmap.py", # all tests fail with PermissionError, so does the mmap test in test_memoryview and cpyext/test_abstract
     "test_multiprocessing_graalpy.py", # import _winapi
     "test_patched_pip.py",
     "test_pathlib.py",
-    "test_pdb.py", # Tends to hit GR-41935
     "test_posix.py", # import posix
-    "test_pyio.py",
+    "test_pyio.py", # pyio imports msvcrt
     "test_signal.py",
-    "test_struct.py",
     "test_structseq.py", # import posix
-    "test_subprocess.py",
-    "test_thread.py", # sys.getwindowsversion
-    "test_traceback.py",
-    "test_zipimport.py", # sys.getwindowsversion
-    "test_ssl_java_integration.py",
-    "cpyext/test_abstract.py",
-    "cpyext/test_functions.py",
-    "cpyext/test_long.py",
-    "cpyext/test_member.py",
-    "cpyext/test_memoryview.py",
-    "cpyext/test_misc.py",
+    "cpyext/test_abstract.py", # mmap PermissionError
+    "cpyext/test_member.py", # Too many assumptions about long size
+    "cpyext/test_memoryview.py", # Failure in test_memoryview_read_0dim
     "cpyext/test_mmap.py",
-    "cpyext/test_slice.py",
-    "cpyext/test_shutdown.py",
-    "cpyext/test_thread.py",
-    "cpyext/test_unicode.py",
-    "cpyext/test_wiki.py",
-    "cpyext/test_tp_slots.py", # Temporarily disabled due to GR-54345
+    "cpyext/test_shutdown.py", # Uses gcc-specific attribute
+    "cpyext/test_wiki.py", # Assumptions about long size
 ]
 
 [[test_rules]]