diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d50ec34e54d710..70dd342ab66a47 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -645,6 +645,27 @@ are set. .. versionadded:: 3.14 +To highlight inline code in your description or epilog text, you can use +backticks:: + + >>> parser = argparse.ArgumentParser( + ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... epilog='''Examples: + ... `python -m myapp --verbose` + ... `python -m myapp --config settings.json` + ... ''') + +When colors are enabled, the text inside backticks will be displayed in a +distinct color to help examples stand out. When colors are disabled, backticks +are preserved as-is, which is readable in plain text. + +.. note:: + + Backtick markup only applies to description and epilog text. It does not + apply to individual argument ``help`` strings. + +.. versionadded:: 3.15 + The add_argument() method ------------------------- diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index c5ea78c1683761..08cb1770d7a22e 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -210,6 +210,12 @@ Functions :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks a :class:`~importlib.machinery.ModuleSpec`. + .. versionchanged:: next + If *module* is a lazy module that has not yet been materialized (i.e., + loaded via :class:`importlib.util.LazyLoader` and not yet accessed), + calling :func:`reload` is a no-op and returns the module unchanged. + This prevents the reload from unintentionally triggering the lazy load. + .. warning:: This function is not thread-safe. Calling it from multiple threads can result in unexpected behavior. It's recommended to use the :class:`threading.Lock` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 58f3a8ecd7c2e0..4e9946fd4567cd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2869,8 +2869,8 @@ ABCs and Protocols for working with I/O --------------------------------------- .. class:: IO[AnyStr] - TextIO[AnyStr] - BinaryIO[AnyStr] + TextIO + BinaryIO Generic class ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` and ``BinaryIO(IO[bytes])`` diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1d8b838cd2e7f0..afefcc15ab2d55 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -423,6 +423,10 @@ argparse default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) +* Added backtick markup support in description and epilog text to highlight + inline code when color output is enabled. + (Contributed by Savannah Ostrowski in :gh:`142390`.) + calendar -------- diff --git a/Lib/argparse.py b/Lib/argparse.py index ee7ebc4696a0f7..633fec69ea4615 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -517,7 +517,27 @@ def _format_text(self, text): text = text % dict(prog=self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' + text = self._fill_text(text, text_width, indent) + text = self._apply_text_markup(text) + return text + '\n\n' + + def _apply_text_markup(self, text): + """Apply color markup to text. + + Supported markup: + `...` - inline code (rendered with prog_extra color) + + When colors are disabled, backticks are preserved as-is. + """ + t = self._theme + if not t.reset: + return text + text = _re.sub( + r'`([^`]+)`', + rf'{t.prog_extra}\1{t.reset}', + text, + ) + return text def _format_action(self, action): # determine the required width and the entry label diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index a7d57561ead046..694fea806f7944 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -97,6 +97,11 @@ def reload(module): The module must have been successfully imported before. """ + # If a LazyModule has not yet been materialized, reload is a no-op. + if importlib_util := sys.modules.get('importlib.util'): + if lazy_module_type := getattr(importlib_util, '_LazyModule', None): + if isinstance(module, lazy_module_type): + return module try: name = module.__spec__.name except AttributeError: diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index 2940f263f7ff29..03eb2274d23e68 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -275,16 +275,8 @@ body.resizing-sidebar { } /* View Mode Section */ -.view-mode-section { - padding-bottom: 20px; - border-bottom: 1px solid var(--border); -} - -.view-mode-section .section-title { - margin-bottom: 12px; -} - -.view-mode-section .toggle-switch { +.view-mode-section .section-content { + display: flex; justify-content: center; } @@ -316,7 +308,7 @@ body.resizing-sidebar { } .section-content { - transition: max-height var(--transition-normal), opacity var(--transition-normal); + transition: max-height var(--transition-slow) ease-out, opacity var(--transition-normal) ease-out, padding var(--transition-normal) ease-out; max-height: 1000px; opacity: 1; } @@ -324,7 +316,9 @@ body.resizing-sidebar { .collapsible.collapsed .section-content { max-height: 0; opacity: 0; - margin-bottom: -10px; + padding-top: 0; + pointer-events: none; + transition: max-height var(--transition-slow) ease-in, opacity var(--transition-normal) ease-in, padding var(--transition-normal) ease-in; } /* -------------------------------------------------------------------------- @@ -634,10 +628,6 @@ body.resizing-sidebar { Legend -------------------------------------------------------------------------- */ -.legend-section { - margin-top: auto; - padding-top: 12px; -} .legend { display: flex; @@ -1023,3 +1013,7 @@ body.resizing-sidebar { border-color: #8e44ad; box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); } + +.toggle-switch:focus-visible { + border-radius: 4px; +} diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index 6345320bd2555d..17fd95af859587 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -1302,6 +1302,17 @@ function initFlamegraph() { } } +// Keyboard shortcut: Enter/Space activates toggle switches +document.addEventListener('keydown', function(e) { + if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') { + return; + } + if ((e.key === 'Enter' || e.key === ' ') && e.target.classList.contains('toggle-switch')) { + e.preventDefault(); + e.target.click(); + } +}); + if (document.readyState === "loading") { document.addEventListener("DOMContentLoaded", initFlamegraph); } else { diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index 02855563f83f7c..936c9adfc8c519 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -102,12 +102,19 @@ -
-

View Mode

-
- Flamegraph -
- Inverted Flamegraph + diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css index d51636a3bf7d61..39bdd52e943ac7 100644 --- a/Lib/profiling/sampling/_shared_assets/base.css +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -37,6 +37,7 @@ /* Transitions */ --transition-fast: 0.15s ease; --transition-normal: 0.25s ease; + --transition-slow: 0.3s ease; } /* Light theme (default) */ diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 0f93e8ea740770..758af98d5cb046 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7568,6 +7568,101 @@ def test_error_and_warning_not_colorized_when_disabled(self): self.assertNotIn('\x1b[', warn) self.assertIn('warning:', warn) + def test_backtick_markup_in_epilog(self): + parser = argparse.ArgumentParser( + prog='PROG', + color=True, + epilog='Example: `python -m myapp --verbose`', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f'Example: {prog_extra}python -m myapp --verbose{reset}', + help_text) + self.assertNotIn('`', help_text) + + def test_backtick_markup_in_description(self): + parser = argparse.ArgumentParser( + prog='PROG', + color=True, + description='Run `python -m myapp` to start.', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f'Run {prog_extra}python -m myapp{reset} to start.', + help_text) + + def test_backtick_markup_multiple(self): + parser = argparse.ArgumentParser( + prog='PROG', + color=True, + epilog='Try `app run` or `app test`.', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f'{prog_extra}app run{reset}', help_text) + self.assertIn(f'{prog_extra}app test{reset}', help_text) + + def test_backtick_markup_not_applied_when_color_disabled(self): + # When color is disabled, backticks are preserved as-is + parser = argparse.ArgumentParser( + prog='PROG', + color=False, + epilog='Example: `python -m myapp`', + ) + + help_text = parser.format_help() + self.assertIn('`python -m myapp`', help_text) + self.assertNotIn('\x1b[', help_text) + + def test_backtick_markup_with_format_string(self): + parser = argparse.ArgumentParser( + prog='myapp', + color=True, + epilog='Run `%(prog)s --help` for more info.', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f'{prog_extra}myapp --help{reset}', help_text) + + def test_backtick_markup_in_subparser(self): + parser = argparse.ArgumentParser(prog='PROG', color=True) + subparsers = parser.add_subparsers() + sub = subparsers.add_parser( + 'sub', + description='Run `PROG sub --foo` to start.', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = sub.format_help() + self.assertIn(f'{prog_extra}PROG sub --foo{reset}', help_text) + + def test_backtick_markup_special_regex_chars(self): + parser = argparse.ArgumentParser( + prog='PROG', + color=True, + epilog='`grep "foo.*bar" | sort`', + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f'{prog_extra}grep "foo.*bar" | sort{reset}', help_text) + def test_print_help_uses_target_file_for_color_decision(self): parser = argparse.ArgumentParser(prog='PROG', color=True) parser.add_argument('--opt') diff --git a/Lib/test/test_free_threading/test_zlib.py b/Lib/test/test_free_threading/test_zlib.py new file mode 100644 index 00000000000000..7c4ed04f4a7c9e --- /dev/null +++ b/Lib/test/test_free_threading/test_zlib.py @@ -0,0 +1,80 @@ +import itertools +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +zlib = import_helper.import_module("zlib") + +from test.test_zlib import HAMLET_SCENE + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestZlib(unittest.TestCase): + def test_compressor(self): + comp = zlib.compressobj() + + # First compress() outputs zlib header + header = comp.compress(HAMLET_SCENE) + self.assertGreater(len(header), 0) + + def worker(): + # it should return empty bytes as it buffers data internally + data = comp.compress(HAMLET_SCENE) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS - 1) + full_compressed = header + comp.flush() + decompressed = zlib.decompress(full_compressed) + # The decompressed data should be HAMLET_SCENE repeated NTHREADS times + self.assertEqual(decompressed, HAMLET_SCENE * NTHREADS) + + def test_decompressor_concurrent_attribute_reads(self): + input_data = HAMLET_SCENE * NTHREADS + compressed = zlib.compress(input_data) + + decomp = zlib.decompressobj() + decomp_size_per_loop = len(input_data) // 1000 + decompressed_parts = [] + + def decomp_worker(): + # Decompress in chunks, which updates eof, unused_data, unconsumed_tail + decompressed_parts.append( + decomp.decompress(compressed, decomp_size_per_loop) + ) + while decomp.unconsumed_tail: + decompressed_parts.append( + decomp.decompress( + decomp.unconsumed_tail, decomp_size_per_loop + ) + ) + + def decomp_attr_reader(): + # Read attributes concurrently while another thread decompresses + for _ in range(1000): + _ = decomp.unused_data + _ = decomp.unconsumed_tail + _ = decomp.eof + + counter = itertools.count() + + def worker(): + # First thread decompresses, others read attributes + if next(counter) == 0: + decomp_worker() + else: + decomp_attr_reader() + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + self.assertTrue(decomp.eof) + self.assertEqual(decomp.unused_data, b"") + decompressed = b"".join(decompressed_parts) + self.assertEqual(decompressed, HAMLET_SCENE * NTHREADS) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index e48fad8898f0ef..c6b26ad75b97f9 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -10,6 +10,9 @@ from test.support import threading_helper from test.test_importlib import util as test_util +# Make sure sys.modules[util] is in sync with the import. +# That is needed as other tests may reload util. +sys.modules['importlib.util'] = util class CollectInit: @@ -192,7 +195,7 @@ def test_lazy_self_referential_modules(self): sys.modules['json'] = module loader.exec_module(module) - # Trigger load with attribute lookup, ensure expected behavior + # Trigger load with attribute lookup, ensure expected behavior. test_load = module.loads('{}') self.assertEqual(test_load, {}) @@ -224,6 +227,26 @@ def __delattr__(self, name): with self.assertRaises(AttributeError): del module.CONSTANT + def test_reload(self): + # Reloading a lazy module that hasn't been materialized is a no-op. + module = self.new_module() + sys.modules[TestingImporter.module_name] = module + + # Change the source code to add a new attribute. + TestingImporter.source_code = 'attr = 42\nnew_attr = 123\n__name__ = {!r}'.format(TestingImporter.mutated_name) + self.assertIsInstance(module, util._LazyModule) + + # Reload the module (should be a no-op since not materialized). + reloaded = importlib.reload(module) + self.assertIs(reloaded, module) + self.assertIsInstance(module, util._LazyModule) + + # Access the new attribute (should trigger materialization, and new_attr should exist). + self.assertEqual(module.attr, 42) + self.assertNotIsInstance(module, util._LazyModule) + self.assertTrue(hasattr(module, 'new_attr')) + self.assertEqual(module.new_attr, 123) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-14-14-40.gh-issue-116738.x7aaBF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-14-14-40.gh-issue-116738.x7aaBF.rst new file mode 100644 index 00000000000000..dcf4d0efa6f293 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-08-14-14-40.gh-issue-116738.x7aaBF.rst @@ -0,0 +1,2 @@ +Make the attributes in :mod:`zlib` thread-safe on the :term:`free threaded +` build. diff --git a/Misc/NEWS.d/next/Library/2025-10-09-15-46-18.gh-issue-139686.XwIZB2.rst b/Misc/NEWS.d/next/Library/2025-10-09-15-46-18.gh-issue-139686.XwIZB2.rst new file mode 100644 index 00000000000000..00dd3447eebb9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-09-15-46-18.gh-issue-139686.XwIZB2.rst @@ -0,0 +1 @@ +Make importlib.reload no-op for lazy modules. diff --git a/Misc/NEWS.d/next/Library/2025-12-07-22-13-28.gh-issue-142389.J9v904.rst b/Misc/NEWS.d/next/Library/2025-12-07-22-13-28.gh-issue-142389.J9v904.rst new file mode 100644 index 00000000000000..44e9cea7b54a8f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-07-22-13-28.gh-issue-142389.J9v904.rst @@ -0,0 +1 @@ +Add backtick markup support in :mod:`argparse` description and epilog text to highlight inline code when color output is enabled. diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 6bac09aa6c2a6c..f546f3ff1cb864 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -8,6 +8,7 @@ #endif #include "Python.h" +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_CHAR_RELAXED #include "zlib.h" #include "stdbool.h" @@ -181,15 +182,6 @@ OutputBuffer_WindowOnError(_BlocksOutputBuffer *buffer, _Uint32Window *window) } -#define ENTER_ZLIB(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define LEAVE_ZLIB(obj) PyThread_release_lock((obj)->lock); - - /* The following parameters are copied from zutil.h, version 0.95 */ #define DEFLATED 8 #if MAX_MEM_LEVEL >= 8 @@ -228,7 +220,7 @@ typedef struct char eof; bool is_initialised; PyObject *zdict; - PyThread_type_lock lock; + PyMutex mutex; } compobject; #define _compobject_CAST(op) ((compobject *)op) @@ -291,12 +283,7 @@ newcompobject(PyTypeObject *type) Py_DECREF(self); return NULL; } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; return self; } @@ -720,10 +707,10 @@ compobject_dealloc_impl(PyObject *op, int (*dealloc)(z_streamp)) PyTypeObject *type = Py_TYPE(op); PyObject_GC_UnTrack(op); compobject *self = _compobject_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); if (self->is_initialised) { (void)dealloc(&self->zst); } - PyThread_free_lock(self->lock); Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); Py_XDECREF(self->zdict); @@ -777,7 +764,7 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, _BlocksOutputBuffer buffer = {.writer = NULL}; zlibstate *state = PyType_GetModuleState(cls); - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); self->zst.next_in = data->buf; Py_ssize_t ibuflen = data->len; @@ -819,7 +806,7 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, OutputBuffer_OnError(&buffer); return_value = NULL; success: - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return return_value; } @@ -909,7 +896,7 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, max_length = -1; } - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); self->zst.next_in = data->buf; ibuflen = data->len; @@ -962,7 +949,7 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, if (err == Z_STREAM_END) { /* This is the logical place to call inflateEnd, but the old behaviour of only calling it on flush() is preserved. */ - self->eof = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(self->eof, 1); } else if (err != Z_OK && err != Z_BUF_ERROR) { /* We will only get Z_BUF_ERROR if the output buffer was full but there wasn't more output when we tried again, so it is @@ -981,7 +968,7 @@ zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, OutputBuffer_OnError(&buffer); return_value = NULL; success: - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return return_value; } @@ -1014,7 +1001,7 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); self->zst.avail_in = 0; @@ -1070,7 +1057,7 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) OutputBuffer_OnError(&buffer); return_value = NULL; success: - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return return_value; } @@ -1094,9 +1081,9 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls) if (!return_value) return NULL; /* Copy the zstream state - * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe + * We use mutex to make this thread-safe */ - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); int err = deflateCopy(&return_value->zst, &self->zst); switch (err) { case Z_OK: @@ -1120,11 +1107,11 @@ zlib_Compress_copy_impl(compobject *self, PyTypeObject *cls) /* Mark it as being initialized */ return_value->is_initialised = 1; - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return (PyObject *)return_value; error: - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); Py_XDECREF(return_value); return NULL; } @@ -1178,9 +1165,9 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls) if (!return_value) return NULL; /* Copy the zstream state - * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe + * We use mutex to make this thread-safe */ - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); int err = inflateCopy(&return_value->zst, &self->zst); switch (err) { case Z_OK: @@ -1205,11 +1192,11 @@ zlib_Decompress_copy_impl(compobject *self, PyTypeObject *cls) /* Mark it as being initialized */ return_value->is_initialised = 1; - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return (PyObject *)return_value; error: - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); Py_XDECREF(return_value); return NULL; } @@ -1282,10 +1269,10 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, return NULL; } - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); if (PyObject_GetBuffer(self->unconsumed_tail, &data, PyBUF_SIMPLE) == -1) { - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return NULL; } @@ -1333,7 +1320,7 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, /* If at end of stream, clean up any memory allocated by zlib. */ if (err == Z_STREAM_END) { - self->eof = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(self->eof, 1); self->is_initialised = 0; err = inflateEnd(&self->zst); if (err != Z_OK) { @@ -1352,7 +1339,7 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, return_value = NULL; success: PyBuffer_Release(&data); - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return return_value; } @@ -1361,7 +1348,7 @@ typedef struct { PyObject_HEAD z_stream zst; PyObject *zdict; - PyThread_type_lock lock; + PyMutex mutex; PyObject *unused_data; uint8_t *input_buffer; Py_ssize_t input_buffer_size; @@ -1387,7 +1374,7 @@ ZlibDecompressor_dealloc(PyObject *op) PyTypeObject *type = Py_TYPE(op); PyObject_GC_UnTrack(op); ZlibDecompressor *self = ZlibDecompressor_CAST(op); - PyThread_free_lock(self->lock); + assert(!PyMutex_IsLocked(&self->mutex)); if (self->is_initialised) { inflateEnd(&self->zst); } @@ -1545,7 +1532,7 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length) } while(err != Z_STREAM_END && self->avail_in_real != 0); if (err == Z_STREAM_END) { - self->eof = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(self->eof, 1); self->is_initialised = 0; /* Unlike the Decompress object we call inflateEnd here as there are no backwards compatibility issues */ @@ -1633,7 +1620,7 @@ decompress(ZlibDecompressor *self, uint8_t *data, } if (self->eof) { - self->needs_input = 0; + FT_ATOMIC_STORE_CHAR_RELAXED(self->needs_input, 0); if (self->avail_in_real > 0) { PyObject *unused_data = PyBytes_FromStringAndSize( @@ -1646,10 +1633,10 @@ decompress(ZlibDecompressor *self, uint8_t *data, } else if (self->avail_in_real == 0) { self->zst.next_in = NULL; - self->needs_input = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(self->needs_input, 1); } else { - self->needs_input = 0; + FT_ATOMIC_STORE_CHAR_RELAXED(self->needs_input, 0); /* If we did not use the input buffer, we now have to copy the tail from the caller's buffer into the @@ -1718,14 +1705,14 @@ zlib__ZlibDecompressor_decompress_impl(ZlibDecompressor *self, { PyObject *result = NULL; - ENTER_ZLIB(self); + PyMutex_Lock(&self->mutex); if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); } else { result = decompress(self, data->buf, data->len, max_length); } - LEAVE_ZLIB(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -1767,12 +1754,7 @@ zlib__ZlibDecompressor_impl(PyTypeObject *type, int wbits, PyObject *zdict) self->zst.next_in = NULL; self->zst.avail_in = 0; self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; int err = inflateInit2(&(self->zst), wbits); switch (err) { case Z_OK: @@ -1827,10 +1809,36 @@ static PyMethodDef ZlibDecompressor_methods[] = { {NULL} }; +static PyObject * +Decomp_unused_data_get(PyObject *op, void *Py_UNUSED(ignored)) +{ + compobject *self = _compobject_CAST(op); + PyMutex_Lock(&self->mutex); + assert(self->unused_data != NULL); + PyObject *result = Py_NewRef(self->unused_data); + PyMutex_Unlock(&self->mutex); + return result; +} + +static PyObject * +Decomp_unconsumed_tail_get(PyObject *op, void *Py_UNUSED(ignored)) +{ + compobject *self = _compobject_CAST(op); + PyMutex_Lock(&self->mutex); + assert(self->unconsumed_tail != NULL); + PyObject *result = Py_NewRef(self->unconsumed_tail); + PyMutex_Unlock(&self->mutex); + return result; +} + +static PyGetSetDef Decomp_getset[] = { + {"unused_data", Decomp_unused_data_get, NULL, NULL}, + {"unconsumed_tail", Decomp_unconsumed_tail_get, NULL, NULL}, + {NULL}, +}; + #define COMP_OFF(x) offsetof(compobject, x) static PyMemberDef Decomp_members[] = { - {"unused_data", _Py_T_OBJECT, COMP_OFF(unused_data), Py_READONLY}, - {"unconsumed_tail", _Py_T_OBJECT, COMP_OFF(unconsumed_tail), Py_READONLY}, {"eof", Py_T_BOOL, COMP_OFF(eof), Py_READONLY}, {NULL}, }; @@ -1844,11 +1852,26 @@ PyDoc_STRVAR(ZlibDecompressor_unused_data__doc__, PyDoc_STRVAR(ZlibDecompressor_needs_input_doc, "True if more input is needed before more decompressed data can be produced."); +static PyObject * +ZlibDecompressor_unused_data_get(PyObject *op, void *Py_UNUSED(ignored)) +{ + ZlibDecompressor *self = ZlibDecompressor_CAST(op); + PyMutex_Lock(&self->mutex); + assert(self->unused_data != NULL); + PyObject *result = Py_NewRef(self->unused_data); + PyMutex_Unlock(&self->mutex); + return result; +} + +static PyGetSetDef ZlibDecompressor_getset[] = { + {"unused_data", ZlibDecompressor_unused_data_get, NULL, + ZlibDecompressor_unused_data__doc__}, + {NULL}, +}; + static PyMemberDef ZlibDecompressor_members[] = { {"eof", Py_T_BOOL, offsetof(ZlibDecompressor, eof), Py_READONLY, ZlibDecompressor_eof__doc__}, - {"unused_data", Py_T_OBJECT_EX, offsetof(ZlibDecompressor, unused_data), - Py_READONLY, ZlibDecompressor_unused_data__doc__}, {"needs_input", Py_T_BOOL, offsetof(ZlibDecompressor, needs_input), Py_READONLY, ZlibDecompressor_needs_input_doc}, {NULL}, @@ -2074,6 +2097,7 @@ static PyType_Slot Decomptype_slots[] = { {Py_tp_traverse, compobject_traverse}, {Py_tp_methods, Decomp_methods}, {Py_tp_members, Decomp_members}, + {Py_tp_getset, Decomp_getset}, {0, 0}, }; @@ -2093,6 +2117,7 @@ static PyType_Slot ZlibDecompressor_type_slots[] = { {Py_tp_dealloc, ZlibDecompressor_dealloc}, {Py_tp_traverse, ZlibDecompressor_traverse}, {Py_tp_members, ZlibDecompressor_members}, + {Py_tp_getset, ZlibDecompressor_getset}, {Py_tp_new, zlib__ZlibDecompressor}, {Py_tp_doc, (char *)zlib__ZlibDecompressor__doc__}, {Py_tp_methods, ZlibDecompressor_methods}, diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index edf14ce578bce6..27fbc311bbc1d6 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,6 +3,7 @@ import argparse import os import pathlib +import platform import sys import tarfile import time @@ -44,20 +45,38 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): def fetch_release(tag, tarball_dir, *, org='python', verbose=False): - url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{tag}.tar.xz' + arch = os.environ.get('PreferredToolArchitecture') + if not arch: + machine = platform.machine() + arch = 'ARM64' if machine == 'ARM64' else 'AMD64' + elif arch.lower() in ('x86', 'x64'): + arch = 'AMD64' reporthook = None if verbose: reporthook = print tarball_dir.mkdir(parents=True, exist_ok=True) - output_path = tarball_dir / f'{tag}.tar.xz' - retrieve_with_retries(url, output_path, reporthook) + + arch_filename = f'{tag}-{arch}.tar.xz' + arch_url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{arch_filename}' + try: + output_path = tarball_dir / arch_filename + retrieve_with_retries(arch_url, output_path, reporthook) + return output_path + except OSError: + if verbose: + print(f'{arch_filename} not found, trying generic binary...') + + generic_filename = f'{tag}.tar.xz' + generic_url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{generic_filename}' + output_path = tarball_dir / generic_filename + retrieve_with_retries(generic_url, output_path, reporthook) return output_path def extract_tarball(externals_dir, tarball_path, tag): output_path = externals_dir / tag with tarfile.open(tarball_path) as tf: - tf.extractall(os.fspath(externals_dir)) + tf.extractall(os.fspath(externals_dir), filter='data') return output_path diff --git a/Tools/jit/README.md b/Tools/jit/README.md index c70c0c47d94ad2..8eadb3349ba6da 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -43,6 +43,15 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. +By default, the architecture of the LLVM tools is auto-detected based on the host machine. To override this, set the `PreferredToolArchitecture` environment variable before building: + +```sh +set PreferredToolArchitecture=AMD64 +PCbuild\build.bat --experimental-jit +``` + +Valid values are`x64`, `x86` and `ARM64`. + Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md index 1a954755611d19..0b851c67d65c83 100644 --- a/Tools/jit/jit_infra.md +++ b/Tools/jit/jit_infra.md @@ -8,21 +8,28 @@ When we update LLVM, we need to also update the LLVM release artifact for Window To update the LLVM release artifact for Windows builds, follow these steps: 1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). -1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). -1. Extract and repackage the tarball with the correct directory structure. For example: +1. Download Windows artifacts for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz` and `clang+llvm-21.1.4-aarch64-pc-windows-msvc.tar.xz`). +1. Extract and repackage each tarball with the correct directory structure. For example: ```bash + # For x86_64 (AMD64) tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 - tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0-x64.tar.xz + rm -rf llvm-21.1.4.0 + + # For ARM64 + tar -xf clang+llvm-21.1.4-aarch64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-aarch64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0-ARM64.tar.xz ``` - The tarball must contain a top-level directory named `llvm-{version}.0/`. + Each tarball must contain a top-level directory named `llvm-{version}.0/`. 1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). -1. Create a new release with the updated LLVM artifact. +1. Create a new release with the LLVM artifacts. - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). - - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). - - Upload the asset (you can leave all other fields the same). + - Specify the release title (e.g. `LLVM 21.1.4`). + - Upload both platform-specific assets to the same release. ### Other notes - You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. -- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- The artifact filename must include the architecture suffix (e.g. `llvm-21.1.4.0-x64.tar.xz`, `llvm-21.1.4.0-ARM64.tar.xz`). - You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file