Use the C locale when passing file paths to libtcod

Fixes encoding errors which prevented existing files from loading correctly.

Author: HexDecimal

CHANGELOG.md

@@ -4,6 +4,8 @@ Changes relevant to the users of python-tcod are documented here.
 This project adheres to [Semantic Versioning](https://semver.org/) since version `2.0.0`.
 
 ## [Unreleased]
+### Fixed
+- Fixed errors loading files where their paths are non-ASCII and the C locale is not UTF-8.
 
 ## [16.2.0] - 2023-09-20
 ### Changed

tcod/_internal.py

@@ -2,7 +2,10 @@
 from __future__ import annotations
 
 import functools
+import locale
+import sys
 import warnings
+from pathlib import Path
 from types import TracebackType
 from typing import Any, AnyStr, Callable, NoReturn, SupportsInt, TypeVar, cast
 
@@ -126,6 +129,16 @@ def _fmt(string: str, stacklevel: int = 2) -> bytes:
     return string.encode("utf-8").replace(b"%", b"%%")
 
 
+def _path_encode(path: Path) -> bytes:
+    """Return a bytes file path for the current C locale."""
+    try:
+        return str(path).encode(locale.getlocale()[1] or "utf-8")
+    except UnicodeEncodeError as exc:
+        if sys.version_info >= (3, 11):
+            exc.add_note("""Consider calling 'locale.setlocale(locale.LC_CTYPES, ".UTF8")' to support Unicode paths.""")
+        raise
+
+
 class _PropagateException:
     """Context manager designed to propagate exceptions outside of a cffi callback context.
 

tcod/console.py

@@ -17,7 +17,7 @@
 
 import tcod._internal
 import tcod.constants
-from tcod._internal import _check, deprecate
+from tcod._internal import _check, _path_encode, deprecate
 from tcod.cffi import ffi, lib
 
 
@@ -1301,9 +1301,9 @@ def load_xp(path: str | PathLike[str], order: Literal["C", "F"] = "C") -> tuple[
         console.rgba[is_transparent] = (ord(" "), (0,), (0,))
     """
     path = Path(path).resolve(strict=True)
-    layers = _check(tcod.lib.TCOD_load_xp(bytes(path), 0, ffi.NULL))
+    layers = _check(tcod.lib.TCOD_load_xp(_path_encode(path), 0, ffi.NULL))
     consoles = ffi.new("TCOD_Console*[]", layers)
-    _check(tcod.lib.TCOD_load_xp(bytes(path), layers, consoles))
+    _check(tcod.lib.TCOD_load_xp(_path_encode(path), layers, consoles))
     return tuple(Console._from_cdata(console_p, order=order) for console_p in consoles)
 
 
@@ -1364,7 +1364,7 @@ def save_xp(
         tcod.lib.TCOD_save_xp(
             len(consoles_c),
             consoles_c,
-            bytes(path),
+            _path_encode(path),
             compress_level,
         )
     )

tcod/image.py

@@ -18,7 +18,7 @@
 from numpy.typing import ArrayLike, NDArray
 
 import tcod.console
-from tcod._internal import _console, deprecate
+from tcod._internal import _console, _path_encode, deprecate
 from tcod.cffi import ffi, lib
 
 
@@ -72,7 +72,7 @@ def from_file(cls, path: str | PathLike[str]) -> Image:
         .. versionadded:: 16.0
         """
         path = Path(path).resolve(strict=True)
-        return cls._from_cdata(ffi.gc(lib.TCOD_image_load(bytes(path)), lib.TCOD_image_delete))
+        return cls._from_cdata(ffi.gc(lib.TCOD_image_load(_path_encode(path)), lib.TCOD_image_delete))
 
     def clear(self, color: tuple[int, int, int]) -> None:
         """Fill this entire Image with color.
@@ -306,7 +306,7 @@ def save_as(self, filename: str | PathLike[str]) -> None:
         .. versionchanged:: 16.0
             Added PathLike support.
         """
-        lib.TCOD_image_save(self.image_c, bytes(Path(filename)))
+        lib.TCOD_image_save(self.image_c, _path_encode(Path(filename)))
 
     @property
     def __array_interface__(self) -> dict[str, Any]:
@@ -364,7 +364,7 @@ def load(filename: str | PathLike[str]) -> NDArray[np.uint8]:
 
     .. versionadded:: 11.4
     """
-    image = Image._from_cdata(ffi.gc(lib.TCOD_image_load(bytes(Path(filename))), lib.TCOD_image_delete))
+    image = Image._from_cdata(ffi.gc(lib.TCOD_image_load(_path_encode(Path(filename))), lib.TCOD_image_delete))
     array: NDArray[np.uint8] = np.asarray(image, dtype=np.uint8)
     height, width, depth = array.shape
     if depth == 3:

tcod/libtcodpy.py

@@ -31,6 +31,7 @@
     _console,
     _fmt,
     _int,
+    _path_encode,
     _PropagateException,
     _unicode,
     _unpack_char_p,
@@ -991,7 +992,7 @@ def console_set_custom_font(
         Added PathLike support.  `fontFile` no longer takes bytes.
     """
     fontFile = Path(fontFile).resolve(strict=True)
-    _check(lib.TCOD_console_set_custom_font(bytes(fontFile), flags, nb_char_horiz, nb_char_vertic))
+    _check(lib.TCOD_console_set_custom_font(_path_encode(fontFile), flags, nb_char_horiz, nb_char_vertic))
 
 
 @deprecate("Check `con.width` instead.")
@@ -1806,7 +1807,7 @@ def console_from_file(filename: str | PathLike[str]) -> tcod.console.Console:
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    return tcod.console.Console._from_cdata(_check_p(lib.TCOD_console_from_file(bytes(filename))))
+    return tcod.console.Console._from_cdata(_check_p(lib.TCOD_console_from_file(_path_encode(filename))))
 
 
 @deprecate("Call the `Console.blit` method instead.")
@@ -1985,7 +1986,7 @@ def console_load_asc(con: tcod.console.Console, filename: str | PathLike[str]) -
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    return bool(lib.TCOD_console_load_asc(_console(con), bytes(filename)))
+    return bool(lib.TCOD_console_load_asc(_console(con), _path_encode(filename)))
 
 
 @deprecate("This format is not actively supported")
@@ -1998,7 +1999,7 @@ def console_save_asc(con: tcod.console.Console, filename: str | PathLike[str]) -
     .. versionchanged:: 16.0
         Added PathLike support.
     """
-    return bool(lib.TCOD_console_save_asc(_console(con), bytes(Path(filename))))
+    return bool(lib.TCOD_console_save_asc(_console(con), _path_encode(Path(filename))))
 
 
 @deprecate("This format is not actively supported")
@@ -2012,7 +2013,7 @@ def console_load_apf(con: tcod.console.Console, filename: str | PathLike[str]) -
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    return bool(lib.TCOD_console_load_apf(_console(con), bytes(filename)))
+    return bool(lib.TCOD_console_load_apf(_console(con), _path_encode(filename)))
 
 
 @deprecate("This format is not actively supported")
@@ -2025,7 +2026,7 @@ def console_save_apf(con: tcod.console.Console, filename: str | PathLike[str]) -
     .. versionchanged:: 16.0
         Added PathLike support.
     """
-    return bool(lib.TCOD_console_save_apf(_console(con), bytes(Path(filename))))
+    return bool(lib.TCOD_console_save_apf(_console(con), _path_encode(Path(filename))))
 
 
 @deprecate("Use tcod.console.load_xp to load this file.")
@@ -2040,7 +2041,7 @@ def console_load_xp(con: tcod.console.Console, filename: str | PathLike[str]) ->
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    return bool(lib.TCOD_console_load_xp(_console(con), bytes(filename)))
+    return bool(lib.TCOD_console_load_xp(_console(con), _path_encode(filename)))
 
 
 @deprecate("Use tcod.console.save_xp to save this console.")
@@ -2050,7 +2051,7 @@ def console_save_xp(con: tcod.console.Console, filename: str | PathLike[str], co
     .. versionchanged:: 16.0
         Added PathLike support.
     """
-    return bool(lib.TCOD_console_save_xp(_console(con), bytes(Path(filename)), compress_level))
+    return bool(lib.TCOD_console_save_xp(_console(con), _path_encode(Path(filename)), compress_level))
 
 
 @deprecate("Use tcod.console.load_xp to load this file.")
@@ -2061,7 +2062,7 @@ def console_from_xp(filename: str | PathLike[str]) -> tcod.console.Console:
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    return tcod.console.Console._from_cdata(_check_p(lib.TCOD_console_from_xp(bytes(filename))))
+    return tcod.console.Console._from_cdata(_check_p(lib.TCOD_console_from_xp(_path_encode(filename))))
 
 
 @deprecate("Use tcod.console.load_xp to load this file.")
@@ -2074,7 +2075,7 @@ def console_list_load_xp(
         Added PathLike support.
     """
     filename = Path(filename).resolve(strict=True)
-    tcod_list = lib.TCOD_console_list_from_xp(bytes(filename))
+    tcod_list = lib.TCOD_console_list_from_xp(_path_encode(filename))
     if tcod_list == ffi.NULL:
         return None
     try:
@@ -2102,7 +2103,7 @@ def console_list_save_xp(
     try:
         for console in console_list:
             lib.TCOD_list_push(tcod_list, _console(console))
-        return bool(lib.TCOD_console_list_save_xp(tcod_list, bytes(Path(filename)), compress_level))
+        return bool(lib.TCOD_console_list_save_xp(tcod_list, _path_encode(Path(filename)), compress_level))
     finally:
         lib.TCOD_list_delete(tcod_list)
 
@@ -3436,7 +3437,7 @@ def mouse_get_status() -> Mouse:
 
 @pending_deprecate()
 def namegen_parse(filename: str | PathLike[str], random: tcod.random.Random | None = None) -> None:
-    lib.TCOD_namegen_parse(bytes(Path(filename)), random or ffi.NULL)
+    lib.TCOD_namegen_parse(_path_encode(Path(filename)), random or ffi.NULL)
 
 
 @pending_deprecate()
@@ -3639,7 +3640,7 @@ def _pycall_parser_error(msg: Any) -> None:
 def parser_run(parser: Any, filename: str | PathLike[str], listener: Any = None) -> None:
     global _parser_listener
     if not listener:
-        lib.TCOD_parser_run(parser, bytes(Path(filename)), ffi.NULL)
+        lib.TCOD_parser_run(parser, _path_encode(Path(filename)), ffi.NULL)
         return
 
     propagate_manager = _PropagateException()
@@ -3658,7 +3659,7 @@ def parser_run(parser: Any, filename: str | PathLike[str], listener: Any = None)
     with _parser_callback_lock:
         _parser_listener = listener
         with propagate_manager:
-            lib.TCOD_parser_run(parser, bytes(Path(filename)), c_listener)
+            lib.TCOD_parser_run(parser, _path_encode(Path(filename)), c_listener)
 
 
 @deprecate("libtcod objects are deleted automatically.")
@@ -4079,7 +4080,7 @@ def sys_save_screenshot(name: str | PathLike[str] | None = None) -> None:
     .. versionchanged:: 16.0
         Added PathLike support.
     """
-    lib.TCOD_sys_save_screenshot(bytes(Path(name)) if name is not None else ffi.NULL)
+    lib.TCOD_sys_save_screenshot(_path_encode(Path(name)) if name is not None else ffi.NULL)
 
 
 # custom fullscreen resolution

tcod/tileset.py

@@ -21,7 +21,7 @@
 from numpy.typing import ArrayLike, NDArray
 
 import tcod.console
-from tcod._internal import _check, _console, _raise_tcod_error, deprecate
+from tcod._internal import _check, _console, _path_encode, _raise_tcod_error, deprecate
 from tcod.cffi import ffi, lib
 
 
@@ -268,7 +268,7 @@ def load_truetype_font(path: str | PathLike[str], tile_width: int, tile_height:
     This function is provisional.  The API may change.
     """
     path = Path(path).resolve(strict=True)
-    cdata = lib.TCOD_load_truetype_font_(bytes(path), tile_width, tile_height)
+    cdata = lib.TCOD_load_truetype_font_(_path_encode(path), tile_width, tile_height)
     if not cdata:
         raise RuntimeError(ffi.string(lib.TCOD_get_error()))
     return Tileset._claim(cdata)
@@ -296,7 +296,7 @@ def set_truetype_font(path: str | PathLike[str], tile_width: int, tile_height: i
         Use :any:`load_truetype_font` instead.
     """
     path = Path(path).resolve(strict=True)
-    if lib.TCOD_tileset_load_truetype_(bytes(path), tile_width, tile_height):
+    if lib.TCOD_tileset_load_truetype_(_path_encode(path), tile_width, tile_height):
         raise RuntimeError(ffi.string(lib.TCOD_get_error()))
 
 
@@ -314,7 +314,7 @@ def load_bdf(path: str | PathLike[str]) -> Tileset:
     .. versionadded:: 11.10
     """
     path = Path(path).resolve(strict=True)
-    cdata = lib.TCOD_load_bdf(bytes(path))
+    cdata = lib.TCOD_load_bdf(_path_encode(path))
     if not cdata:
         raise RuntimeError(ffi.string(lib.TCOD_get_error()).decode())
     return Tileset._claim(cdata)
@@ -343,7 +343,7 @@ def load_tilesheet(path: str | PathLike[str], columns: int, rows: int, charmap:
     mapping = []
     if charmap is not None:
         mapping = list(itertools.islice(charmap, columns * rows))
-    cdata = lib.TCOD_tileset_load(bytes(path), columns, rows, len(mapping), mapping)
+    cdata = lib.TCOD_tileset_load(_path_encode(path), columns, rows, len(mapping), mapping)
     if not cdata:
         _raise_tcod_error()
     return Tileset._claim(cdata)