diff --git a/dev/fsspec_inspector/generate_flavours.py b/dev/fsspec_inspector/generate_flavours.py index 324f007a..2d58e1c3 100644 --- a/dev/fsspec_inspector/generate_flavours.py +++ b/dev/fsspec_inspector/generate_flavours.py @@ -142,6 +142,15 @@ def _fix_azure_blob_file_system(x: str) -> str: return x +def _fix_data_file_system(x: str) -> str: + return re.sub( + "sep = '/'", + "sep = '' # type: ignore[assignment]\n " + "altsep = ' ' # type: ignore[assignment]", + x, + ) + + def _fix_memfs_file_system(x: str) -> str: return re.sub( "_MemFS", @@ -184,6 +193,7 @@ def _fix_xrootd_file_system(x: str) -> str: FIX_SOURCE = { "AbstractFileSystem": _fix_abstract_file_system, "AzureBlobFileSystem": _fix_azure_blob_file_system, + "DataFileSystem": _fix_data_file_system, "MemFS": _fix_memfs_file_system, "MemoryFileSystem": _fix_memory_file_system, "OSSFileSystem": _fix_oss_file_system, diff --git a/upath/_flavour.py b/upath/_flavour.py index 19eb62a3..3274bcb2 100644 --- a/upath/_flavour.py +++ b/upath/_flavour.py @@ -274,7 +274,7 @@ def sep(self) -> str: # type: ignore[override] @property def altsep(self) -> str | None: # type: ignore[override] - return None + return getattr(self._spec, "altsep", None) def isabs(self, path: JoinablePathLike) -> bool: path = self.strip_protocol(path) diff --git a/upath/_flavour_sources.py b/upath/_flavour_sources.py index 457551c0..32366a4e 100644 --- a/upath/_flavour_sources.py +++ b/upath/_flavour_sources.py @@ -341,7 +341,8 @@ class DataFileSystemFlavour(AbstractFileSystemFlavour): __orig_version__ = '2025.10.0' protocol = ('data',) root_marker = '' - sep = '/' + sep = "" # type: ignore[assignment] + altsep = " " # type: ignore[assignment] class DatabricksFileSystemFlavour(AbstractFileSystemFlavour): diff --git a/upath/core.py b/upath/core.py index a8a72669..90ac7fef 100644 --- a/upath/core.py +++ b/upath/core.py @@ -785,7 +785,7 @@ class UPath(_UPathMixin, WritablePath, ReadablePath): if TYPE_CHECKING: # noqa: C901 _chain: Chain _chain_parser: FSSpecChainParser - _fs_cached: bool + _fs_cached: AbstractFileSystem _raw_urlpaths: Sequence[JoinablePathLike] _relative_base: str | None @@ -965,6 +965,8 @@ def with_segments(self, *pathsegments: JoinablePathLike) -> Self: protocol=self._protocol, **self._storage_options, ) + if hasattr(self, "_fs_cached"): + new_instance._fs_cached = self._fs_cached if is_relative: new_instance._relative_base = self._relative_base @@ -1755,7 +1757,10 @@ def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: if exists and not exist_ok: raise FileExistsError(str(self)) if not exists: - self.fs.touch(self.path, truncate=True) + try: + self.fs.touch(self.path, truncate=True) + except NotImplementedError: + _raise_unsupported(type(self).__name__, "touch") else: try: self.fs.touch(self.path, truncate=False) diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index 3f037796..02c89db3 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -1,9 +1,11 @@ from __future__ import annotations import sys +from collections.abc import Iterator from typing import TYPE_CHECKING from typing import Any +from upath import UnsupportedOperation from upath._chain import DEFAULT_CHAIN_PARSER from upath._flavour import upath_strip_protocol from upath.core import UPath @@ -13,8 +15,10 @@ from typing import Literal if sys.version_info >= (3, 11): + from typing import Self from typing import Unpack else: + from typing_extensions import Self from typing_extensions import Unpack from upath._chain import FSSpecChainParser @@ -166,3 +170,35 @@ def __init__( super().__init__( *args, protocol=protocol, chain_parser=chain_parser, **storage_options ) + + def iterdir(self) -> Iterator[Self]: + try: + yield from super().iterdir() + except NotImplementedError: + raise UnsupportedOperation + + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + raise UnsupportedOperation + + def mkdir( + self, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise UnsupportedOperation + + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation + + def write_bytes(self, data: bytes) -> int: + raise UnsupportedOperation("DataPath does not support writing") + + def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + raise UnsupportedOperation("DataPath does not support writing") diff --git a/upath/implementations/data.py b/upath/implementations/data.py index 83dce046..0b8dfdc1 100644 --- a/upath/implementations/data.py +++ b/upath/implementations/data.py @@ -1,9 +1,12 @@ from __future__ import annotations import sys +from collections.abc import Iterator from collections.abc import Sequence from typing import TYPE_CHECKING +from urllib.parse import quote_plus +from upath._protocol import get_upath_protocol from upath.core import UnsupportedOperation from upath.core import UPath from upath.types import JoinablePathLike @@ -45,10 +48,49 @@ def __str__(self) -> str: return self.parser.join(*self._raw_urlpaths) def with_segments(self, *pathsegments: JoinablePathLike) -> Self: - raise UnsupportedOperation("path operation not supported by DataPath") + try: + (segment,) = pathsegments + except ValueError: + raise UnsupportedOperation("join not supported by DataPath") + if get_upath_protocol(segment) != "data": + raise ValueError(f"requires a data URI, got: {segment!r}") + return type(self)(segment, protocol="data", **self.storage_options) + + @property + def name(self) -> str: + return quote_plus(self.path) + + @property + def stem(self) -> str: + return quote_plus(self.path) + + @property + def suffix(self) -> str: + return "" + + @property + def suffixes(self) -> list[str]: + return [] + + def with_name(self, name: str) -> Self: + raise UnsupportedOperation("with_name not supported by DataPath") def with_suffix(self, suffix: str) -> Self: - raise UnsupportedOperation("path operation not supported by DataPath") + raise UnsupportedOperation("with_suffix not supported by DataPath") + + def with_stem(self, stem: str) -> Self: + raise UnsupportedOperation("with_stem not supported by DataPath") + + @property + def parent(self) -> Self: + return self + + @property + def parents(self) -> Sequence[Self]: + return [] + + def full_match(self, pattern, *, case_sensitive: bool | None = None) -> bool: + return super().full_match(pattern, case_sensitive=case_sensitive) def mkdir( self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False @@ -66,3 +108,19 @@ def write_text( newline: str | None = None, ) -> int: raise UnsupportedOperation("DataPath does not support writing") + + def iterdir(self) -> Iterator[Self]: + raise NotADirectoryError + + def glob( + self, pattern, *, case_sensitive=None, recurse_symlinks=False + ) -> Iterator[Self]: + return iter([]) + + def rglob( + self, pattern, *, case_sensitive=None, recurse_symlinks=False + ) -> Iterator[Self]: + return iter([]) + + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation diff --git a/upath/implementations/github.py b/upath/implementations/github.py index b95dd11e..07e07ba4 100644 --- a/upath/implementations/github.py +++ b/upath/implementations/github.py @@ -8,6 +8,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING +from upath.core import UnsupportedOperation from upath.core import UPath from upath.types import JoinablePathLike @@ -56,3 +57,29 @@ def parts(self) -> Sequence[str]: return parts[1:] else: return parts + + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + raise UnsupportedOperation + + def mkdir( + self, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise UnsupportedOperation + + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation + + def write_bytes(self, data: bytes) -> int: + raise UnsupportedOperation + + def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + raise UnsupportedOperation("GitHubPath does not support writing") diff --git a/upath/implementations/http.py b/upath/implementations/http.py index 166819f5..02394375 100644 --- a/upath/implementations/http.py +++ b/upath/implementations/http.py @@ -10,6 +10,7 @@ from fsspec.asyn import sync +from upath import UnsupportedOperation from upath._stat import UPathStatResult from upath.core import UPath from upath.types import JoinablePathLike @@ -124,3 +125,29 @@ def resolve( break return resolved_path + + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + raise UnsupportedOperation + + def mkdir( + self, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise UnsupportedOperation + + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation + + def write_bytes(self, data: bytes) -> int: + raise UnsupportedOperation("DataPath does not support writing") + + def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + raise UnsupportedOperation("DataPath does not support writing") diff --git a/upath/implementations/tar.py b/upath/implementations/tar.py index 9853b4e3..8310a850 100644 --- a/upath/implementations/tar.py +++ b/upath/implementations/tar.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from upath._stat import UPathStatResult +from upath.core import UnsupportedOperation from upath.core import UPath from upath.types import JoinablePathLike from upath.types import StatResultType @@ -41,6 +42,32 @@ def __init__( **storage_options: Unpack[TarStorageOptions], ) -> None: ... + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + raise UnsupportedOperation + + def mkdir( + self, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise UnsupportedOperation + + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation + + def write_bytes(self, data: bytes) -> int: + raise UnsupportedOperation("DataPath does not support writing") + + def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + raise UnsupportedOperation("DataPath does not support writing") + def stat( self, *, diff --git a/upath/implementations/zip.py b/upath/implementations/zip.py index 237cae15..8d34b08e 100644 --- a/upath/implementations/zip.py +++ b/upath/implementations/zip.py @@ -2,8 +2,8 @@ import sys from typing import TYPE_CHECKING -from zipfile import ZipInfo +from upath.core import UnsupportedOperation from upath.core import UPath from upath.types import JoinablePathLike @@ -35,41 +35,36 @@ def __init__( **storage_options: Unpack[ZipStorageOptions], ) -> None: ... - if sys.version_info >= (3, 11): + @classmethod + def _transform_init_args(cls, args, protocol, storage_options): + if storage_options.get("mode") in {"a", "x", "w"}: + raise UnsupportedOperation( + "ZipPath write mode disabled in universal-pathlib" + ) + return super()._transform_init_args(args, protocol, storage_options) - def mkdir( - self, - mode: int = 0o777, - parents: bool = False, - exist_ok: bool = False, - ) -> None: - is_dir = self.is_dir() - if is_dir and not exist_ok: - raise FileExistsError(f"File exists: {self.path!r}") - elif not is_dir: - zipfile = self.fs.zip - zipfile.mkdir(self.path, mode) + def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: + raise UnsupportedOperation - else: + def mkdir( + self, + mode: int = 0o777, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise UnsupportedOperation - def mkdir( - self, - mode: int = 0o777, - parents: bool = False, - exist_ok: bool = False, - ) -> None: - is_dir = self.is_dir() - if is_dir and not exist_ok: - raise FileExistsError(f"File exists: {self.path!r}") - elif not is_dir: - dirname = self.path - if dirname and not dirname.endswith("/"): - dirname += "/" - zipfile = self.fs.zip - zinfo = ZipInfo(dirname) - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 - zinfo.file_size = 0 - zinfo.external_attr |= 0x10 - zipfile.writestr(zinfo, b"") + def unlink(self, missing_ok: bool = False) -> None: + raise UnsupportedOperation + + def write_bytes(self, data: bytes) -> int: + raise UnsupportedOperation("DataPath does not support writing") + + def write_text( + self, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + raise UnsupportedOperation("DataPath does not support writing") diff --git a/upath/tests/cases.py b/upath/tests/cases.py index 309a8de2..24485e69 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -8,6 +8,7 @@ import pytest from fsspec import __version__ as fsspec_version from fsspec import filesystem +from fsspec import get_filesystem_class from packaging.version import Version from pathlib_abc import PathParser from pathlib_abc import vfspath @@ -32,6 +33,9 @@ class JoinablePathTests: path: UPath + def test_is_correct_class(self): + raise NotImplementedError("must override") + def test_parser(self): parser = self.path.parser assert isinstance(parser, PathParser) @@ -128,7 +132,7 @@ def test_copy_path(self): assert path.drive == copy_path.drive assert path.root == copy_path.root assert path.parts == copy_path.parts - assert path.fs.storage_options == copy_path.fs.storage_options + assert path.storage_options == copy_path.storage_options def test_pickling(self): path = self.path @@ -137,7 +141,7 @@ def test_pickling(self): assert type(path) is type(recovered_path) assert str(path) == str(recovered_path) - assert path.fs.storage_options == recovered_path.fs.storage_options + assert path.storage_options == recovered_path.storage_options def test_pickling_child_path(self): path = self.path / "subfolder" / "subsubfolder" @@ -149,7 +153,6 @@ def test_pickling_child_path(self): assert path.drive == recovered_path.drive assert path.root == recovered_path.root assert path.parts == recovered_path.parts - assert path.fs.storage_options == recovered_path.fs.storage_options assert path.storage_options == recovered_path.storage_options def test_as_uri(self): @@ -161,14 +164,11 @@ def test_as_uri(self): def test_protocol(self): protocol = self.path.protocol - protocols = [p] if isinstance((p := type(self.path.fs).protocol), str) else p + fs_cls = get_filesystem_class(protocol) + protocols = [p] if isinstance((p := fs_cls.protocol), str) else p print(protocol, protocols) assert protocol in protocols - def test_storage_options(self): - storage_options = self.path.storage_options - assert storage_options == self.path.fs.storage_options - def test_hashable(self): assert hash(self.path) @@ -270,6 +270,14 @@ class ReadablePathTests: path: UPath + @pytest.fixture(autouse=True) + def path_file(self, path): + self.path_file = self.path.joinpath("file1.txt") + + def test_storage_options_match_fsspec(self): + storage_options = self.path.storage_options + assert storage_options == self.path.fs.storage_options + def test_stat(self): stat_ = self.path.stat() @@ -293,11 +301,11 @@ def test_stat_dir_st_mode(self): assert stat.S_ISDIR(base.st_mode) def test_stat_file_st_mode(self): - file1 = self.path.joinpath("file1.txt").stat() + file1 = self.path_file.stat() assert stat.S_ISREG(file1.st_mode) def test_stat_st_size(self): - file1 = self.path.joinpath("file1.txt").stat() + file1 = self.path_file.stat() assert file1.st_size == 11 @pytest.mark.parametrize( @@ -386,6 +394,8 @@ def test_iterdir(self, local_testdir): assert len(up_iter) == len(pl_iter) assert {p.name for p in pl_iter} == {u.name for u in up_iter} + + def test_iterdir_parent_iteration(self): assert next(self.path.parent.iterdir()).exists() def test_iterdir2(self, local_testdir): @@ -399,7 +409,6 @@ def test_iterdir2(self, local_testdir): assert len(up_iter) == len(pl_iter) assert {p.name for p in pl_iter} == {u.name for u in up_iter} - assert next(self.path.parent.iterdir()).exists() def test_iterdir_trailing_slash(self): files_noslash = list(self.path.joinpath("folder1").iterdir()) @@ -420,44 +429,41 @@ def test_home(self): self.path.home() def test_open(self): - p = self.path.joinpath("file1.txt") + p = self.path_file with p.open(mode="r") as f: assert f.read() == "hello world" with p.open(mode="rb") as f: assert f.read() == b"hello world" def test_open_buffering(self): - p = self.path.joinpath("file1.txt") + p = self.path_file p.open(buffering=-1) def test_open_block_size(self): - p = self.path.joinpath("file1.txt") + p = self.path_file with p.open(mode="r", block_size=8192) as f: assert f.read() == "hello world" def test_open_errors(self): - p = self.path.joinpath("file1.txt") + p = self.path_file with p.open(mode="r", encoding="ascii", errors="strict") as f: assert f.read() == "hello world" - def test_read_bytes(self, pathlib_base): + def test_read_bytes(self): mock = self.path.joinpath("file2.txt") - pl = pathlib_base.joinpath("file2.txt") - assert mock.read_bytes() == pl.read_bytes() + assert mock.read_bytes() == b"hello world" - def test_read_text(self, local_testdir): + def test_read_text(self): upath = self.path.joinpath("file1.txt") - assert ( - upath.read_text() == Path(local_testdir).joinpath("file1.txt").read_text() - ) + assert upath.read_text() == "hello world" def test_read_text_encoding(self): - upath = self.path.joinpath("file1.txt") + upath = self.path_file content = upath.read_text(encoding="utf-8") assert content == "hello world" def test_read_text_errors(self): - upath = self.path.joinpath("file1.txt") + upath = self.path_file content = upath.read_text(encoding="ascii", errors="strict") assert content == "hello world" @@ -468,9 +474,12 @@ def test_rglob(self, pathlib_base): assert len(result) == len(expected) def test_walk(self, local_testdir): + def _raise(x): + raise x + # collect walk results from UPath upath_walk = [] - for dirpath, dirnames, filenames in self.path.walk(): + for dirpath, dirnames, filenames in self.path.walk(on_error=_raise): rel_dirpath = dirpath.relative_to(self.path) upath_walk.append((str(rel_dirpath), sorted(dirnames), sorted(filenames))) upath_walk.sort() @@ -485,9 +494,12 @@ def test_walk(self, local_testdir): assert upath_walk == os_walk def test_walk_top_down_false(self): + def _raise(x): + raise x + # test walk with top_down=False returns directories after their contents paths_seen = [] - for dirpath, _, _ in self.path.walk(top_down=False): + for dirpath, _, _ in self.path.walk(top_down=False, on_error=_raise): paths_seen.append(dirpath) # in bottom-up walk, parent directories should come after children @@ -522,7 +534,7 @@ def test_info(self): def test_copy_local(self, tmp_path: Path): target = UPath(tmp_path) / "target-file1.txt" - source = self.path / "file1.txt" + source = self.path_file content = source.read_text() source.copy(target) assert target.exists() @@ -532,16 +544,16 @@ def test_copy_into_local(self, tmp_path: Path): target_dir = UPath(tmp_path) / "target-dir" target_dir.mkdir() - source = self.path / "file1.txt" + source = self.path_file content = source.read_text() source.copy_into(target_dir) - target = target_dir / "file1.txt" + target = target_dir / source.name assert target.exists() assert target.read_text() == content def test_copy_memory(self, clear_fsspec_memory_cache): target = UPath("memory:///target-file1.txt") - source = self.path / "file1.txt" + source = self.path_file content = source.read_text() source.copy(target) assert target.exists() @@ -551,15 +563,15 @@ def test_copy_into_memory(self, clear_fsspec_memory_cache): target_dir = UPath("memory:///target-dir") target_dir.mkdir() - source = self.path / "file1.txt" + source = self.path_file content = source.read_text() source.copy_into(target_dir) - target = target_dir / "file1.txt" + target = target_dir / source.name assert target.exists() assert target.read_text() == content def test_read_with_fsspec(self): - p = self.path.joinpath("file2.txt") + p = self.path_file protocol = p.protocol storage_options = p.storage_options @@ -587,7 +599,57 @@ def test_owner(self): # ============================================================================= -class WritablePathTests: +class _CommonWritablePathTests: + + SUPPORTS_EMPTY_DIRS = True + + path: UPath + + def test_chmod(self): + with pytest.raises(NotImplementedError): + self.path_file.chmod(777) + + def test_lchmod(self): + with pytest.raises(UnsupportedOperation): + self.path.lchmod(mode=0o777) + + def test_symlink_to(self): + with pytest.raises(UnsupportedOperation): + self.path_file.symlink_to("target") + with pytest.raises(UnsupportedOperation): + self.path.joinpath("link").symlink_to("target") + + def test_hardlink_to(self): + with pytest.raises(UnsupportedOperation): + self.path_file.symlink_to("target") + with pytest.raises(UnsupportedOperation): + self.path.joinpath("link").hardlink_to("target") + + +class NonWritablePathTests(_CommonWritablePathTests): + + def test_mkdir_raises(self): + with pytest.raises(UnsupportedOperation): + self.path.mkdir() + + def test_touch_raises(self): + with pytest.raises(UnsupportedOperation): + self.path.touch() + + def test_unlink(self): + with pytest.raises(UnsupportedOperation): + self.path.unlink() + + def test_write_bytes(self): + with pytest.raises(UnsupportedOperation): + self.path_file.write_bytes(b"abc") + + def test_write_text(self): + with pytest.raises(UnsupportedOperation): + self.path_file.write_text("abc") + + +class WritablePathTests(_CommonWritablePathTests): """Tests for WritablePath interface. These tests verify operations that write to the filesystem: @@ -597,10 +659,6 @@ class WritablePathTests: - Removing files/directories (unlink, rmdir) """ - SUPPORTS_EMPTY_DIRS = True - - path: UPath - def test_mkdir(self): new_dir = self.path.joinpath("new_dir") new_dir.mkdir() @@ -699,22 +757,6 @@ def test_write_text_errors(self): path.write_text(s, encoding="ascii", errors="strict") assert path.read_text(encoding="ascii") == s - def test_chmod(self): - with pytest.raises(NotImplementedError): - self.path.joinpath("file1.txt").chmod(777) - - def test_lchmod(self): - with pytest.raises(UnsupportedOperation): - self.path.lchmod(mode=0o777) - - def test_symlink_to(self): - with pytest.raises(UnsupportedOperation): - self.path.joinpath("link").symlink_to("target") - - def test_hardlink_to(self): - with pytest.raises(UnsupportedOperation): - self.path.joinpath("link").hardlink_to("target") - class ReadWritePathTests: """Tests requiring both ReadablePath and WritablePath interfaces. diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index 2f5e7b1c..c9333c6f 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -585,12 +585,16 @@ def mock_hf_api(pathlib_base, monkeypatch, hf_test_repo): # noqa: C901 hf_file_system = pytest.importorskip( "huggingface_hub.hf_file_system", reason="hf tests require huggingface_hub" ) + httpx = pytest.importorskip("httpx") class MockedHfApi(huggingface_hub.HfApi): def repo_info(self, repo_id, *args, repo_type=None, **kwargs): if repo_id != hf_test_repo: - raise huggingface_hub.errors.RepositoryNotFoundError(repo_id) + raise huggingface_hub.errors.RepositoryNotFoundError( + repo_id, + response=httpx.Response(404, request=...), + ) elif repo_type is None or repo_type == "model": return huggingface_hub.hf_api.ModelInfo(id=repo_id) elif repo_type == "dataset": @@ -602,7 +606,10 @@ def repo_info(self, repo_id, *args, repo_type=None, **kwargs): def get_paths_info(self, repo_id, paths, *args, **kwargs): if repo_id != hf_test_repo: - raise huggingface_hub.errors.RepositoryNotFoundError(repo_id) + raise huggingface_hub.errors.RepositoryNotFoundError( + repo_id, + response=httpx.Response(404, request=...), + ) paths_info = [] for path in paths: if path: @@ -628,7 +635,9 @@ def list_repo_tree( self, repo_id, path_in_repo, *args, recursive=False, **kwargs ): if repo_id != hf_test_repo: - raise huggingface_hub.errors.RepositoryNotFoundError(repo_id) + raise huggingface_hub.errors.RepositoryNotFoundError( + repo_id, response=httpx.Response(404, request=...) + ) pathlib_dir = pathlib_base / path_in_repo if path_in_repo else pathlib_base for path in pathlib_dir.rglob("*") if recursive else pathlib_dir.glob("*"): if path.is_file(): diff --git a/upath/tests/implementations/test_azure.py b/upath/tests/implementations/test_azure.py index 30b683ac..eb82e930 100644 --- a/upath/tests/implementations/test_azure.py +++ b/upath/tests/implementations/test_azure.py @@ -4,12 +4,14 @@ from upath.implementations.cloud import AzurePath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base from ..utils import skip_on_windows @skip_on_windows -@pytest.mark.usefixtures("path") -class TestAzurePath(BaseTests): +class TestAzurePath(BaseTests, metaclass=OverrideMeta): SUPPORTS_EMPTY_DIRS = False @pytest.fixture(autouse=True, scope="function") @@ -23,9 +25,18 @@ def path(self, azurite_credentials, azure_fixture): self.path = UPath(azure_fixture, **self.storage_options) self.prepare_file_system() - def test_is_AzurePath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, AzurePath) + @overrides_base + def test_protocol(self): + # test all valid protocols for azure... + protocol = self.path.protocol + protocols = ["abfs", "abfss", "adl", "az"] + assert protocol in protocols + + @extends_base def test_rmdir(self): new_dir = self.path / "new_dir_rmdir" new_dir.mkdir() @@ -38,11 +49,7 @@ def test_rmdir(self): with pytest.raises(NotADirectoryError): (self.path / "a" / "file.txt").rmdir() - def test_protocol(self): - # test all valid protocols for azure... - protocol = self.path.protocol - assert protocol in ["abfs", "abfss", "adl", "az"] - + @extends_base def test_broken_mkdir(self): path = UPath( "az://new-container/", diff --git a/upath/tests/implementations/test_cached.py b/upath/tests/implementations/test_cached.py index e7b757f2..e7708797 100644 --- a/upath/tests/implementations/test_cached.py +++ b/upath/tests/implementations/test_cached.py @@ -4,9 +4,11 @@ from upath.implementations.cached import SimpleCachePath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base -class TestSimpleCachePath(BaseTests): +class TestSimpleCachePath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): if not local_testdir.startswith("/"): @@ -15,5 +17,6 @@ def path(self, local_testdir): self.path = UPath(path) self.prepare_file_system() - def test_is_SimpleCachePath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, SimpleCachePath) diff --git a/upath/tests/implementations/test_data.py b/upath/tests/implementations/test_data.py index b869c268..b539e30e 100644 --- a/upath/tests/implementations/test_data.py +++ b/upath/tests/implementations/test_data.py @@ -1,240 +1,224 @@ import stat -import fsspec import pytest from upath import UnsupportedOperation from upath import UPath from upath.implementations.data import DataPath -from upath.tests.cases import BaseTests -from ..utils import xfail_if_version +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import overrides_base -pytestmark = xfail_if_version( - "fsspec", lt="2023.12.2", reason="fsspec<2023.12.2 does not support data" -) - -class TestUPathDataPath(BaseTests): +class TestUPathDataPath( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): """ Unit-tests for the DataPath implementation of UPath. """ @pytest.fixture(autouse=True) def path(self): - """ - Fixture for the UPath instance to be tested. - """ - path = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12PYeuECAASTAlbqXbfWAAAAAElFTkSuQmCC" # noqa: E501 + path = "data:text/plain;base64,aGVsbG8gd29ybGQ=" self.path = UPath(path) - def test_is_DataPath(self): - """ - Test that the path is a GitHubPath instance. - """ + @pytest.fixture(autouse=True) + def path_file(self, path): + self.path_file = self.path + + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, DataPath) + @overrides_base def test_with_segments(self): + # DataPath does not support joins, so in all usual cases it'll raise with pytest.raises(UnsupportedOperation): - super().test_with_segments() - - def test_is_relative_to(self): - with pytest.raises(UnsupportedOperation): - super().test_is_relative_to() - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_stat_dir_st_mode(self): - super().test_stat_dir_st_mode() - - def test_stat_file_st_mode(self): - assert self.path.is_file() - assert stat.S_ISREG(self.path.stat().st_mode) - - def test_stat_st_size(self): - assert self.path.stat().st_size == 69 - - def test_exists(self): - # datapath exists is always true... - path = self.path - assert path.exists() - - @pytest.mark.skip(reason="DataPath does support joins or globs") - def test_glob(self, pathlib_base): - with pytest.raises(NotImplementedError): - pathlib_base.glob("*") - - def test_is_dir(self): - assert not self.path.is_dir() - - def test_is_file(self): - assert self.path.is_file() - - def test_iterdir(self): - with pytest.raises(NotADirectoryError): - list(self.path.iterdir()) - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_iterdir2(self): - pass - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_iterdir_trailing_slash(self): - pass - - def test_mkdir(self): - with pytest.raises(FileExistsError): - self.path.mkdir() - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_mkdir_exists_ok_true(self): - pass - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_mkdir_exists_ok_false(self): - pass - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_mkdir_parents_true_exists_ok_true(self): - pass - - @pytest.mark.skip(reason="DataPath does not have directories") - def test_mkdir_parents_true_exists_ok_false(self): - pass - - def test_open(self): - p = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - with p.open(mode="r") as f: - assert f.read() == "hello world" - with p.open(mode="rb") as f: - assert f.read() == b"hello world" - - def test_open_buffering(self): - self.path.open(buffering=-1) + self.path.with_segments("data:text/plain;base64,", "aGVsbG8K") + # but you can instantiate with a single full url + self.path.with_segments("data:text/plain;base64,aGVsbG8K") - def test_open_block_size(self): - p = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - with p.open(mode="r", block_size=8192) as f: - assert f.read() == "hello world" + @overrides_base + def test_parents(self): + # DataPath is always a absolute path with no parents + assert self.path.parents == [] - def test_open_errors(self): - p = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - with p.open(mode="r", encoding="ascii", errors="strict") as f: - assert f.read() == "hello world" + @overrides_base + def test_with_name(self): + # DataPath does not support name changes + with pytest.raises(UnsupportedOperation): + self.path.with_name("newname") - def test_read_bytes(self, pathlib_base): - assert len(self.path.read_bytes()) == 69 + @overrides_base + def test_with_suffix(self): + # DataPath does not support suffix changes + with pytest.raises(UnsupportedOperation): + self.path.with_suffix(".new") - def test_read_text(self, local_testdir): - assert UPath("data:base64,SGVsbG8gV29ybGQ=").read_text() == "Hello World" + @overrides_base + def test_with_stem(self): + # DataPath does not support stem changes + with pytest.raises(UnsupportedOperation): + self.path.with_stem("newname") - def test_parents(self): - with pytest.raises(NotImplementedError): - self.path.parents[0] + @overrides_base + def test_suffix(self): + # DataPath does not have suffixes + assert self.path.suffix == "" - def test_rename(self): - with pytest.raises(NotImplementedError): - self.path.rename("newname") + @overrides_base + def test_suffixes(self): + # DataPath does not have suffixes + assert self.path.suffixes == [] - @pytest.mark.skip("DataPath does not support rename") - def test_rename_with_target_relative(self): - pass + @overrides_base + def test_repr_after_with_name(self): + with pytest.raises(UnsupportedOperation): + repr(self.path.with_name("data:,ABC")) - @pytest.mark.skip("DataPath does not support rename") - def test_rename_with_target_absolute(self): - pass + @overrides_base + def test_repr_after_with_suffix(self): + with pytest.raises(UnsupportedOperation): + repr(self.path.with_suffix("")) - def test_rglob(self, pathlib_base): - with pytest.raises(NotImplementedError): - list(self.path.rglob("*")) + @overrides_base + def test_child_path(self): + # DataPath does not support joins, so child paths are unsupported + with pytest.raises(UnsupportedOperation): + super().test_child_path() - def test_touch(self): - self.path.touch() + @overrides_base + def test_pickling_child_path(self): + # DataPath does not support joins, so child paths are unsupported + with pytest.raises(UnsupportedOperation): + super().test_pickling_child_path() - def test_touch_exists_ok_false(self): - with pytest.raises(FileExistsError): - self.path.touch(exist_ok=False) + @overrides_base + def test_relative_to(self): + # DataPath only relative_to with itself + with pytest.raises(ValueError): + self.path.relative_to("data:,ABC") + self.path.relative_to(self.path) - def test_touch_exists_ok_true(self): - self.path.touch() + @overrides_base + def test_is_relative_to(self): + # DataPath only relative_to with itself + assert not self.path.is_relative_to("data:,ABC") + assert self.path.is_relative_to(self.path) - def test_touch_unlink(self): - self.path.touch() - with pytest.raises(NotImplementedError): - self.path.unlink() + @overrides_base + def test_full_match(self): + assert self.path.full_match("*") + assert not self.path.full_match("xxx") - def test_write_bytes(self, pathlib_base): - with pytest.raises(NotImplementedError): - self.path.write_bytes(b"test") + @overrides_base + def test_trailing_slash_joinpath_is_identical(self): + # DataPath has no slashes, and is not joinable + with pytest.raises(UnsupportedOperation): + super().test_trailing_slash_joinpath_is_identical() - def test_write_text(self, pathlib_base): - with pytest.raises(NotImplementedError): - self.path.write_text("test") + @overrides_base + def test_trailing_slash_is_stripped(self): + # DataPath has no slashes, and is not joinable + with pytest.raises(UnsupportedOperation): + super().test_trailing_slash_is_stripped() - def test_read_with_fsspec(self): - pth = self.path - fs = fsspec.filesystem(pth.protocol, **pth.storage_options) - assert fs.cat_file(pth.path) == pth.read_bytes() + @overrides_base + def test_private_url_attr_in_sync(self): + # DataPath does not support joins, so we check on self.path + assert self.path._url - @pytest.mark.skip(reason="DataPath does not support joins") - def test_pickling_child_path(self): - pass + @overrides_base + def test_stat_dir_st_mode(self): + # DataPath does not have directories + assert not stat.S_ISDIR(self.path.stat().st_mode) - @pytest.mark.skip(reason="DataPath does not support joins") - def test_child_path(self): - pass + @overrides_base + def test_exists(self): + # A valid DataPath always exists + assert self.path.exists() - def test_with_name(self): - with pytest.raises(NotImplementedError): - self.path.with_name("newname") + @overrides_base + def test_glob(self): + # DataPath does not have dirs, joins or globs + assert list(self.path.glob("*")) == [] - def test_with_suffix(self): - with pytest.raises(NotImplementedError): - self.path.with_suffix(".new") + @overrides_base + def test_rglob(self): + # DataPath does not have dirs, joins or globs + assert list(self.path.rglob("*")) == [] - def test_suffix(self): - assert self.path.suffix == "" + @overrides_base + def test_is_dir(self): + # DataPath does not have directories + assert not self.path.is_dir() - def test_suffixes(self): - assert self.path.suffixes == [] + @overrides_base + def test_is_file(self): + # DataPath is always a file + assert self.path.is_file() - def test_with_stem(self): - with pytest.raises(NotImplementedError): - self.path.with_stem("newname") + @overrides_base + def test_iterdir(self): + # DataPath does not have directories + with pytest.raises(NotADirectoryError): + self.path.iterdir() - @pytest.mark.skip(reason="DataPath does not support joins") - def test_repr_after_with_suffix(self): - pass + @overrides_base + def test_iterdir_parent_iteration(self): + with pytest.raises(NotADirectoryError): + super().test_iterdir_parent_iteration() - @pytest.mark.skip(reason="DataPath does not support joins") - def test_repr_after_with_name(self): - pass + @overrides_base + def test_iterdir2(self): + # DataPath does not have directories, or joins + with pytest.raises(NotADirectoryError): + self.path_file.iterdir() - @pytest.mark.skip(reason="DataPath does not support directories") - def test_rmdir_no_dir(self): - pass + @overrides_base + def test_iterdir_trailing_slash(self): + # DataPath does not have directories, or joins + with pytest.raises(UnsupportedOperation): + super().test_iterdir_trailing_slash() - @pytest.mark.skip(reason="DataPath does not support directories") - def test_iterdir_no_dir(self): - pass + @overrides_base + def test_read_bytes(self): + assert self.path.read_bytes() == b"hello world" - @pytest.mark.skip(reason="DataPath does not support joins") - def test_private_url_attr_in_sync(self): - pass + @overrides_base + def test_read_text(self): + assert self.path.read_text() == "hello world" - @pytest.mark.skip(reason="DataPath does not support joins") - def test_fsspec_compat(self): - pass + @overrides_base + def test_walk(self): + # DataPath does not have directories + assert list(self.path.walk()) == [] - def test_rmdir_not_empty(self): - with pytest.raises(NotADirectoryError): - self.path.rmdir() + @overrides_base + def test_walk_top_down_false(self): + # DataPath does not have directories + assert list(self.path.walk(top_down=False)) == [] + @overrides_base def test_samefile(self): - f1 = self.path + # DataPath doesn't have joins, so only identical paths are samefile + f1 = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") + f2 = UPath("data:text/plain;base64,SGVsbG8gd29ybGQ=") + assert f1.samefile(f2) is False + assert f1.samefile(f2.path) is False assert f1.samefile(f1) is True + assert f1.samefile(f1.path) is True + @overrides_base def test_info(self): + # DataPath info checks p0 = self.path assert p0.info.exists() is True @@ -242,102 +226,19 @@ def test_info(self): assert p0.info.is_dir() is False assert p0.info.is_symlink() is False - def test_copy_local(self, tmp_path): - target = UPath(tmp_path) / "target-file1.txt" - - source = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - content = source.read_text() - source.copy(target) - assert target.exists() - assert target.read_text() == content - - def test_copy_into_local(self, tmp_path): - target_dir = UPath(tmp_path) / "target-dir" - target_dir.mkdir() - - source = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - content = source.read_text() - source.copy_into(target_dir) - target = target_dir / source.name - assert target.exists() - assert target.read_text() == content - - def test_copy_memory(self, clear_fsspec_memory_cache): - target = UPath("memory:///target-file1.txt") - - source = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - content = source.read_text() - source.copy(target) - assert target.exists() - assert target.read_text() == content - - def test_copy_into_memory(self, clear_fsspec_memory_cache): - target_dir = UPath("memory:///target-dir") - target_dir.mkdir() - - source = UPath("data:text/plain;base64,aGVsbG8gd29ybGQ=") - content = source.read_text() - source.copy_into(target_dir) - target = target_dir / source.name - assert target.exists() - assert target.read_text() == content - - @pytest.mark.skip(reason="DataPath does not support unlink") - def test_move_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="DataPath does not support unlink") - def test_move_into_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="DataPath does not support unlink") - def test_move_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="DataPath does not support unlink") - def test_move_into_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="DataPath does not support relative_to") - def test_relative_to(self): - pass - - @pytest.mark.skip(reason="DataPath does not support joins") - def test_trailing_slash_joinpath_is_identical(self): - pass - - @pytest.mark.skip(reason="DataPath does not support joins") - def test_trailing_slash_is_stripped(self): - pass - - @pytest.mark.skip(reason="DataPath does not support joins") - def test_parents_are_absolute(self): - pass - - @pytest.mark.skip(reason="DataPath does not support write_text") - def test_write_text_encoding(self): - pass - - @pytest.mark.skip(reason="DataPath does not support write_text") - def test_write_text_errors(self): - pass - - @pytest.mark.skip(reason="base test incompatible with DataPath") - def test_read_text_encoding(self): - pass - - @pytest.mark.skip(reason="base test incompatible with DataPath") - def test_read_text_errors(self): - pass + @overrides_base + def test_mkdir_raises(self): + # DataPaths always exist and are files + with pytest.raises(FileExistsError): + self.path_file.mkdir() - @pytest.mark.skip(reason="DataPath does not support walk") - def test_walk(self, local_testdir): - pass + @overrides_base + def test_touch_raises(self): + # DataPaths always exist, so touch is a noop + self.path_file.touch() - @pytest.mark.skip(reason="DataPath does not support walk") - def test_walk_top_down_false(self): - pass - - @pytest.mark.skip(reason="DataPath does not support full_match") - def test_full_match(self): - pass + @overrides_base + def test_unlink(self): + # DataPaths can't be deleted + with pytest.raises(UnsupportedOperation): + self.path_file.unlink() diff --git a/upath/tests/implementations/test_ftp.py b/upath/tests/implementations/test_ftp.py index d8f46758..b7edc7ce 100644 --- a/upath/tests/implementations/test_ftp.py +++ b/upath/tests/implementations/test_ftp.py @@ -1,21 +1,30 @@ import pytest from upath import UPath +from upath.implementations.ftp import FTPPath from upath.tests.cases import BaseTests from upath.tests.utils import skip_on_windows +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base + @skip_on_windows -class TestUPathFTP(BaseTests): +class TestUPathFTP(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, ftp_server): self.path = UPath("", protocol="ftp", **ftp_server) self.prepare_file_system() + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, FTPPath) -def test_ftp_path_mtime(ftp_server): - path = UPath("file1.txt", protocol="ftp", **ftp_server) - path.touch() - mtime = path.stat().st_mtime - assert isinstance(mtime, float) + @extends_base + def test_ftp_path_mtime(self, ftp_server): + path = UPath("file1.txt", protocol="ftp", **ftp_server) + path.touch() + mtime = path.stat().st_mtime + assert isinstance(mtime, float) diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py index 30db5645..41d16c63 100644 --- a/upath/tests/implementations/test_gcs.py +++ b/upath/tests/implementations/test_gcs.py @@ -5,12 +5,14 @@ from upath.implementations.cloud import GCSPath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base from ..utils import skip_on_windows @skip_on_windows -@pytest.mark.usefixtures("path") -class TestGCSPath(BaseTests): +class TestGCSPath(BaseTests, metaclass=OverrideMeta): SUPPORTS_EMPTY_DIRS = False @pytest.fixture(autouse=True, scope="function") @@ -18,9 +20,11 @@ def path(self, gcs_fixture): path, endpoint_url = gcs_fixture self.path = UPath(path, endpoint_url=endpoint_url, token="anon") - def test_is_GCSPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, GCSPath) + @extends_base def test_rmdir(self): dirname = "rmdir_test" mock_dir = self.path.joinpath(dirname) @@ -31,10 +35,6 @@ def test_rmdir(self): with pytest.raises(NotADirectoryError): self.path.joinpath("file1.txt").rmdir() - @pytest.mark.skip - def test_makedirs_exist_ok_false(self): - pass - @skip_on_windows def test_mkdir_in_empty_bucket(docker_gcs): diff --git a/upath/tests/implementations/test_github.py b/upath/tests/implementations/test_github.py index d132702c..c0120232 100644 --- a/upath/tests/implementations/test_github.py +++ b/upath/tests/implementations/test_github.py @@ -7,10 +7,15 @@ from upath import UPath from upath.implementations.github import GitHubPath -from upath.tests.cases import BaseTests + +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import overrides_base pytestmark = pytest.mark.skipif( - os.environ.get("CI") + os.environ.get("CI", False) and not ( platform.system() == "Linux" and sys.version_info[:2] in {(3, 9), (3, 13)} ), @@ -54,7 +59,12 @@ def class_decorator(cls): @wrap_all_tests(xfail_on_github_connection_error) -class TestUPathGitHubPath(BaseTests): +class TestUPathGitHubPath( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): """ Unit-tests for the GitHubPath implementation of UPath. """ @@ -67,76 +77,6 @@ def path(self): path = "github://ap--:universal_pathlib@test_data/data" self.path = UPath(path) - def test_is_GitHubPath(self): - """ - Test that the path is a GitHubPath instance. - """ + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, GitHubPath) - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_mkdir(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_mkdir_exists_ok_false(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_mkdir_parents_true_exists_ok_false(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_rename(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_rename2(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_touch(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_touch_unlink(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_write_bytes(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_write_text(self): - pass - - @pytest.mark.skip(reason="GitHub filesystem is read-only") - def test_fsspec_compat(self): - pass - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_move_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_move_into_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_move_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_move_into_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_rename_with_target_absolute(self, target_factory): - return super().test_rename_with_target_str_absolute(target_factory) - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_write_text_encoding(self): - return super().test_write_text_encoding() - - @pytest.mark.skip(reason="Only testing read on GithubPath") - def test_write_text_errors(self): - return super().test_write_text_errors() diff --git a/upath/tests/implementations/test_hdfs.py b/upath/tests/implementations/test_hdfs.py index 6ece8ad8..cc4fe2f9 100644 --- a/upath/tests/implementations/test_hdfs.py +++ b/upath/tests/implementations/test_hdfs.py @@ -6,15 +6,18 @@ from upath.implementations.hdfs import HDFSPath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base @pytest.mark.hdfs -class TestUPathHDFS(BaseTests): +class TestUPathHDFS(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir, hdfs): host, user, port = hdfs path = f"hdfs:{local_testdir}" self.path = UPath(path, host=host, user=user, port=port) - def test_is_HDFSPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, HDFSPath) diff --git a/upath/tests/implementations/test_hf.py b/upath/tests/implementations/test_hf.py index d5727d89..4b30dbfc 100644 --- a/upath/tests/implementations/test_hf.py +++ b/upath/tests/implementations/test_hf.py @@ -1,10 +1,15 @@ import pytest from fsspec import get_filesystem_class +from upath import UnsupportedOperation from upath import UPath from upath.implementations.cloud import HfPath -from ..cases import BaseTests +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import overrides_base try: get_filesystem_class("hf") @@ -31,84 +36,22 @@ def test_hfpath(): raise -class TestUPathHf(BaseTests): +class TestUPathHf( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): @pytest.fixture(autouse=True, scope="function") def path(self, hf_fixture_with_readonly_mocked_hf_api): self.path = UPath(hf_fixture_with_readonly_mocked_hf_api) - @pytest.mark.skip - def test_mkdir(self): - pass + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, HfPath) - @pytest.mark.skip - def test_mkdir_exists_ok_false(self): - pass - - @pytest.mark.skip - def test_mkdir_exists_ok_true(self): - pass - - @pytest.mark.skip - def test_mkdir_parents_true_exists_ok_true(self): - pass - - @pytest.mark.skip - def test_mkdir_parents_true_exists_ok_false(self): - pass - - @pytest.mark.skip - def test_makedirs_exist_ok_true(self): - pass - - @pytest.mark.skip - def test_makedirs_exist_ok_false(self): - pass - - @pytest.mark.skip - def test_touch(self): - pass - - @pytest.mark.skip - def test_touch_unlink(self): - pass - - @pytest.mark.skip - def test_write_bytes(self, pathlib_base): - pass - - @pytest.mark.skip - def test_write_text(self, pathlib_base): - pass - - def test_fsspec_compat(self): - pass - - def test_rename(self): - pass - - def test_rename2(self): - pass - - def test_move_local(self, tmp_path): - pass - - def test_move_into_local(self, tmp_path): - pass - - def test_move_memory(self, clear_fsspec_memory_cache): - pass - - def test_move_into_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="HfPath does not support listing repositories") - def test_iterdir(self, local_testdir): - pass - - @pytest.mark.skip(reason="HfPath does not support listing repositories") - def test_iterdir2(self, local_testdir): - pass - - @pytest.mark.skip(reason="HfPath does not currently test write") - def test_rename_with_target_absolute(self, target_factory): - return super().test_rename_with_target_absolute(target_factory) + @overrides_base + def test_iterdir_parent_iteration(self): + # HfPath does not support listing all available Repositories + with pytest.raises(UnsupportedOperation): + super().test_iterdir_parent_iteration() diff --git a/upath/tests/implementations/test_http.py b/upath/tests/implementations/test_http.py index f3b22d43..66d76e62 100644 --- a/upath/tests/implementations/test_http.py +++ b/upath/tests/implementations/test_http.py @@ -1,15 +1,17 @@ import pytest # noqa: F401 -from fsspec import __version__ as fsspec_version from fsspec import get_filesystem_class -from packaging.version import Version from upath import UPath from upath.implementations.http import HTTPPath -from ..cases import BaseTests +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base from ..utils import skip_on_windows from ..utils import xfail_if_no_ssl_connection -from ..utils import xfail_if_version try: get_filesystem_class("http") @@ -36,113 +38,42 @@ def test_httppath(internet_connection): @xfail_if_no_ssl_connection -def test_httpspath(): +def test_httpspath(internet_connection): path = UPath("https://example.com") assert isinstance(path, HTTPPath) assert path.exists() @skip_on_windows -class TestUPathHttp(BaseTests): +class TestUPathHttp( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): @pytest.fixture(autouse=True, scope="function") def path(self, http_fixture): self.path = UPath(http_fixture) + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, HTTPPath) + + @extends_base def test_work_at_root(self): assert "folder" in (f.name for f in self.path.parent.iterdir()) - @pytest.mark.skip - def test_mkdir(self): - pass - - @pytest.mark.parametrize( - "pattern", - ( - "*.txt", - pytest.param( - "*", - marks=xfail_if_version( - "fsspec", - gt="2023.10.0", - lt="2024.5.0", - reason="requires fsspec>=2024.5.0", - ), - ), - pytest.param( - "**/*.txt", - marks=( - pytest.mark.xfail(reason="requires fsspec>=2023.9.0") - if Version(fsspec_version) < Version("2023.9.0") - else () - ), - ), - ), - ) - def test_glob(self, pathlib_base, pattern): - super().test_glob(pathlib_base, pattern) - - @pytest.mark.skip - def test_mkdir_exists_ok_false(self): - pass - - @pytest.mark.skip - def test_mkdir_exists_ok_true(self): - pass - - @pytest.mark.skip - def test_mkdir_parents_true_exists_ok_true(self): - pass - - @pytest.mark.skip - def test_mkdir_parents_true_exists_ok_false(self): - pass - - @pytest.mark.skip - def test_makedirs_exist_ok_true(self): - pass - - @pytest.mark.skip - def test_makedirs_exist_ok_false(self): - pass - - @pytest.mark.skip - def test_touch(self): - pass - - @pytest.mark.skip - def test_touch_unlink(self): - pass - - @pytest.mark.skip - def test_write_bytes(self, pathlib_base): - pass - - @pytest.mark.skip - def test_write_text(self, pathlib_base): - pass - - def test_fsspec_compat(self): - pass - + @extends_base def test_resolve(self): # Also tests following redirects, because the test server issues a # 301 redirect for `http://127.0.0.1:8080/folder` to # `http://127.0.0.1:8080/folder/` assert str(self.path.resolve()).endswith("/") - def test_rename(self): - with pytest.raises(NotImplementedError): - return super().test_rename() - - def test_rename2(self): - with pytest.raises(NotImplementedError): - return super().test_rename() - - @xfail_if_version("fsspec", lt="2024.2.0", reason="requires fsspec>=2024.2.0") - def test_stat_dir_st_mode(self): - super().test_stat_dir_st_mode() - + @overrides_base def test_info(self): + # HTTPPath folders are files too + p0 = self.path.joinpath("file1.txt") p1 = self.path.joinpath("folder1") @@ -157,34 +88,6 @@ def test_info(self): assert p1.info.is_dir() is True assert p1.info.is_symlink() is False - @pytest.mark.skip(reason="HttpPath does not support unlink") - def test_move_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="HttpPath does not support unlink") - def test_move_into_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="HttpPath does not support unlink") - def test_move_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="HttpPath does not support unlink") - def test_move_into_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="Only testing read on HttpPath") - def test_rename_with_target_absolute(self, target_factory): - return super().test_rename_with_target_absolute(target_factory) - - @pytest.mark.skip(reason="Only testing read on HttpPath") - def test_write_text_encoding(self): - return super().test_write_text_encoding() - - @pytest.mark.skip(reason="Only testing read on HttpPath") - def test_write_text_errors(self): - return super().test_write_text_errors() - @pytest.mark.parametrize( "args,parts", diff --git a/upath/tests/implementations/test_local.py b/upath/tests/implementations/test_local.py index db1bfd56..aef91139 100644 --- a/upath/tests/implementations/test_local.py +++ b/upath/tests/implementations/test_local.py @@ -5,56 +5,50 @@ from upath import UPath from upath.implementations.local import LocalPath -from upath.tests.cases import BaseTests -from upath.tests.utils import xfail_if_version +from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base +from ..utils import xfail_if_version -class TestFSSpecLocal(BaseTests): + +class TestFSSpecLocal(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): path = f"file://{local_testdir}" self.path = UPath(path) - def test_is_LocalPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, LocalPath) + @overrides_base def test_cwd(self): + # .cwd() is implemented for local filesystems cwd = type(self.path).cwd() assert isinstance(cwd, LocalPath) assert cwd.path == Path.cwd().as_posix() + @overrides_base def test_home(self): + # .home() is implemented for local filesystems cwd = type(self.path).home() assert isinstance(cwd, LocalPath) assert cwd.path == Path.home().as_posix() + @overrides_base def test_chmod(self): + # .chmod() works for local filesystems self.path.joinpath("file1.txt").chmod(777) @xfail_if_version("fsspec", lt="2023.10.0", reason="requires fsspec>=2023.10.0") -class TestRayIOFSSpecLocal(BaseTests): +class TestRayIOFSSpecLocal(TestFSSpecLocal): @pytest.fixture(autouse=True) def path(self, local_testdir): path = f"local://{local_testdir}" self.path = UPath(path) - def test_is_LocalPath(self): - assert isinstance(self.path, LocalPath) - - def test_cwd(self): - cwd = type(self.path).cwd() - assert isinstance(cwd, LocalPath) - assert cwd.path == Path.cwd().as_posix() - - def test_home(self): - cwd = type(self.path).home() - assert isinstance(cwd, LocalPath) - assert cwd.path == Path.home().as_posix() - - def test_chmod(self): - self.path.joinpath("file1.txt").chmod(777) - @pytest.mark.parametrize( "protocol,path", diff --git a/upath/tests/implementations/test_memory.py b/upath/tests/implementations/test_memory.py index 7a0b9aea..c98222fa 100644 --- a/upath/tests/implementations/test_memory.py +++ b/upath/tests/implementations/test_memory.py @@ -4,9 +4,11 @@ from upath.implementations.memory import MemoryPath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base -class TestMemoryPath(BaseTests): +class TestMemoryPath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): if not local_testdir.startswith("/"): @@ -15,7 +17,8 @@ def path(self, local_testdir): self.path = UPath(path) self.prepare_file_system() - def test_is_MemoryPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, MemoryPath) diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index 249ed172..d4027fdc 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -3,12 +3,15 @@ import sys import fsspec -import pytest # noqa: F401 +import pytest from upath import UPath from upath.implementations.cloud import S3Path from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base def silence_botocore_datetime_deprecation(cls): @@ -25,7 +28,7 @@ def silence_botocore_datetime_deprecation(cls): @silence_botocore_datetime_deprecation -class TestUPathS3(BaseTests): +class TestUPathS3(BaseTests, metaclass=OverrideMeta): SUPPORTS_EMPTY_DIRS = False @pytest.fixture(autouse=True) @@ -35,13 +38,11 @@ def path(self, s3_fixture): self.anon = anon self.s3so = s3so - def test_is_S3Path(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, S3Path) - def test_chmod(self): - # todo - pass - + @extends_base def test_rmdir(self): dirname = "rmdir_test" mock_dir = self.path.joinpath(dirname) @@ -51,11 +52,13 @@ def test_rmdir(self): with pytest.raises(NotADirectoryError): self.path.joinpath("file1.txt").rmdir() - def test_relative_to(self): + @extends_base + def test_relative_to_extra(self): assert "file.txt" == str( UPath("s3://test_bucket/file.txt").relative_to(UPath("s3://test_bucket")) ) + @extends_base def test_iterdir_root(self): client_kwargs = self.path.storage_options["client_kwargs"] bucket_path = UPath("s3://other_test_bucket", client_kwargs=client_kwargs) @@ -68,6 +71,7 @@ def test_iterdir_root(self): assert x.name != "" assert x.exists() + @extends_base @pytest.mark.parametrize( "joiner", [["bucket", "path", "file"], ["bucket/path/file"]] ) @@ -76,10 +80,12 @@ def test_no_bucket_joinpath(self, joiner): path = path.joinpath(*joiner) assert str(path) == "s3://bucket/path/file" + @extends_base def test_creating_s3path_with_bucket(self): path = UPath("s3://", bucket="bucket", anon=self.anon, **self.s3so) assert str(path) == "s3://bucket/" + @extends_base def test_iterdir_with_plus_in_name(self, s3_with_plus_chr_name): bucket, anon, s3so = s3_with_plus_chr_name p = UPath( @@ -93,6 +99,7 @@ def test_iterdir_with_plus_in_name(self, s3_with_plus_chr_name): (file,) = files assert file == p.joinpath("file.txt") + @extends_base @pytest.mark.xfail(reason="fsspec/universal_pathlib#144") def test_rglob_with_double_fwd_slash(self, s3_with_double_fwd_slash_files): import boto3 diff --git a/upath/tests/implementations/test_sftp.py b/upath/tests/implementations/test_sftp.py index 093d14b4..7ad10c06 100644 --- a/upath/tests/implementations/test_sftp.py +++ b/upath/tests/implementations/test_sftp.py @@ -1,43 +1,24 @@ import pytest from upath import UPath -from upath.tests.cases import BaseTests -from upath.tests.utils import skip_on_windows -from upath.tests.utils import xfail_if_version - -_xfail_old_fsspec = xfail_if_version( - "fsspec", - lt="2022.7.0", - reason="fsspec<2022.7.0 sftp does not support create_parents", -) +from upath.implementations.sftp import SFTPPath + +from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base +from ..utils import skip_on_windows @skip_on_windows -class TestUPathSFTP(BaseTests): +class TestUPathSFTP(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, ssh_fixture): self.path = UPath(ssh_fixture) - @_xfail_old_fsspec - def test_mkdir(self): - super().test_mkdir() - - @_xfail_old_fsspec - def test_mkdir_exists_ok_true(self): - super().test_mkdir_exists_ok_true() - - @_xfail_old_fsspec - def test_mkdir_exists_ok_false(self): - super().test_mkdir_exists_ok_false() - - @_xfail_old_fsspec - def test_mkdir_parents_true_exists_ok_false(self): - super().test_mkdir_parents_true_exists_ok_false() - - @_xfail_old_fsspec - def test_mkdir_parents_true_exists_ok_true(self): - super().test_mkdir_parents_true_exists_ok_true() + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, SFTPPath) @pytest.mark.parametrize( diff --git a/upath/tests/implementations/test_smb.py b/upath/tests/implementations/test_smb.py index f4046137..5e16cd44 100644 --- a/upath/tests/implementations/test_smb.py +++ b/upath/tests/implementations/test_smb.py @@ -1,19 +1,27 @@ import pytest -from fsspec import __version__ as fsspec_version -from packaging.version import Version from upath import UPath -from upath.tests.cases import BaseTests -from upath.tests.utils import skip_on_windows + +from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import overrides_base +from ..utils import skip_on_windows @skip_on_windows -class TestUPathSMB(BaseTests): +class TestUPathSMB(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, smb_fixture): self.path = UPath(smb_fixture) + @overrides_base + def test_is_correct_class(self): + from upath.implementations.smb import SMBPath + + assert isinstance(self.path, SMBPath) + + @overrides_base @pytest.mark.parametrize( "pattern", ( @@ -24,14 +32,7 @@ def path(self, smb_fixture): reason="SMBFileSystem.info appends '/' to dirs" ), ), - pytest.param( - "**/*.txt", - marks=( - pytest.mark.xfail(reason="requires fsspec>=2023.9.0") - if Version(fsspec_version) < Version("2023.9.0") - else () - ), - ), + "**/*.txt", ), ) def test_glob(self, pathlib_base, pattern): diff --git a/upath/tests/implementations/test_tar.py b/upath/tests/implementations/test_tar.py index c528a7e2..6430b252 100644 --- a/upath/tests/implementations/test_tar.py +++ b/upath/tests/implementations/test_tar.py @@ -5,7 +5,11 @@ from upath import UPath from upath.implementations.tar import TarPath -from ..cases import BaseTests +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import overrides_base @pytest.fixture(scope="function") @@ -17,84 +21,22 @@ def tarred_testdir_file(local_testdir, tmp_path_factory): return str(tar_path) -class TestTarPath(BaseTests): +class TestTarPath( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): @pytest.fixture(autouse=True) def path(self, tarred_testdir_file): self.path = UPath("tar://", fo=tarred_testdir_file) # self.prepare_file_system() done outside of UPath - def test_is_TarPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, TarPath) - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_mkdir(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_mkdir_exists_ok_false(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_mkdir_parents_true_exists_ok_false(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_rename(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_rename2(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_touch(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_touch_unlink(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_write_bytes(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_write_text(self): - pass - - @pytest.mark.skip(reason="Tar filesystem is read-only") - def test_fsspec_compat(self): - pass - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_move_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_move_into_local(self, tmp_path): - pass - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_move_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_move_into_memory(self, clear_fsspec_memory_cache): - pass - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_rename_with_target_absolute(self, target_factory): - return super().test_rename_with_target_str_absolute(target_factory) - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_write_text_encoding(self): - return super().test_write_text_encoding() - - @pytest.mark.skip(reason="Only testing read on TarPath") - def test_write_text_errors(self): - return super().test_write_text_errors() - @pytest.fixture(scope="function") def tarred_testdir_file_in_memory(tarred_testdir_file, clear_fsspec_memory_cache): diff --git a/upath/tests/implementations/test_webdav.py b/upath/tests/implementations/test_webdav.py index 9ac9778c..36543edc 100644 --- a/upath/tests/implementations/test_webdav.py +++ b/upath/tests/implementations/test_webdav.py @@ -3,33 +3,30 @@ import pytest from upath import UPath +from upath.implementations.webdav import WebdavPath from ..cases import BaseTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base -class TestUPathWebdav(BaseTests): +class TestUPathWebdav(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True, scope="function") def path(self, webdav_fixture): self.path = UPath(webdav_fixture, auth=("USER", "PASSWORD")) - def test_fsspec_compat(self): - pass + @overrides_base + def test_is_correct_class(self): + assert isinstance(self.path, WebdavPath) - def test_storage_options(self): - # we need to add base_url to storage options for webdav filesystems, - # to be able to serialize the http protocol to string... - storage_options = self.path.storage_options - base_url = storage_options["base_url"] - assert storage_options == self.path.fs.storage_options + @extends_base + def test_storage_options_base_url(self): + # ensure that base_url is correct + base_url = self.path.storage_options["base_url"] assert base_url == self.path.fs.client.base_url - def test_read_with_fsspec(self): - # this test used to fail with fsspec<2022.5.0 because webdav was not - # registered in fsspec. But when UPath(webdav_fixture) is called, to - # run the BaseTests, the upath.implementations.webdav module is - # imported, which registers the webdav implementation in fsspec. - super().test_read_with_fsspec() - + @overrides_base @pytest.mark.parametrize( "target_factory", [ diff --git a/upath/tests/implementations/test_zip.py b/upath/tests/implementations/test_zip.py index 28779e32..61a60be5 100644 --- a/upath/tests/implementations/test_zip.py +++ b/upath/tests/implementations/test_zip.py @@ -3,10 +3,16 @@ import pytest +from upath import UnsupportedOperation from upath import UPath from upath.implementations.zip import ZipPath -from ..cases import BaseTests +from ..cases import JoinablePathTests +from ..cases import NonWritablePathTests +from ..cases import ReadablePathTests +from ..utils import OverrideMeta +from ..utils import extends_base +from ..utils import overrides_base @pytest.fixture(scope="function") @@ -33,8 +39,12 @@ def empty_zipped_testdir_file(tmp_path): return str(zip_path) -class TestZipPath(BaseTests): - +class TestZipPath( + JoinablePathTests, + ReadablePathTests, + NonWritablePathTests, + metaclass=OverrideMeta, +): @pytest.fixture(autouse=True) def path(self, zipped_testdir_file, request): try: @@ -47,115 +57,18 @@ def path(self, zipped_testdir_file, request): finally: self.path.fs.clear_instance_cache() - def test_is_ZipPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, ZipPath) - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_mkdir(self): - super().test_mkdir() - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_mkdir_exists_ok_true(self): - super().test_mkdir_exists_ok_true() - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_mkdir_exists_ok_false(self): - super().test_mkdir_exists_ok_false() - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_mkdir_parents_true_exists_ok_true(self): - super().test_mkdir_parents_true_exists_ok_true() - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_mkdir_parents_true_exists_ok_false(self): - super().test_mkdir_parents_true_exists_ok_false() - - def test_rename(self): - with pytest.raises(NotImplementedError): - super().test_rename() # delete is not implemented in fsspec - - def test_move_local(self, tmp_path): - with pytest.raises(NotImplementedError): - super().test_move_local(tmp_path) # delete is not implemented in fsspec - - def test_move_into_local(self, tmp_path): - with pytest.raises(NotImplementedError): - super().test_move_into_local( - tmp_path - ) # delete is not implemented in fsspec - - def test_move_memory(self, clear_fsspec_memory_cache): - with pytest.raises(NotImplementedError): - super().test_move_memory(clear_fsspec_memory_cache) - - def test_move_into_memory(self, clear_fsspec_memory_cache): - with pytest.raises(NotImplementedError): - super().test_move_into_memory(clear_fsspec_memory_cache) - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_touch(self): - super().test_touch() - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_touch_unlink(self): - with pytest.raises(NotImplementedError): - super().test_touch_unlink() # delete is not implemented in fsspec - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_write_bytes(self): - fn = "test_write_bytes.txt" - s = b"hello_world" - path = self.path.joinpath(fn) - path.write_bytes(s) - so = {**path.storage_options, "mode": "r"} - urlpath = str(path) - path.fs.close() - assert UPath(urlpath, **so).read_bytes() == s - - @pytest.mark.parametrize( - "path", [("w",)], ids=["zipfile_mode_write"], indirect=True - ) - def test_write_text(self): - fn = "test_write_text.txt" - s = "hello_world" - path = self.path.joinpath(fn) - path.write_text(s) - so = {**path.storage_options, "mode": "r"} - urlpath = str(path) - path.fs.close() - assert UPath(urlpath, **so).read_text() == s - - @pytest.mark.skip(reason="fsspec zipfile filesystem is either read xor write mode") - def test_fsspec_compat(self): - pass - - @pytest.mark.skip(reason="fsspec zipfile filesystem is either read xor write mode") - def test_rename_with_target_absolute(self, target_factory): - return super().test_rename_with_target_absolute(target_factory) - - @pytest.mark.skip(reason="fsspec zipfile filesystem is either read xor write mode") - def test_write_text_encoding(self): - return super().test_write_text_encoding() - - @pytest.mark.skip(reason="fsspec zipfile filesystem is either read xor write mode") - def test_write_text_errors(self): - return super().test_write_text_errors() + @extends_base + def test_write_mode_is_disabled(self, tmp_path): + with pytest.raises(UnsupportedOperation): + UPath("zip://", fo=tmp_path.joinpath("myzip.zip"), mode="a") + with pytest.raises(UnsupportedOperation): + UPath("zip://", fo=tmp_path.joinpath("myzip.zip"), mode="x") + with pytest.raises(UnsupportedOperation): + UPath("zip://", fo=tmp_path.joinpath("myzip.zip"), mode="w") @pytest.fixture(scope="function") diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index ef8138de..822cc8c7 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -15,7 +15,9 @@ from upath.types import WritablePath from .cases import BaseTests +from .utils import OverrideMeta from .utils import only_on_windows +from .utils import overrides_base from .utils import skip_on_windows @@ -43,7 +45,7 @@ def test_UPath_file_protocol_no_warning(): assert len(w) == 0 -class TestUpath(BaseTests): +class TestUpath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): with warnings.catch_warnings(): @@ -55,29 +57,20 @@ def path(self, local_testdir): root = "/" if sys.platform.startswith("win") else "" self.path = UPath(f"mock:{root}{local_testdir}") - def test_fsspec_compat(self): - pass - - def test_cwd(self): - with pytest.raises( - NotImplementedError, - match=r".+Path[.]cwd\(\) is unsupported", - ): - type(self.path).cwd() + @overrides_base + def test_is_correct_class(self): + # testing dynamically created UPath classes + from upath.implementations._experimental import _MockPath - def test_home(self): - with pytest.raises( - NotImplementedError, - match=r".+Path[.]home\(\) is unsupported", - ): - type(self.path).home() + assert isinstance(self.path, _MockPath) + @overrides_base @pytest.mark.skipif( sys.platform.startswith("win"), reason="mock fs is not well defined on windows", ) def test_parents_are_absolute(self): - return super().test_parents_are_absolute() + super().test_parents_are_absolute() def test_multiple_backend_paths(local_testdir): diff --git a/upath/tests/test_extensions.py b/upath/tests/test_extensions.py index fdd802a5..7a3bc8c8 100644 --- a/upath/tests/test_extensions.py +++ b/upath/tests/test_extensions.py @@ -11,10 +11,14 @@ from upath.implementations.local import PosixUPath from upath.implementations.local import WindowsUPath from upath.implementations.memory import MemoryPath -from upath.tests.cases import BaseTests +from .cases import BaseTests +from .utils import OverrideMeta +from .utils import extends_base +from .utils import overrides_base -class TestProxyMemoryPath(BaseTests): + +class TestProxyMemoryPath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): if not local_testdir.startswith("/"): @@ -22,49 +26,60 @@ def path(self, local_testdir): self.path = ProxyUPath(f"memory:{local_testdir}") self.prepare_file_system() - def test_is_ProxyUPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, ProxyUPath) - def test_is_not_MemoryPath(self): + @extends_base + def test_is_not_wrapped_class(self): assert not isinstance(self.path, MemoryPath) -class TestProxyFilePath(BaseTests): +class TestProxyFilePath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): self.path = ProxyUPath(f"file://{local_testdir}") self.prepare_file_system() - def test_is_ProxyUPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, ProxyUPath) - def test_is_not_FilePath(self): + @extends_base + def test_is_not_wrapped_class(self): assert not isinstance(self.path, FilePath) + @overrides_base def test_chmod(self): self.path.joinpath("file1.txt").chmod(777) + @overrides_base def test_cwd(self): + # ProxyUPath.cwd() works differently on the instance self.path.cwd() with pytest.raises(UnsupportedOperation): type(self.path).cwd() -class TestProxyPathlibPath(BaseTests): +class TestProxyPathlibPath(BaseTests, metaclass=OverrideMeta): @pytest.fixture(autouse=True) def path(self, local_testdir): self.path = ProxyUPath(f"{local_testdir}") self.prepare_file_system() - def test_is_ProxyUPath(self): + @overrides_base + def test_is_correct_class(self): assert isinstance(self.path, ProxyUPath) - def test_is_not_PosixUPath_WindowsUPath(self): + @extends_base + def test_is_not_wrapped_class(self): assert not isinstance(self.path, (PosixUPath, WindowsUPath)) + @overrides_base def test_chmod(self): self.path.joinpath("file1.txt").chmod(777) + @overrides_base @pytest.mark.skipif( sys.version_info < (3, 12), reason="storage options only handled in 3.12+" ) @@ -73,6 +88,7 @@ def test_eq(self): if sys.version_info < (3, 12): + @overrides_base def test_storage_options_dont_affect_hash(self): # On Python < 3.12, storage_options trigger warnings for LocalPath with pytest.warns( @@ -81,14 +97,17 @@ def test_storage_options_dont_affect_hash(self): ): super().test_storage_options_dont_affect_hash() + @overrides_base def test_group(self): pytest.importorskip("grp") self.path.group() + @overrides_base def test_owner(self): pytest.importorskip("pwd") self.path.owner() + @overrides_base def test_readlink(self): try: os.readlink @@ -97,14 +116,17 @@ def test_readlink(self): with pytest.raises((OSError, UnsupportedOperation)): self.path.readlink() + @overrides_base def test_protocol(self): assert self.path.protocol == "" + @overrides_base def test_as_uri(self): assert self.path.as_uri().startswith("file://") if sys.version_info < (3, 10): + @overrides_base def test_lstat(self): # On Python < 3.10, stat(follow_symlinks=False) triggers warnings with pytest.warns( @@ -116,21 +138,25 @@ def test_lstat(self): else: + @overrides_base def test_lstat(self): st = self.path.lstat() assert st is not None + @overrides_base def test_relative_to(self): base = self.path child = self.path / "folder1" / "file1.txt" relative = child.relative_to(base) assert str(relative) == f"folder1{os.sep}file1.txt" + @overrides_base def test_cwd(self): self.path.cwd() with pytest.raises(UnsupportedOperation): type(self.path).cwd() + @overrides_base def test_lchmod(self): # setup a = self.path.joinpath("a") @@ -146,9 +172,11 @@ def test_lchmod(self): with cm: b.lchmod(mode=0o777) + @overrides_base def test_symlink_to(self): self.path.joinpath("link").symlink_to(self.path) + @overrides_base def test_hardlink_to(self): try: self.path.joinpath("link").hardlink_to(self.path) diff --git a/upath/tests/utils.py b/upath/tests/utils.py index f69fe2bd..5900ffbe 100644 --- a/upath/tests/utils.py +++ b/upath/tests/utils.py @@ -63,3 +63,82 @@ def temporary_register(protocol, cls): finally: m.clear() get_upath_class.cache_clear() + + +def extends_base(method): + """Decorator to ensure a method extends the base class and does NOT + override a method in base classes. + + Use this decorator in implementation-specific test classes to ensure that + test methods don't accidentally override methods defined in test base classes. + + Example: + class TestSpecificImpl(TestBaseClass, metaclass=OverrideMeta): + @extends_base + def test_something(self): # Raises TypeError if base has this method + ... + + @extends_base + def test_new_method(self): # This is fine - no override + ... + """ + method.__override_check__ = False + return method + + +def overrides_base(method): + """Decorator to ensure a method DOES override a method in base classes. + + Use this decorator in implementation-specific test classes to ensure that + test methods intentionally override methods defined in test base classes. + + Example: + class TestSpecificImpl(TestBaseClass, metaclass=OverrideMeta): + @overrides_base + def test_something(self): # Raises TypeError if base lacks this method + ... + + @overrides_base + def test_new_method(self): # Raises TypeError - no method to override + ... + """ + method.__override_check__ = True + return method + + +class OverrideMeta(type): + """Metaclass that enforces @extends_base and @overrides_base decorator constraints. + + When a class uses this metaclass: + - Methods decorated with @extends_base are checked to ensure they don't + override a method from any base class. + - Methods decorated with @overrides_base are checked to ensure they do + override a method from at least one base class. + """ + + def __new__(mcs, name, bases, namespace): + for attr_name, attr_value in namespace.items(): + if not callable(attr_value): + continue + + check = getattr(attr_value, "__override_check__", None) + if check is None: + continue + + has_in_base = any(hasattr(base, attr_name) for base in bases) + + if check is False and has_in_base: + base_name = next(b.__name__ for b in bases if hasattr(b, attr_name)) + raise TypeError( + f"Method '{attr_name}' in class '{name}' is decorated " + f"with @extends_base but overrides a method from base " + f"class '{base_name}'" + ) + elif check is True and not has_in_base: + raise TypeError( + f"Method '{attr_name}' in class '{name}' is decorated " + f"with @overrides_base but does not override any method from " + f"base classes" + ) + + return super().__new__(mcs, name, bases, namespace)