From ac928ef8ed7842d6ae9663372313a04111cfc8f7 Mon Sep 17 00:00:00 2001 From: darrenhuai <60621295+darrenhuai@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:04:53 -0700 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=20fix(discovery):=20stop=20a?= =?UTF-8?q?=20non-UTF-8=20or=20unreadable=20.python-version=20aborting=20d?= =?UTF-8?q?iscovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolving a pyenv, mise or asdf shim reads .python-version from the working directory up to the filesystem root, then $PYENV_ROOT/version, and each file was opened as strict UTF-8 with no error handling. A file holding a byte that is not UTF-8, such as a Latin-1 comment or the UTF-16 that Windows PowerShell 5.1 writes for `"3.12" > .python-version`, raised UnicodeDecodeError, and one the user may not read raised PermissionError. Both escaped get_interpreter and iter_interpreters, so virtualenv and tox stopped with a traceback for anyone with a shim on PATH who asked for an interpreter other than the running one. pyenv reads the file as bytes, so it takes the version from the Latin-1 file without complaint. The file is now decoded with errors="replace", so a byte that does not decode no longer hides the versions around it, the same result pyenv's reader gives. A file that cannot be opened or read is treated like one that names no version: it is logged at debug level and the search moves on, as the rest of the module does with filesystem errors. --- .../+unreadable-python-version.bugfix.rst | 2 ++ src/python_discovery/_discovery.py | 16 ++++++++++---- tests/test_discovery.py | 19 ++++++++++++++++ tests/test_discovery_extra.py | 22 +++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/+unreadable-python-version.bugfix.rst diff --git a/docs/changelog/+unreadable-python-version.bugfix.rst b/docs/changelog/+unreadable-python-version.bugfix.rst new file mode 100644 index 0000000..fa17572 --- /dev/null +++ b/docs/changelog/+unreadable-python-version.bugfix.rst @@ -0,0 +1,2 @@ +Stop a ``.python-version`` file that is not valid UTF-8, or that cannot be read, from aborting discovery through a +pyenv, mise or asdf shim; the first is read the way pyenv reads it, and the second is skipped like a missing file. diff --git a/src/python_discovery/_discovery.py b/src/python_discovery/_discovery.py index 65a12de..476474a 100644 --- a/src/python_discovery/_discovery.py +++ b/src/python_discovery/_discovery.py @@ -533,10 +533,8 @@ def _read_python_version_file(start: str | Path, *, search_parents: bool = True) current = start while True: candidate = os.path.join(current, ".python-version") if Path(current).is_dir() else current - if Path(candidate).is_file(): - with Path(candidate).open(encoding="utf-8") as fh: - if versions := [v for line in fh if (v := line.strip()) and not v.startswith("#")]: - return versions + if Path(candidate).is_file() and (versions := _versions_in_file(candidate)): + return versions if not search_parents: return None parent = Path(current).parent @@ -545,6 +543,16 @@ def _read_python_version_file(start: str | Path, *, search_parents: bool = True) current = parent +def _versions_in_file(path: str | Path) -> list[str]: + try: + # pyenv reads raw bytes, so a byte that is not UTF-8 (a Latin-1 comment, say) must not hide the versions + with Path(path).open(encoding="utf-8", errors="replace") as fh: + return [v for line in fh if (v := line.strip()) and not v.startswith("#")] + except OSError: # skip a file we cannot read, the same as one that names no version + _LOGGER.debug("failed to read %s", path, exc_info=True) + return [] + + class PathPythonInfo(PythonInfo): """python info from path.""" diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 5d464ef..0349c54 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -358,6 +358,25 @@ def test_shim_uses_python_version_file(tmp_path: Path, monkeypatch: pytest.Monke assert mock_from_exe.call_args_list[0][0][0] == str(real_binary) +def test_shim_python_version_file_not_utf8(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = _create_version_manager(tmp_path, "PYENV_ROOT") + _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") + shim = root / "shims" / ("python2.7.exe" if IS_WIN else "python2.7") + shim.touch(mode=0o755) + # what `"2.7.18" > .python-version` writes in Windows PowerShell 5.1 + (tmp_path / ".python-version").write_bytes("2.7.18\r\n".encode("utf-16")) + + monkeypatch.setenv("PATH", str(root / "shims")) + monkeypatch.setenv("PYENV_ROOT", str(root)) + monkeypatch.delenv("PYENV_VERSION", raising=False) + monkeypatch.chdir(tmp_path) + + with patch("python_discovery._discovery.PathPythonInfo.from_exe") as mock_from_exe: + mock_from_exe.return_value = None + get_interpreter("python2.7", []) + assert mock_from_exe.call_args_list[0][0][0] == str(shim) + + def test_shim_pyenv_version_env_takes_priority_over_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: root = _create_version_manager(tmp_path, "PYENV_ROOT") _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") diff --git a/tests/test_discovery_extra.py b/tests/test_discovery_extra.py index 2c17fd3..90bf574 100644 --- a/tests/test_discovery_extra.py +++ b/tests/test_discovery_extra.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import sys from pathlib import Path @@ -230,6 +231,27 @@ def test_read_python_version_file_empty_versions(tmp_path: Path) -> None: assert result is None +def test_read_python_version_file_not_utf8(tmp_path: Path) -> None: + (tmp_path / ".python-version").write_bytes(b"# pinned by M\xfcller\n3.12.0\n") + result = _read_python_version_file(str(tmp_path), search_parents=False) + assert result == ["3.12.0"] + + +def test_read_python_version_file_unreadable_is_skipped( + tmp_path: Path, mocker: MockerFixture, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.DEBUG) + child = tmp_path / "child" + child.mkdir() + (child / ".python-version").write_text("3.12.0\n", encoding="utf-8") + (tmp_path / ".python-version").write_text("3.11.0\n", encoding="utf-8") + with (tmp_path / ".python-version").open(encoding="utf-8") as parent_file: + mocker.patch.object(Path, "open", side_effect=[PermissionError("Permission denied"), parent_file]) + result = _read_python_version_file(str(child)) + assert result == ["3.11.0"] + assert f"failed to read {child / '.python-version'}" in caplog.messages + + def test_get_interpreter_multi_spec_all_fail(session_cache: DiskCache) -> None: result = get_interpreter(["magic-one", "magic-two"], cache=session_cache) assert result is None From 45244b5a83e0a67a9d7257fb9329dbb20c18bc01 Mon Sep 17 00:00:00 2001 From: darrenhuai <60621295+darrenhuai@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:06:20 -0700 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20fix(discovery):=20skip=20dir?= =?UTF-8?q?ectories=20on=20the=20walk=20that=20can't=20be=20searched?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Python 3.9 to 3.13, Path.is_dir and Path.is_file re-raise a PermissionError from stat, so a working directory under a folder without search permission still aborted the .python-version walk. os.path.isdir and os.path.isfile return False there. The shim test now pins with a Latin-1 comment and checks that the pinned interpreter is probed. --- .../+unreadable-python-version.bugfix.rst | 3 ++- src/python_discovery/_discovery.py | 5 +++-- tests/test_discovery.py | 7 +++---- tests/test_discovery_extra.py | 17 ++++++++++++++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/changelog/+unreadable-python-version.bugfix.rst b/docs/changelog/+unreadable-python-version.bugfix.rst index fa17572..98cbeff 100644 --- a/docs/changelog/+unreadable-python-version.bugfix.rst +++ b/docs/changelog/+unreadable-python-version.bugfix.rst @@ -1,2 +1,3 @@ Stop a ``.python-version`` file that is not valid UTF-8, or that cannot be read, from aborting discovery through a -pyenv, mise or asdf shim; the first is read the way pyenv reads it, and the second is skipped like a missing file. +pyenv, mise or asdf shim; a stray byte no longer hides the versions it names, and an unreadable file is skipped like a +missing one. diff --git a/src/python_discovery/_discovery.py b/src/python_discovery/_discovery.py index 476474a..8511852 100644 --- a/src/python_discovery/_discovery.py +++ b/src/python_discovery/_discovery.py @@ -532,8 +532,9 @@ def _read_python_version_file(start: str | Path, *, search_parents: bool = True) """Read a ``.python-version`` file, optionally searching parent directories.""" current = start while True: - candidate = os.path.join(current, ".python-version") if Path(current).is_dir() else current - if Path(candidate).is_file() and (versions := _versions_in_file(candidate)): + # unlike pathlib before 3.14, os.path answers False when a directory on the way can't be searched + candidate = os.path.join(current, ".python-version") if os.path.isdir(current) else current + if os.path.isfile(candidate) and (versions := _versions_in_file(candidate)): return versions if not search_parents: return None diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 0349c54..8505d33 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -360,11 +360,10 @@ def test_shim_uses_python_version_file(tmp_path: Path, monkeypatch: pytest.Monke def test_shim_python_version_file_not_utf8(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: root = _create_version_manager(tmp_path, "PYENV_ROOT") - _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") + real_binary = _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") shim = root / "shims" / ("python2.7.exe" if IS_WIN else "python2.7") shim.touch(mode=0o755) - # what `"2.7.18" > .python-version` writes in Windows PowerShell 5.1 - (tmp_path / ".python-version").write_bytes("2.7.18\r\n".encode("utf-16")) + (tmp_path / ".python-version").write_bytes(b"# pinned by M\xfcller\n2.7.18\n") monkeypatch.setenv("PATH", str(root / "shims")) monkeypatch.setenv("PYENV_ROOT", str(root)) @@ -374,7 +373,7 @@ def test_shim_python_version_file_not_utf8(tmp_path: Path, monkeypatch: pytest.M with patch("python_discovery._discovery.PathPythonInfo.from_exe") as mock_from_exe: mock_from_exe.return_value = None get_interpreter("python2.7", []) - assert mock_from_exe.call_args_list[0][0][0] == str(shim) + assert mock_from_exe.call_args_list[0][0][0] == str(real_binary) def test_shim_pyenv_version_env_takes_priority_over_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_discovery_extra.py b/tests/test_discovery_extra.py index 90bf574..d067f76 100644 --- a/tests/test_discovery_extra.py +++ b/tests/test_discovery_extra.py @@ -249,7 +249,22 @@ def test_read_python_version_file_unreadable_is_skipped( mocker.patch.object(Path, "open", side_effect=[PermissionError("Permission denied"), parent_file]) result = _read_python_version_file(str(child)) assert result == ["3.11.0"] - assert f"failed to read {child / '.python-version'}" in caplog.messages + assert ("python_discovery._discovery", logging.DEBUG, f"failed to read {child / '.python-version'}") in ( + caplog.record_tuples + ) + + +@pytest.mark.skipif(IS_WIN or os.geteuid() == 0, reason="POSIX search permission that root bypasses") +def test_read_python_version_file_unsearchable_directory_is_skipped(tmp_path: Path) -> None: + (tmp_path / ".python-version").write_text("3.11.0\n", encoding="utf-8") + locked = tmp_path / "locked" + (locked / "work").mkdir(parents=True) + locked.chmod(0o600) + try: + result = _read_python_version_file(str(locked / "work")) + finally: + locked.chmod(0o700) + assert result == ["3.11.0"] def test_get_interpreter_multi_spec_all_fail(session_cache: DiskCache) -> None: From 4caa022d73adf0591e7d5e25980a8ec9fe11cd63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 30 Oct 2026 11:12:49 -0700 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A7=AA=20test(discovery):=20cover=20s?= =?UTF-8?q?hims=20via=20public=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercise the unreadable file and unsearchable directory through get_interpreter with real file permissions instead of calling the private reader and patching Path.open, and drop the reader-level Latin-1 test that the shim test already covers. Relabel the docs flowchart: a file that exists but names no readable version now falls through to the parent directories. --- docs/explanation.rst | 2 +- tests/test_discovery.py | 60 +++++++++++++++++++++++++++++------ tests/test_discovery_extra.py | 37 --------------------- 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index 125532b..1047892 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -52,7 +52,7 @@ detects these shims and resolves them to the actual binary. flowchart TD Shim["Shim detected"] --> EnvVar{"PYENV_VERSION
set?"} EnvVar -->|Yes| Use["Use that version"] - EnvVar -->|No| File{".python-version
file exists?"} + EnvVar -->|No| File{".python-version
names a version?"} File -->|Yes| Use File -->|No| Global{"pyenv global
version exists?"} Global -->|Yes| Use diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 8505d33..89f4d9b 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -260,6 +260,11 @@ def test_discovery_via_version_specifier(session_cache: DiskCache) -> None: assert interpreter.implementation == "CPython" +_POSIX_PERMISSIONS: Final[pytest.MarkDecorator] = pytest.mark.skipif( + IS_WIN or os.geteuid() == 0, reason="POSIX file permissions, which root bypasses" +) + + def _create_version_manager(tmp_path: Path, env_var: str) -> Path: root = tmp_path / env_var.lower() root.mkdir() @@ -358,22 +363,59 @@ def test_shim_uses_python_version_file(tmp_path: Path, monkeypatch: pytest.Monke assert mock_from_exe.call_args_list[0][0][0] == str(real_binary) -def test_shim_python_version_file_not_utf8(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - root = _create_version_manager(tmp_path, "PYENV_ROOT") - real_binary = _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") - shim = root / "shims" / ("python2.7.exe" if IS_WIN else "python2.7") - shim.touch(mode=0o755) +def test_shim_python_version_file_not_utf8( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, pyenv_probed: Callable[[], tuple[str, ...]] +) -> None: (tmp_path / ".python-version").write_bytes(b"# pinned by M\xfcller\n2.7.18\n") + monkeypatch.chdir(tmp_path) + + assert pyenv_probed() == ("versions", "2.7.18") + + +@_POSIX_PERMISSIONS +def test_shim_unreadable_python_version_file_is_skipped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, pyenv_probed: Callable[[], tuple[str, ...]] +) -> None: + (tmp_path / ".python-version").write_text("2.7.18\n", encoding="utf-8") + (unreadable := tmp_path / "child" / ".python-version").parent.mkdir() + unreadable.write_text("3.12.0\n", encoding="utf-8") + unreadable.chmod(0) + monkeypatch.chdir(unreadable.parent) + + assert pyenv_probed() == ("versions", "2.7.18") + +@_POSIX_PERMISSIONS +def test_shim_unsearchable_directory_is_skipped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, pyenv_probed: Callable[[], tuple[str, ...]] +) -> None: + (tmp_path / ".python-version").write_text("2.7.18\n", encoding="utf-8") + (work := tmp_path / "locked" / "work").mkdir(parents=True) + monkeypatch.chdir(work) + work.parent.chmod(0o600) + try: + assert pyenv_probed() == ("versions", "2.7.18") + finally: + work.parent.chmod(0o700) + + +@pytest.fixture +def pyenv_probed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture +) -> Callable[[], tuple[str, ...]]: + root = _create_version_manager(tmp_path, "PYENV_ROOT") + _create_versioned_binary(root, ("versions",), "2.7.18", "python2.7") + (root / "shims" / ("python2.7.exe" if IS_WIN else "python2.7")).touch(mode=0o755) monkeypatch.setenv("PATH", str(root / "shims")) monkeypatch.setenv("PYENV_ROOT", str(root)) monkeypatch.delenv("PYENV_VERSION", raising=False) - monkeypatch.chdir(tmp_path) + from_exe = mocker.patch("python_discovery._discovery.PathPythonInfo.from_exe", return_value=None) - with patch("python_discovery._discovery.PathPythonInfo.from_exe") as mock_from_exe: - mock_from_exe.return_value = None + def _probed() -> tuple[str, ...]: get_interpreter("python2.7", []) - assert mock_from_exe.call_args_list[0][0][0] == str(real_binary) + return Path(from_exe.call_args_list[0].args[0]).relative_to(root).parts[:2] + + return _probed def test_shim_pyenv_version_env_takes_priority_over_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_discovery_extra.py b/tests/test_discovery_extra.py index d067f76..2c17fd3 100644 --- a/tests/test_discovery_extra.py +++ b/tests/test_discovery_extra.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import os import sys from pathlib import Path @@ -231,42 +230,6 @@ def test_read_python_version_file_empty_versions(tmp_path: Path) -> None: assert result is None -def test_read_python_version_file_not_utf8(tmp_path: Path) -> None: - (tmp_path / ".python-version").write_bytes(b"# pinned by M\xfcller\n3.12.0\n") - result = _read_python_version_file(str(tmp_path), search_parents=False) - assert result == ["3.12.0"] - - -def test_read_python_version_file_unreadable_is_skipped( - tmp_path: Path, mocker: MockerFixture, caplog: pytest.LogCaptureFixture -) -> None: - caplog.set_level(logging.DEBUG) - child = tmp_path / "child" - child.mkdir() - (child / ".python-version").write_text("3.12.0\n", encoding="utf-8") - (tmp_path / ".python-version").write_text("3.11.0\n", encoding="utf-8") - with (tmp_path / ".python-version").open(encoding="utf-8") as parent_file: - mocker.patch.object(Path, "open", side_effect=[PermissionError("Permission denied"), parent_file]) - result = _read_python_version_file(str(child)) - assert result == ["3.11.0"] - assert ("python_discovery._discovery", logging.DEBUG, f"failed to read {child / '.python-version'}") in ( - caplog.record_tuples - ) - - -@pytest.mark.skipif(IS_WIN or os.geteuid() == 0, reason="POSIX search permission that root bypasses") -def test_read_python_version_file_unsearchable_directory_is_skipped(tmp_path: Path) -> None: - (tmp_path / ".python-version").write_text("3.11.0\n", encoding="utf-8") - locked = tmp_path / "locked" - (locked / "work").mkdir(parents=True) - locked.chmod(0o600) - try: - result = _read_python_version_file(str(locked / "work")) - finally: - locked.chmod(0o700) - assert result == ["3.11.0"] - - def test_get_interpreter_multi_spec_all_fail(session_cache: DiskCache) -> None: result = get_interpreter(["magic-one", "magic-two"], cache=session_cache) assert result is None