diff --git a/rope/base/libutils.py b/rope/base/libutils.py index 0974b11e7..f473d3274 100644 --- a/rope/base/libutils.py +++ b/rope/base/libutils.py @@ -32,11 +32,15 @@ def path_to_resource(project, path, type=None): def path_relative_to_project_root(project, path): - return relative(project.address, path) + return _relative(project.address, path) @utils.deprecated() def relative(root, path): + return _relative(root, path) + + +def _relative(root, path): root = rope.base.project._realpath(root).replace(os.path.sep, "/") path = rope.base.project._realpath(path).replace(os.path.sep, "/") if path == root: diff --git a/rope/base/project.py b/rope/base/project.py index c7f6a2f8e..0da2f776c 100644 --- a/rope/base/project.py +++ b/rope/base/project.py @@ -221,7 +221,9 @@ def __init__( self._init_prefs(prefs) if ropefolder is not None: self.prefs.add("ignored_resources", ropefolder) - self._init_source_folders() + for path in self.prefs.get("source_folders", []): + folder = self.get_resource(path) + self._custom_source_folders.append(folder) def __repr__(self): return '<{}.{} "{}">'.format( @@ -230,12 +232,6 @@ def __repr__(self): self.address, ) - @utils.deprecated("Delete once deprecated functions are gone") - def _init_source_folders(self): - for path in self.prefs.get("source_folders", []): - folder = self.get_resource(path) - self._custom_source_folders.append(folder) - # INFO: It was decided not to cache source folders, since: # - Does not take much time when the root folder contains # packages, that is most of the time diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index 7ae8fb718..9b2b81d21 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -4,6 +4,7 @@ import contextlib import json +import multiprocessing import re import secrets import sqlite3 @@ -11,11 +12,11 @@ import warnings from collections import OrderedDict from concurrent.futures import Future, ProcessPoolExecutor, as_completed -from datetime import datetime +from datetime import datetime, timezone from hashlib import sha256 from itertools import chain from pathlib import Path -from threading import local +from threading import Lock, local from typing import ( TYPE_CHECKING, Generator, @@ -55,11 +56,23 @@ from collections.abc import Collection +def _process_pool() -> ProcessPoolExecutor: + """Return a pool whose workers never come from forking this process. + + Forking a process that has threads (an editor, a language server or a + test runner usually do) can deadlock the child. ``forkserver`` (the + POSIX default since Python 3.14) or ``spawn`` start workers cleanly. + """ + methods = multiprocessing.get_all_start_methods() + method = "forkserver" if "forkserver" in methods else "spawn" + return ProcessPoolExecutor(mp_context=multiprocessing.get_context(method)) + + def get_future_names( packages: List[Package], underlined: bool, job_set: taskhandle.BaseJobSet ) -> Generator[Future[Collection[Name]], None, None]: """Get all names as futures.""" - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for package in packages: for module in get_files(package, underlined): job_set.started_job(module.modname) @@ -143,6 +156,9 @@ def __init__( DeprecationWarning, ) self.thread_local = local() + # Every thread gets its own connection; close() closes all of them. + self._connections: list[sqlite3.Connection] = [] + self._connections_lock = Lock() self.connection = self.create_database_connection( project=project, memory=memory, @@ -186,10 +202,14 @@ def calculate_project_hash(data: str) -> str: else: project_hash = calculate_project_hash(project.ropefolder.real_path) return sqlite3.connect( - f"file:rope-{project_hash}:?mode=memory&cache=shared", uri=True + f"file:rope-{project_hash}:?mode=memory&cache=shared", + uri=True, + check_same_thread=False, ) else: - return sqlite3.connect(project.ropefolder.pathlib / "autoimport.db") + return sqlite3.connect( + project.ropefolder.pathlib / "autoimport.db", check_same_thread=False + ) @property def connection(self) -> sqlite3.Connection: @@ -199,7 +219,7 @@ def connection(self) -> sqlite3.Connection: This makes sure AutoImport can be shared across threads. """ if not hasattr(self.thread_local, "connection"): - self.thread_local.connection = self.create_database_connection( + self.connection = self.create_database_connection( project=self.project, memory=self.memory, ) @@ -208,6 +228,8 @@ def connection(self) -> sqlite3.Connection: @connection.setter def connection(self, value: sqlite3.Connection): self.thread_local.connection = value + with self._connections_lock: + self._connections.append(value) def _setup_db(self): models.Metadata.create_table(self.connection) @@ -390,7 +412,7 @@ def generate_cache( models.Package.delete_by_package_name, (self.project_package.name,) ) futures = [] - with ProcessPoolExecutor() as executor: + with _process_pool() as executor: for file in resources: job_set.started_job(f"Working on {file.path}") module = self._resource_to_module(file, underlined) @@ -458,9 +480,12 @@ def update_module(self, module: str): self.generate_modules_cache([module]) def close(self): - """Close the autoimport database.""" - self.connection.commit() - self.connection.close() + """Close the autoimport database, including connections other threads opened.""" + with self._connections_lock: + connections, self._connections = self._connections, [] + for connection in connections: + connection.commit() + connection.close() def get_name_locations(self, name): """Return a list of ``(resource, lineno)`` tuples.""" @@ -503,7 +528,7 @@ def clear_cache(self): data = ( versioning.calculate_version_hash(self.project), json.dumps(versioning.get_version_hash_data(self.project)), - datetime.utcnow().isoformat(), + datetime.now(timezone.utc).isoformat(), ) assert models.Metadata.columns == [ "version_hash", diff --git a/ropetest/contrib/autoimport/autoimporttest.py b/ropetest/contrib/autoimport/autoimporttest.py index 072ab1ed8..5d9745a3e 100644 --- a/ropetest/contrib/autoimport/autoimporttest.py +++ b/ropetest/contrib/autoimport/autoimporttest.py @@ -14,7 +14,7 @@ @pytest.fixture def autoimport(project: Project): - with closing(AutoImport(project)) as ai: + with closing(AutoImport(project, memory=True)) as ai: yield ai @@ -28,11 +28,13 @@ def database_list(connection): return list(connection.execute("PRAGMA database_list")) -def test_in_memory_database_share_cache(project, project2): +def test_in_memory_database_share_cache(request, project, project2): ai_1 = AutoImport(project, memory=True) ai_2 = AutoImport(project, memory=True) ai_3 = AutoImport(project2, memory=True) + for ai in (ai_1, ai_2, ai_3): + request.addfinalizer(ai.close) with ai_1.connection: ai_1.connection.execute("CREATE TABLE shared(data)") @@ -46,16 +48,16 @@ def test_autoimport_connection_parameter_with_in_memory( project: Project, autoimport: AutoImport, ): - connection = AutoImport.create_database_connection(memory=True) - assert is_in_memory_database(connection) + with closing(AutoImport.create_database_connection(memory=True)) as connection: + assert is_in_memory_database(connection) def test_autoimport_connection_parameter_with_project( project: Project, autoimport: AutoImport, ): - connection = AutoImport.create_database_connection(project=project) - assert not is_in_memory_database(connection) + with closing(AutoImport.create_database_connection(project=project)) as connection: + assert not is_in_memory_database(connection) def test_autoimport_create_database_connection_conflicting_parameter( @@ -70,16 +72,16 @@ def test_autoimport_memory_parameter_is_true( project: Project, autoimport: AutoImport, ): - ai = AutoImport(project, memory=True) - assert is_in_memory_database(ai.connection) + with closing(AutoImport(project, memory=True)) as ai: + assert is_in_memory_database(ai.connection) def test_autoimport_memory_parameter_is_false( project: Project, autoimport: AutoImport, ): - ai = AutoImport(project, memory=False) - assert not is_in_memory_database(ai.connection) + with closing(AutoImport(project, memory=False)) as ai: + assert not is_in_memory_database(ai.connection) def test_init_py( @@ -115,21 +117,20 @@ def foo(): mod1.write(dedent("""\ foo """)) - autoimport = AutoImport(project, memory=False) - autoimport.generate_cache([mod1_init]) + with closing(AutoImport(project, memory=False)) as autoimport: + autoimport.generate_cache([mod1_init]) - tp = ThreadPoolExecutor(1) - results = tp.submit(autoimport.search, "foo", True).result() - assert [("from pkg1 import foo", "foo")] == results + with ThreadPoolExecutor(1) as tp: + results = tp.submit(autoimport.search, "foo", True).result() + assert [("from pkg1 import foo", "foo")] == results def test_connection(project: Project, project2: Project): - ai1 = AutoImport(project) - ai2 = AutoImport(project) - ai3 = AutoImport(project2) - - assert ai1.connection is not ai2.connection - assert ai1.connection is not ai3.connection + with closing(AutoImport(project, memory=True)) as ai1, closing( + AutoImport(project, memory=True) + ) as ai2, closing(AutoImport(project2, memory=True)) as ai3: + assert ai1.connection is not ai2.connection + assert ai1.connection is not ai3.connection @contextmanager diff --git a/ropetest/contrib/autoimport/conftest.py b/ropetest/contrib/autoimport/conftest.py index 9dca8ccf5..6c1d0d7dc 100644 --- a/ropetest/contrib/autoimport/conftest.py +++ b/ropetest/contrib/autoimport/conftest.py @@ -1,4 +1,7 @@ +import importlib.machinery +import importlib.util import pathlib +import sys import pytest @@ -43,6 +46,17 @@ def example_external_package_path( @pytest.fixture def compiled_lib(): - import _sqlite3 - - return "_sqlite3", pathlib.Path(_sqlite3.__file__) + """Return one stdlib extension module that this interpreter loads from a file. + + Which modules are shared libraries depends on how the interpreter was + built (python-build-standalone links _sqlite3 statically, so it has no + ``__file__``), so the module is discovered from the import system instead + of being hard-coded. + """ + for name in sorted(sys.stdlib_module_names): + spec = importlib.util.find_spec(name) + if spec is not None and isinstance( + spec.loader, importlib.machinery.ExtensionFileLoader + ): + return name, pathlib.Path(spec.origin) + pytest.fail("this interpreter loads no stdlib extension module from a file") diff --git a/ropetest/contrib/autoimport/modeltest.py b/ropetest/contrib/autoimport/modeltest.py index 0044434cc..027ed671b 100644 --- a/ropetest/contrib/autoimport/modeltest.py +++ b/ropetest/contrib/autoimport/modeltest.py @@ -1,4 +1,5 @@ import sqlite3 +from contextlib import closing from typing import Any import pytest @@ -8,7 +9,8 @@ @pytest.fixture def empty_db(): - return sqlite3.connect(":memory:") + with closing(sqlite3.connect(":memory:")) as connection: + yield connection class TestQuery: diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index d42ec14f7..3390839c8 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -1,4 +1,5 @@ import unittest +from contextlib import closing from rope.contrib.autoimport import sqlite as autoimport from rope.contrib.autoimport.defs import Alias @@ -12,9 +13,12 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=False) + self.importer = autoimport.AutoImport( + self.project, observe=False, memory=True + ) def tearDown(self): + self.importer.close() testutils.remove_project(self.project) super().tearDown() @@ -87,7 +91,8 @@ def test_not_caching_underlined_names(self): self.assertEqual(["mod1"], self.importer.get_modules("_myvar")) def test_caching_underlined_names_passing_to_the_constructor(self): - importer = autoimport.AutoImport(self.project, False, True) + importer = autoimport.AutoImport(self.project, False, True, memory=True) + self.addCleanup(importer.close) self.mod1.write("_myvar = None\n") importer.update_resource(self.mod1) self.assertEqual(["mod1"], importer.get_modules("_myvar")) @@ -178,12 +183,12 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self): def test_search_submodule(project, external_fixturepkg): - importer = autoimport.AutoImport(project, observe=False) - importer.update_module("external_fixturepkg") - import_statement = ("from external_fixturepkg import mod1", "mod1") - assert import_statement in importer.search("mod1", exact_match=True) - assert import_statement in importer.search("mo") - assert import_statement in importer.search("mod1") + with closing(autoimport.AutoImport(project, observe=False, memory=True)) as importer: + importer.update_module("external_fixturepkg") + import_statement = ("from external_fixturepkg import mod1", "mod1") + assert import_statement in importer.search("mod1", exact_match=True) + assert import_statement in importer.search("mo") + assert import_statement in importer.search("mod1") class AutoImportObservingTest(unittest.TestCase): @@ -193,9 +198,12 @@ def setUp(self): self.mod1 = testutils.create_module(self.project, "mod1") self.pkg = testutils.create_package(self.project, "pkg") self.mod2 = testutils.create_module(self.project, "mod2", self.pkg) - self.importer = autoimport.AutoImport(self.project, observe=True) + self.importer = autoimport.AutoImport( + self.project, observe=True, memory=True + ) def tearDown(self): + self.importer.close() testutils.remove_project(self.project) super().tearDown() diff --git a/ropetest/doatest.py b/ropetest/doatest.py index 3396cf163..5d8ba82fb 100644 --- a/ropetest/doatest.py +++ b/ropetest/doatest.py @@ -27,7 +27,13 @@ def try_CVE_2014_3539_exploit(self, receiver, payload): # Simulated attacker writing to the socket # Assume the attacker guesses the port correctly; 3037 is used by # default if it is available. - attacker_proc = multiprocessing.Process( + # Never fork this (multi-threaded) test process: see + # rope.contrib.autoimport.sqlite._process_pool. + methods = multiprocessing.get_all_start_methods() + context = multiprocessing.get_context( + "forkserver" if "forkserver" in methods else "spawn" + ) + attacker_proc = context.Process( target=cve_2014_3539_attacker, args=(receiver.data_port, payload) ) diff --git a/ropetest/projecttest.py b/ropetest/projecttest.py index 1f1b37d83..c0bcd28a4 100644 --- a/ropetest/projecttest.py +++ b/ropetest/projecttest.py @@ -5,6 +5,7 @@ from textwrap import dedent from unittest.mock import patch +from rope.base import libutils from rope.base.exceptions import ResourceNotFoundError, RopeError from rope.base.fscommands import FileSystemCommands from rope.base.libutils import path_to_resource @@ -24,6 +25,15 @@ def setUp(self): self._make_sample_project() self.no_project = NoProject() + def test_deprecated_relative_still_returns_the_project_relative_path(self): + with self.assertWarns(DeprecationWarning): + relative = libutils.relative(self.project_root, self.sample_path) + self.assertEqual( + libutils.path_relative_to_project_root(self.project, self.sample_path), + relative, + ) + self.assertEqual(self.sample_file, relative) + def _make_sample_project(self): self.sample_file = "sample_file.txt" self.sample_path = os.path.join(self.project_root, "sample_file.txt") @@ -115,9 +125,8 @@ def test_creating_file_inside_folder(self): file = self.project.get_resource(file_path) file.write("sample notes") self.assertEqual(file_path, file.path) - self.assertEqual( - "sample notes", open(os.path.join(self.project.address, file_path)).read() - ) + with open(os.path.join(self.project.address, file_path)) as written: + self.assertEqual("sample notes", written.read()) def test_failing_when_creating_file_inside_non_existent_folder(self): with self.assertRaises(ResourceNotFoundError): diff --git a/ropetest/refactor/__init__.py b/ropetest/refactor/__init__.py index 4d5bee1ad..4754d1438 100644 --- a/ropetest/refactor/__init__.py +++ b/ropetest/refactor/__init__.py @@ -1121,11 +1121,15 @@ def test_getting_job_name(self): # recommended name/job_name attribute self.assertEqual("test job set", jobs.name) self.assertEqual(None, jobs.job_name) - # deprecated getters - self.assertEqual("test job set", jobs.get_name()) - self.assertEqual(None, jobs.get_active_job_name()) + # deprecated getters still answer, and say they are deprecated + with self.assertWarns(DeprecationWarning): + self.assertEqual("test job set", jobs.get_name()) + with self.assertWarns(DeprecationWarning): + self.assertEqual(None, jobs.get_active_job_name()) jobs.started_job("job1") # recommended name/job_name attribute - self.assertEqual("test job set", jobs.get_name()) + self.assertEqual("test job set", jobs.name) + self.assertEqual("job1", jobs.job_name) # deprecated getters - self.assertEqual("job1", jobs.get_active_job_name()) + with self.assertWarns(DeprecationWarning): + self.assertEqual("job1", jobs.get_active_job_name()) diff --git a/ropetest/refactor/movetest.py b/ropetest/refactor/movetest.py index 2e467b82b..f0c285115 100644 --- a/ropetest/refactor/movetest.py +++ b/ropetest/refactor/movetest.py @@ -459,7 +459,7 @@ class TestClass: self.origin_module.write(code) with self.assertRaisesRegex( exceptions.RefactoringError, - "Move refactoring should be performed on a global class, function or variable\.", + r"Move refactoring should be performed on a global class, function or variable\.", ): mover = move.create_move( self.project, self.origin_module, code.index("CONSTANT") + 1 @@ -472,7 +472,7 @@ def a_func(): """)) with self.assertRaisesRegex( exceptions.RefactoringError, - "Moving global elements to the same module\.", + r"Moving global elements to the same module\.", ): self._move(self.origin_module, self.origin_module.read().index("a_func"), self.origin_module) diff --git a/ropetest/runmodtest.py b/ropetest/runmodtest.py index ae5234466..8cfdc67b7 100644 --- a/ropetest/runmodtest.py +++ b/ropetest/runmodtest.py @@ -22,11 +22,9 @@ def assert_collects_doa_data(project): @pytest.fixture def captured_run_module(project, tmp_path): def _run_module(file): - runner = project.pycore.run_module( - file, - stdout=stdout.open("w"), - ) - runner.wait_process() + with stdout.open("w") as output: + runner = project.pycore.run_module(file, stdout=output) + runner.wait_process() return stdout.read_text() stdout = tmp_path / "stdout" diff --git a/ropetest/testutils.py b/ropetest/testutils.py index 11f58ccfe..ace75b27f 100644 --- a/ropetest/testutils.py +++ b/ropetest/testutils.py @@ -102,8 +102,8 @@ def skipNotPOSIX(): def time_limit(timeout): if not any(procname in sys.argv[0] for procname in {"pytest", "py.test"}): - # no-op when running tests without pytest - return lambda *args, **kwargs: lambda func: func + # no-op when running tests without pytest: leave the test unchanged + return lambda func: func # do a local import so we don't import pytest when running without pytest import pytest