Fix the warnings the test suite emits at their source

rope/base/libutils.py

@@ -32,11 +32,15 @@ def path_to_resource(project, path, type=None):
 
 
 def path_relative_to_project_root(project, path):
-    return relative(project.address, path)
+    return _relative(project.address, path)
 
 
 @utils.deprecated()
 def relative(root, path):
+    return _relative(root, path)
+
+
+def _relative(root, path):
     root = rope.base.project._realpath(root).replace(os.path.sep, "/")
     path = rope.base.project._realpath(path).replace(os.path.sep, "/")
     if path == root:

rope/base/project.py

@@ -221,7 +221,9 @@ def __init__(
         self._init_prefs(prefs)
         if ropefolder is not None:
             self.prefs.add("ignored_resources", ropefolder)
-        self._init_source_folders()
+        for path in self.prefs.get("source_folders", []):
+            folder = self.get_resource(path)
+            self._custom_source_folders.append(folder)
 
     def __repr__(self):
         return '<{}.{} "{}">'.format(
@@ -230,12 +232,6 @@ def __repr__(self):
             self.address,
         )
 
-    @utils.deprecated("Delete once deprecated functions are gone")
-    def _init_source_folders(self):
-        for path in self.prefs.get("source_folders", []):
-            folder = self.get_resource(path)
-            self._custom_source_folders.append(folder)
-
     # INFO: It was decided not to cache source folders, since:
     #  - Does not take much time when the root folder contains
     #    packages, that is most of the time

rope/contrib/autoimport/sqlite.py

@@ -4,18 +4,19 @@
 
 import contextlib
 import json
+import multiprocessing
 import re
 import secrets
 import sqlite3
 import sys
 import warnings
 from collections import OrderedDict
 from concurrent.futures import Future, ProcessPoolExecutor, as_completed
-from datetime import datetime
+from datetime import datetime, timezone
 from hashlib import sha256
 from itertools import chain
 from pathlib import Path
-from threading import local
+from threading import Lock, local
 from typing import (
     TYPE_CHECKING,
     Generator,
@@ -55,11 +56,23 @@
     from collections.abc import Collection
 
 
+def _process_pool() -> ProcessPoolExecutor:
+    """Return a pool whose workers never come from forking this process.
+
+    Forking a process that has threads (an editor, a language server or a
+    test runner usually do) can deadlock the child. ``forkserver`` (the
+    POSIX default since Python 3.14) or ``spawn`` start workers cleanly.
+    """
+    methods = multiprocessing.get_all_start_methods()
+    method = "forkserver" if "forkserver" in methods else "spawn"
+    return ProcessPoolExecutor(mp_context=multiprocessing.get_context(method))
+
+
 def get_future_names(
     packages: List[Package], underlined: bool, job_set: taskhandle.BaseJobSet
 ) -> Generator[Future[Collection[Name]], None, None]:
     """Get all names as futures."""
-    with ProcessPoolExecutor() as executor:
+    with _process_pool() as executor:
         for package in packages:
             for module in get_files(package, underlined):
                 job_set.started_job(module.modname)
@@ -143,6 +156,9 @@ def __init__(
                 DeprecationWarning,
             )
         self.thread_local = local()
+        # Every thread gets its own connection; close() closes all of them.
+        self._connections: list[sqlite3.Connection] = []
+        self._connections_lock = Lock()
         self.connection = self.create_database_connection(
             project=project,
             memory=memory,
@@ -186,10 +202,14 @@ def calculate_project_hash(data: str) -> str:
             else:
                 project_hash = calculate_project_hash(project.ropefolder.real_path)
             return sqlite3.connect(
-                f"file:rope-{project_hash}:?mode=memory&cache=shared", uri=True
+                f"file:rope-{project_hash}:?mode=memory&cache=shared",
+                uri=True,
+                check_same_thread=False,
             )
         else:
-            return sqlite3.connect(project.ropefolder.pathlib / "autoimport.db")
+            return sqlite3.connect(
+                project.ropefolder.pathlib / "autoimport.db", check_same_thread=False
+            )
 
     @property
     def connection(self) -> sqlite3.Connection:
@@ -199,7 +219,7 @@ def connection(self) -> sqlite3.Connection:
         This makes sure AutoImport can be shared across threads.
         """
         if not hasattr(self.thread_local, "connection"):
-            self.thread_local.connection = self.create_database_connection(
+            self.connection = self.create_database_connection(
                 project=self.project,
                 memory=self.memory,
             )
@@ -208,6 +228,8 @@ def connection(self) -> sqlite3.Connection:
     @connection.setter
     def connection(self, value: sqlite3.Connection):
         self.thread_local.connection = value
+        with self._connections_lock:
+            self._connections.append(value)
 
     def _setup_db(self):
         models.Metadata.create_table(self.connection)
@@ -390,7 +412,7 @@ def generate_cache(
             models.Package.delete_by_package_name, (self.project_package.name,)
         )
         futures = []
-        with ProcessPoolExecutor() as executor:
+        with _process_pool() as executor:
             for file in resources:
                 job_set.started_job(f"Working on {file.path}")
                 module = self._resource_to_module(file, underlined)
@@ -458,9 +480,12 @@ def update_module(self, module: str):
         self.generate_modules_cache([module])
 
     def close(self):
-        """Close the autoimport database."""
-        self.connection.commit()
-        self.connection.close()
+        """Close the autoimport database, including connections other threads opened."""
+        with self._connections_lock:
+            connections, self._connections = self._connections, []
+        for connection in connections:
+            connection.commit()
+            connection.close()
 
     def get_name_locations(self, name):
         """Return a list of ``(resource, lineno)`` tuples."""
@@ -503,7 +528,7 @@ def clear_cache(self):
             data = (
                 versioning.calculate_version_hash(self.project),
                 json.dumps(versioning.get_version_hash_data(self.project)),
-                datetime.utcnow().isoformat(),
+                datetime.now(timezone.utc).isoformat(),
             )
             assert models.Metadata.columns == [
                 "version_hash",

ropetest/contrib/autoimport/autoimporttest.py

@@ -14,7 +14,7 @@
 
 @pytest.fixture
 def autoimport(project: Project):
-    with closing(AutoImport(project)) as ai:
+    with closing(AutoImport(project, memory=True)) as ai:
         yield ai
 
 
@@ -28,11 +28,13 @@ def database_list(connection):
     return list(connection.execute("PRAGMA database_list"))
 
 
-def test_in_memory_database_share_cache(project, project2):
+def test_in_memory_database_share_cache(request, project, project2):
     ai_1 = AutoImport(project, memory=True)
     ai_2 = AutoImport(project, memory=True)
 
     ai_3 = AutoImport(project2, memory=True)
+    for ai in (ai_1, ai_2, ai_3):
+        request.addfinalizer(ai.close)
 
     with ai_1.connection:
         ai_1.connection.execute("CREATE TABLE shared(data)")
@@ -46,16 +48,16 @@ def test_autoimport_connection_parameter_with_in_memory(
     project: Project,
     autoimport: AutoImport,
 ):
-    connection = AutoImport.create_database_connection(memory=True)
-    assert is_in_memory_database(connection)
+    with closing(AutoImport.create_database_connection(memory=True)) as connection:
+        assert is_in_memory_database(connection)
 
 
 def test_autoimport_connection_parameter_with_project(
     project: Project,
     autoimport: AutoImport,
 ):
-    connection = AutoImport.create_database_connection(project=project)
-    assert not is_in_memory_database(connection)
+    with closing(AutoImport.create_database_connection(project=project)) as connection:
+        assert not is_in_memory_database(connection)
 
 
 def test_autoimport_create_database_connection_conflicting_parameter(
@@ -70,16 +72,16 @@ def test_autoimport_memory_parameter_is_true(
     project: Project,
     autoimport: AutoImport,
 ):
-    ai = AutoImport(project, memory=True)
-    assert is_in_memory_database(ai.connection)
+    with closing(AutoImport(project, memory=True)) as ai:
+        assert is_in_memory_database(ai.connection)
 
 
 def test_autoimport_memory_parameter_is_false(
     project: Project,
     autoimport: AutoImport,
 ):
-    ai = AutoImport(project, memory=False)
-    assert not is_in_memory_database(ai.connection)
+    with closing(AutoImport(project, memory=False)) as ai:
+        assert not is_in_memory_database(ai.connection)
 
 
 def test_init_py(
@@ -115,21 +117,20 @@ def foo():
     mod1.write(dedent("""\
         foo
     """))
-    autoimport = AutoImport(project, memory=False)
-    autoimport.generate_cache([mod1_init])
+    with closing(AutoImport(project, memory=False)) as autoimport:
+        autoimport.generate_cache([mod1_init])
 
-    tp = ThreadPoolExecutor(1)
-    results = tp.submit(autoimport.search, "foo", True).result()
-    assert [("from pkg1 import foo", "foo")] == results
+        with ThreadPoolExecutor(1) as tp:
+            results = tp.submit(autoimport.search, "foo", True).result()
+        assert [("from pkg1 import foo", "foo")] == results
 
 
 def test_connection(project: Project, project2: Project):
-    ai1 = AutoImport(project)
-    ai2 = AutoImport(project)
-    ai3 = AutoImport(project2)
-
-    assert ai1.connection is not ai2.connection
-    assert ai1.connection is not ai3.connection
+    with closing(AutoImport(project, memory=True)) as ai1, closing(
+        AutoImport(project, memory=True)
+    ) as ai2, closing(AutoImport(project2, memory=True)) as ai3:
+        assert ai1.connection is not ai2.connection
+        assert ai1.connection is not ai3.connection
 
 
 @contextmanager

ropetest/contrib/autoimport/conftest.py

@@ -1,4 +1,7 @@
+import importlib.machinery
+import importlib.util
 import pathlib
+import sys
 
 import pytest
 
@@ -43,6 +46,17 @@ def example_external_package_path(
 
 @pytest.fixture
 def compiled_lib():
-    import _sqlite3
-
-    return "_sqlite3", pathlib.Path(_sqlite3.__file__)
+    """Return one stdlib extension module that this interpreter loads from a file.
+
+    Which modules are shared libraries depends on how the interpreter was
+    built (python-build-standalone links _sqlite3 statically, so it has no
+    ``__file__``), so the module is discovered from the import system instead
+    of being hard-coded.
+    """
+    for name in sorted(sys.stdlib_module_names):
+        spec = importlib.util.find_spec(name)
+        if spec is not None and isinstance(
+            spec.loader, importlib.machinery.ExtensionFileLoader
+        ):
+            return name, pathlib.Path(spec.origin)
+    pytest.fail("this interpreter loads no stdlib extension module from a file")

ropetest/contrib/autoimport/modeltest.py

@@ -1,4 +1,5 @@
 import sqlite3
+from contextlib import closing
 from typing import Any
 
 import pytest
@@ -8,7 +9,8 @@
 
 @pytest.fixture
 def empty_db():
-    return sqlite3.connect(":memory:")
+    with closing(sqlite3.connect(":memory:")) as connection:
+        yield connection
 
 
 class TestQuery:

ropetest/contrib/autoimporttest.py

@@ -1,4 +1,5 @@
 import unittest
+from contextlib import closing
 
 from rope.contrib.autoimport import sqlite as autoimport
 from rope.contrib.autoimport.defs import Alias
@@ -12,9 +13,12 @@ def setUp(self):
         self.mod1 = testutils.create_module(self.project, "mod1")
         self.pkg = testutils.create_package(self.project, "pkg")
         self.mod2 = testutils.create_module(self.project, "mod2", self.pkg)
-        self.importer = autoimport.AutoImport(self.project, observe=False)
+        self.importer = autoimport.AutoImport(
+            self.project, observe=False, memory=True
+        )
 
     def tearDown(self):
+        self.importer.close()
         testutils.remove_project(self.project)
         super().tearDown()
 
@@ -87,7 +91,8 @@ def test_not_caching_underlined_names(self):
         self.assertEqual(["mod1"], self.importer.get_modules("_myvar"))
 
     def test_caching_underlined_names_passing_to_the_constructor(self):
-        importer = autoimport.AutoImport(self.project, False, True)
+        importer = autoimport.AutoImport(self.project, False, True, memory=True)
+        self.addCleanup(importer.close)
         self.mod1.write("_myvar = None\n")
         importer.update_resource(self.mod1)
         self.assertEqual(["mod1"], importer.get_modules("_myvar"))
@@ -178,12 +183,12 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self):
 
 
 def test_search_submodule(project, external_fixturepkg):
-    importer = autoimport.AutoImport(project, observe=False)
-    importer.update_module("external_fixturepkg")
-    import_statement = ("from external_fixturepkg import mod1", "mod1")
-    assert import_statement in importer.search("mod1", exact_match=True)
-    assert import_statement in importer.search("mo")
-    assert import_statement in importer.search("mod1")
+    with closing(autoimport.AutoImport(project, observe=False, memory=True)) as importer:
+        importer.update_module("external_fixturepkg")
+        import_statement = ("from external_fixturepkg import mod1", "mod1")
+        assert import_statement in importer.search("mod1", exact_match=True)
+        assert import_statement in importer.search("mo")
+        assert import_statement in importer.search("mod1")
 
 
 class AutoImportObservingTest(unittest.TestCase):
@@ -193,9 +198,12 @@ def setUp(self):
         self.mod1 = testutils.create_module(self.project, "mod1")
         self.pkg = testutils.create_package(self.project, "pkg")
         self.mod2 = testutils.create_module(self.project, "mod2", self.pkg)
-        self.importer = autoimport.AutoImport(self.project, observe=True)
+        self.importer = autoimport.AutoImport(
+            self.project, observe=True, memory=True
+        )
 
     def tearDown(self):
+        self.importer.close()
         testutils.remove_project(self.project)
         super().tearDown()
 

ropetest/doatest.py

@@ -27,7 +27,13 @@ def try_CVE_2014_3539_exploit(self, receiver, payload):
         # Simulated attacker writing to the socket
         # Assume the attacker guesses the port correctly; 3037 is used by
         # default if it is available.
-        attacker_proc = multiprocessing.Process(
+        # Never fork this (multi-threaded) test process: see
+        # rope.contrib.autoimport.sqlite._process_pool.
+        methods = multiprocessing.get_all_start_methods()
+        context = multiprocessing.get_context(
+            "forkserver" if "forkserver" in methods else "spawn"
+        )
+        attacker_proc = context.Process(
             target=cve_2014_3539_attacker, args=(receiver.data_port, payload)
         )