diff --git a/morango/sync/db.py b/morango/sync/db.py index aa4f4c96..3d503d02 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -1,38 +1,101 @@ -import logging from contextlib import contextmanager +from threading import local from django.db import connection +from django.db import connections +from django.db import DEFAULT_DB_ALIAS from django.db import transaction +from django.db.models import F +from morango.models.core import InstanceIDModel from morango.sync.backends.utils import load_backend from morango.sync.utils import lock_partitions -logger = logging.getLogger(__name__) +class SyncDatabase: + """Database access for sync, with routed ORM access when no alias is supplied.""" + + def __init__(self, alias=None): + self.alias = alias + self._backends = local() + + @classmethod + def default(cls): + return cls() + + @property + def connection(self): + alias = DEFAULT_DB_ALIAS if self.alias is None else self.alias + return connections[alias] + + @property + def backend(self): + if self.alias is None: + return DBBackend + + conn = self.connection + backend = getattr(self._backends, "backend", None) + if backend is None or backend.connection is not conn: + backend = self._backends.backend = load_backend(conn) + return backend + + def manager(self, model): + manager = model._default_manager + if self.alias is None: + return manager + return manager.db_manager(self.alias) + + def atomic(self, savepoint=True): + return transaction.atomic(using=self.connection.alias, savepoint=savepoint) + + @contextmanager + def begin_transaction(self, sync_filter, isolated=False, shared_lock=False): + """ + Start a transaction and lock the affected partitions. + + :param sync_filter: The sync filter, or None to lock all partitions + :type sync_filter: morango.models.certificates.Filter | None + :param isolated: Whether to set repeatable-read isolation for the transaction + :param shared_lock: Whether to acquire shared rather than exclusive locks + """ + if isolated: + with self.backend._set_transaction_repeatable_read(): + with self.atomic(savepoint=False): + lock_partitions(self.backend, sync_filter=sync_filter, shared=shared_lock) + yield + else: + with self.atomic(): + lock_partitions(self.backend, sync_filter=sync_filter, shared=shared_lock) + yield + + def cursor(self): + return self.connection.cursor() + + def current_instance_and_increment_counter(self): + """Increment the current instance, which must already exist on a pinned alias.""" + if self.alias is None: + return InstanceIDModel.get_current_instance_and_increment_counter() + + with self.atomic(): + instances = self.manager(InstanceIDModel) + instance = instances.get(current=True) + instances.filter(pk=instance.pk).update(counter=F("counter") + 1) + instance.refresh_from_db(using=self.alias, fields=["counter"]) + return instance + + +# Django's default connection proxy resolves the connection on the calling thread. DBBackend = load_backend(connection) +_default = SyncDatabase.default() -@contextmanager def begin_transaction(sync_filter, isolated=False, shared_lock=False): """ - Starts a transaction, sets the transaction isolation level to repeatable read, and locks - affected partitions - - :param sync_filter: The filter for filtering applicable records of the sync - :type sync_filter: morango.models.certificates.Filter|None - :param isolated: Whether to alter the transaction isolation to repeatable-read - :type isolated: bool - :param shared_lock: Whether the advisory lock should be exclusive or shared - :type shared_lock: bool + Start a sync transaction on the default database and lock affected partitions. + + :param sync_filter: The sync filter, or None to lock all partitions + :type sync_filter: morango.models.certificates.Filter | None + :param isolated: Whether to set repeatable-read isolation for the transaction + :param shared_lock: Whether to acquire shared rather than exclusive locks """ - if isolated: - # when isolation is requested, we modify the transaction isolation of the connection for the - # duration of the transaction - with DBBackend._set_transaction_repeatable_read(): - with transaction.atomic(savepoint=False): - lock_partitions(DBBackend, sync_filter=sync_filter, shared=shared_lock) - yield - else: - with transaction.atomic(): - lock_partitions(DBBackend, sync_filter=sync_filter, shared=shared_lock) - yield + return _default.begin_transaction(sync_filter, isolated=isolated, shared_lock=shared_lock) diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index 0a5a69d7..09aad312 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -5,10 +5,13 @@ import json import socket import uuid +from contextlib import contextmanager import factory import mock from django.core.serializers.json import DjangoJSONEncoder +from django.db import connections +from django.db import DEFAULT_DB_ALIAS from django.test.testcases import LiveServerTestCase from django.utils import timezone from facility_profile.models import Facility @@ -36,6 +39,33 @@ from .compat import EnvironmentVarGuard +@contextmanager +def assert_no_default_db_queries(): + """Report SQL sent to the default connection by the calling thread. + + A top-level PostgreSQL transaction emits no SQL through Django's execute + wrapper. Check ``connection.in_atomic_block`` separately when verifying + which connection opened the transaction. + """ + rejected_queries = [] + + def reject_query(execute, sql, params, many, context): + error = AssertionError("Unexpected query on the default database: {}".format(sql)) + rejected_queries.append(error) + raise error + + try: + with connections[DEFAULT_DB_ALIAS].execute_wrapper(reject_query): + yield + except Exception as error: + if not rejected_queries or error in rejected_queries: + raise + raise AssertionError("\n".join(map(str, rejected_queries))) from error + else: + if rejected_queries: + raise AssertionError("\n".join(map(str, rejected_queries))) + + def get_free_tcp_port(): # sandboxing can cause 0 to fail (which is to try for any open port), so fallback to a hardcoded # list afterward diff --git a/tests/testapp/tests/sync/test_db.py b/tests/testapp/tests/sync/test_db.py index 4882939f..88031880 100644 --- a/tests/testapp/tests/sync/test_db.py +++ b/tests/testapp/tests/sync/test_db.py @@ -1,26 +1,120 @@ import threading import uuid +from concurrent.futures import ThreadPoolExecutor from time import sleep import pytest from django.conf import settings from django.db import connection +from django.db import connections +from django.db import transaction from django.test import override_settings +from django.test import TestCase from django.test import TransactionTestCase from django.utils import timezone from morango.models.certificates import Filter +from morango.models.core import DatabaseIDModel +from morango.models.core import InstanceIDModel from morango.models.core import Store from morango.models.core import SyncSession from morango.models.core import TransferSession +from morango.sync.backends.sqlite import SQLWrapper as SQLiteWrapper from morango.sync.backends.utils import load_backend from morango.sync.db import begin_transaction +from morango.sync.db import SyncDatabase +from ..helpers import assert_no_default_db_queries from ..helpers import create_buffer_and_store_dummy_data DBBackend = load_backend(connection) +class SyncDatabaseTestCase(TestCase): + def test_default_manager(self): + self.assertIs(SyncDatabase.default().manager(Store), Store._default_manager) + + def test_default_counter(self): + instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + counter = instance.counter + updated = SyncDatabase.default().current_instance_and_increment_counter() + self.assertEqual(updated.pk, instance.pk) + self.assertEqual(updated.counter, counter + 1) + + def test_guard_rejects_default_query(self): + with self.assertRaisesRegex(AssertionError, "SELECT COUNT"): + with assert_no_default_db_queries(): + Store.objects.using("default").count() + Store.objects.using("default").count() + + def test_guard_rejects_default_savepoint(self): + with self.assertRaisesRegex(AssertionError, "SAVEPOINT"): + with assert_no_default_db_queries(): + with transaction.atomic(): + pass + + def test_guard_reports_a_caught_query_violation(self): + with self.assertRaisesRegex(AssertionError, "SELECT COUNT"): + with assert_no_default_db_queries(): + with self.assertRaises(AssertionError): + Store.objects.using("default").count() + + +@pytest.mark.skipif("default2" not in settings.DATABASES, reason="requires a second database") +@override_settings(DATABASE_ROUTERS=[]) +class PinnedSyncDatabaseTestCase(TransactionTestCase): + databases = {"default", "default2"} + + def test_pinned_manager_and_backend(self): + db = SyncDatabase("default2") + self.assertEqual(db.manager(Store).db, "default2") + self.assertIsInstance(db.backend, SQLiteWrapper) + self.assertIs(db.backend, db.backend) + self.assertIs(db.backend.connection, connections["default2"]) + + def test_pinned_backend_uses_calling_threads_connection(self): + db = SyncDatabase("default2") + main_backend = db.backend + + def get_worker_backend(): + backend = db.backend + self.assertIs(backend.connection, connections["default2"]) + return backend + + with ThreadPoolExecutor(max_workers=1) as executor: + self.assertIsNot(executor.submit(get_worker_backend).result(), main_backend) + self.assertIs(db.backend, main_backend) + + def test_pinned_transaction_and_cursor(self): + db = SyncDatabase("default2") + for isolated in (False, True): + with self.subTest(isolated=isolated): + with assert_no_default_db_queries(), db.begin_transaction(None, isolated=isolated): + self.assertFalse(connection.in_atomic_block) + self.assertTrue(connections["default2"].in_atomic_block) + self.assertEqual(db.manager(Store).count(), 0) + with db.cursor() as cursor: + cursor.execute("SELECT 1") + self.assertEqual(cursor.fetchone(), (1,)) + self.assertFalse(connections["default2"].in_atomic_block) + + def test_pinned_counter_uses_alias_with_default_instance_cached(self): + default_instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + counter = default_instance.counter + instance = InstanceIDModel.objects.get(pk=default_instance.pk) + database = instance.database + db = SyncDatabase("default2") + # Seed a database identity without invoking the routed model save methods. + with assert_no_default_db_queries(): + db.manager(DatabaseIDModel).bulk_create([database]) + db.manager(InstanceIDModel).bulk_create([instance]) + updated = db.current_instance_and_increment_counter() + self.assertEqual(updated.counter, counter + 1) + self.assertEqual(db.manager(InstanceIDModel).get(pk=updated.pk).counter, counter + 1) + default_instance.refresh_from_db() + self.assertEqual(default_instance.counter, counter) + + def _concurrent_store_write(thread_event, store_id): while not thread_event.is_set(): sleep(0.1)