Add startup checks to application hosts to fail early on body storage and database migrations

docs/handling-unavailable-runtime-dependencies.md

@@ -17,3 +17,12 @@ ServiceControl instances handle unavailable runtime dependencies in one of two w
 
 * A failure detected during startup stops the instance immediately.
 * A failure detected after a successful startup does not stop the instance. The instance is expected to try to recover once the dependency becomes available again.
+
+### Startup checks
+
+The Main and Audit instances run startup checks (`IStartupCheck`) in a hosted service that is registered before persistence, ingestion, and component hosted services. A failing check stops the instance before it ingests any message. The SQL Server and PostgreSQL persisters register these checks:
+
+| ClassName | Prerequisites | Description |
+| --- | --- | --- |
+| `PendingMigrationsStartupCheck` | Main or Audit instance using SQL Server or PostgreSQL persistence | Checks that the database has no pending migrations. Runtime startup does not apply migrations; run `--setup` to apply them. |
+| `FileSystemBodyStorageStartupCheck` | Main instance using SQL Server or PostgreSQL persistence with file system body storage | Checks that the configured body storage directory exists and is writable. Setup creates the directory but does not verify write access. |

src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs

@@ -92,6 +92,7 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured()
         static readonly string[] ExpectedHostedServices =
         [
             "GenericWebHostService",                // health endpoint only, no ServiceControl API
+            "StartupCheckHostedService",
             nameof(ErrorIngestion),                 // the reason this host exists
             "HeartbeatMonitoringHostedService",     // warms the endpoint monitor, does not check heartbeats
             "InternalCustomChecksHostedService",    // reports this node's ingestion health to the database

src/ServiceControl.Audit.Persistence.EFCore/Abstractions/EFPersistenceBase.cs

@@ -9,6 +9,7 @@ namespace ServiceControl.Audit.Persistence.EFCore.Abstractions;
 using ServiceControl.Audit.Persistence.EFCore.Implementation.UnitOfWork;
 using ServiceControl.Audit.Persistence.EFCore.Infrastructure.Metrics;
 using ServiceControl.Audit.Persistence.UnitOfWork;
+using ServiceControl.Infrastructure.StartupChecks;
 
 abstract class EFPersistenceBase(EFPersisterSettings settings) : IPersistence
 {
@@ -18,6 +19,8 @@ public void AddPersistence(IServiceCollection services)
         AddQueryServices(services);
         AddBodyStorage(services);
 
+        services.AddScoped<IStartupCheck, PendingMigrationsStartupCheck>();
+
         services.AddSingleton<AuditDataStore>();
         services.AddSingleton<IAuditMessagesViewDataStore>(provider => provider.GetRequiredService<AuditDataStore>());
         services.AddSingleton<ISagaHistoryDataStore>(provider => provider.GetRequiredService<AuditDataStore>());

src/ServiceControl.Audit.Persistence.EFCore/Implementation/PendingMigrationsStartupCheck.cs

@@ -0,0 +1,18 @@
+namespace ServiceControl.Audit.Persistence.EFCore.Implementation;
+
+using Microsoft.EntityFrameworkCore;
+using ServiceControl.Audit.Persistence.EFCore.DbContexts;
+using ServiceControl.Infrastructure.StartupChecks;
+
+sealed class PendingMigrationsStartupCheck(AuditDbContext dbContext) : IStartupCheck
+{
+    public async Task Check(CancellationToken cancellationToken = default)
+    {
+        var pending = (await dbContext.Database.GetPendingMigrationsAsync(cancellationToken).ConfigureAwait(false)).ToList();
+
+        if (pending.Count > 0)
+        {
+            throw new InvalidOperationException($"The audit database has pending migrations. Run the instance with --setup to apply them.");
+        }
+    }
+}

src/ServiceControl.Audit.Persistence.Tests/EFCore/SchemaSetupTests.cs

@@ -6,6 +6,8 @@ namespace ServiceControl.Audit.Persistence.Tests
     using Microsoft.Extensions.Hosting;
     using NUnit.Framework;
     using ServiceControl.Audit.Persistence.EFCore.Abstractions;
+    using ServiceControl.Audit.Persistence.EFCore.DbContexts;
+    using ServiceControl.Audit.Persistence.EFCore.Implementation;
     using ServiceControl.Persistence.Tests;
 
     [TestFixture]
@@ -73,6 +75,43 @@ public async Task Setup_ignores_the_primary_migrations_history_table_in_another_
             }
         }
 
+        [Test]
+        public async Task Pending_migrations_fail_the_startup_check_until_setup_has_applied_them()
+        {
+            var testsConfiguration = new PersistenceTestsConfiguration();
+            var connectionString = await testsConfiguration.GetConnectionString();
+            var schema = $"sc_pending_{Guid.NewGuid():n}";
+
+            try
+            {
+                var exception = Assert.ThrowsAsync<InvalidOperationException>(() => CheckPendingMigrations(testsConfiguration, connectionString, schema));
+                Assert.That(exception.Message, Does.Contain("pending migrations").And.Contain("--setup"));
+
+                await Setup(testsConfiguration, connectionString, schema);
+
+                Assert.DoesNotThrowAsync(() => CheckPendingMigrations(testsConfiguration, connectionString, schema));
+            }
+            finally
+            {
+                await TestSchema.Drop(connectionString, schema);
+            }
+        }
+
+        static async Task CheckPendingMigrations(PersistenceTestsConfiguration testsConfiguration, string connectionString, string schema)
+        {
+            var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000);
+            settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.ConnectionStringKey] = connectionString;
+            settings.PersisterSpecificSettings[EFPersistenceConfigurationBase.SchemaKey] = schema;
+
+            var services = new ServiceCollection();
+            testsConfiguration.CreateConfiguration().Create(settings).AddInstaller(services);
+
+            await using var provider = services.BuildServiceProvider();
+            using var scope = provider.CreateScope();
+
+            await new PendingMigrationsStartupCheck(scope.ServiceProvider.GetRequiredService<AuditDbContext>()).Check();
+        }
+
         static async Task Setup(PersistenceTestsConfiguration testsConfiguration, string connectionString, string schema)
         {
             var settings = new PersistenceSettings(TimeSpan.FromDays(1), true, 100000);

src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs

@@ -69,6 +69,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder,
         // directly and to make things more complex of course the order of registration still matters ;)
         services.AddSingleton(provider => new Lazy<IMessageDispatcher>(provider.GetRequiredService<IMessageDispatcher>));
 
+        services.AddStartupChecks();
         services.AddPersistence(persistenceSettings, persistenceConfiguration);
 
         NServiceBusFactory.Configure(settings, transportCustomization, transportSettings, onCriticalError, configuration);

src/ServiceControl.Hosting/HostApplicationBuilderExtensions.cs

@@ -14,4 +14,6 @@ public static void AddWindowsServiceWithRequestTimeout(this IHostApplicationBuil
             builder.Services.AddSingleton<IHostLifetime, WindowsServiceWithRequestTimeout>();
         }
     }
+
+    public static void AddStartupChecks(this IServiceCollection services) => services.AddHostedService<StartupCheckHostedService>();
 }

src/ServiceControl.Hosting/StartupCheckHostedService.cs

@@ -0,0 +1,25 @@
+namespace ServiceControl.Hosting;
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using ServiceControl.Infrastructure.StartupChecks;
+
+sealed class StartupCheckHostedService(IServiceScopeFactory scopeFactory) : IHostedService
+{
+    public async Task StartAsync(CancellationToken cancellationToken = default)
+    {
+        var scope = scopeFactory.CreateAsyncScope();
+
+        await using (scope.ConfigureAwait(false))
+        {
+            foreach (var check in scope.ServiceProvider.GetServices<IStartupCheck>())
+            {
+                await check.Check(cancellationToken).ConfigureAwait(false);
+            }
+        }
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
+}

src/ServiceControl.Infrastructure.Tests/StartupChecks/StartupCheckHostedServiceTests.cs

@@ -0,0 +1,141 @@
+namespace ServiceControl.Infrastructure.Tests.StartupChecks;
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using NUnit.Framework;
+using ServiceControl.Hosting;
+using ServiceControl.Infrastructure.StartupChecks;
+
+[TestFixture]
+class StartupCheckHostedServiceTests
+{
+    List<string> events;
+
+    [SetUp]
+    public void SetUp() => events = [];
+
+    [Test]
+    public async Task Checks_run_in_order_in_one_scope_that_is_disposed_before_later_hosted_services()
+    {
+        using var host = Build(services =>
+        {
+            services.AddSingleton(events);
+            services.AddScoped<ScopedDependency>();
+            services.AddScoped<IStartupCheck>(p => Resolve(p, "first"));
+            services.AddScoped<IStartupCheck>(p => Resolve(p, "second"));
+        });
+
+        await host.StartAsync();
+
+        Assert.That(events, Is.EqualTo(new[] { "scope created", "first", "second", "scope disposed", "later service started" }));
+    }
+
+    [Test]
+    public void A_failing_check_stops_startup_before_later_hosted_services()
+    {
+        using var host = Build(services =>
+        {
+            services.AddScoped<IStartupCheck>(_ => new ThrowingCheck());
+            services.AddScoped<IStartupCheck>(_ => new RecordingCheck("never", events));
+        });
+
+        var ex = Assert.ThrowsAsync<InvalidOperationException>(() => host.StartAsync());
+
+        Assert.That(ex.Message, Is.EqualTo("check failed"));
+        Assert.That(events, Is.Empty);
+    }
+
+    [Test]
+    public void Cancelling_the_start_token_cancels_the_running_check()
+    {
+        using var cts = new CancellationTokenSource();
+        using var host = Build(services => services.AddScoped<IStartupCheck>(_ => new TokenCheck(t =>
+        {
+            cts.Cancel();
+            t.ThrowIfCancellationRequested();
+        })));
+
+        Assert.CatchAsync<OperationCanceledException>(() => host.StartAsync(cts.Token));
+        Assert.That(events, Is.Empty);
+    }
+
+    [Test]
+    public async Task Startup_succeeds_without_checks()
+    {
+        using var host = Build(_ => { });
+
+        await host.StartAsync();
+
+        Assert.That(events, Is.EqualTo(new[] { "later service started" }));
+    }
+
+    RecordingCheck Resolve(IServiceProvider provider, string name)
+    {
+        provider.GetRequiredService<ScopedDependency>();
+        return new RecordingCheck(name, events);
+    }
+
+    IHost Build(Action<IServiceCollection> configure)
+    {
+        var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings());
+        configure(builder.Services);
+        builder.Services.AddStartupChecks();
+        builder.Services.AddSingleton<IHostedService>(new LaterService(events));
+        return builder.Build();
+    }
+
+    class ScopedDependency : IAsyncDisposable
+    {
+        readonly List<string> events;
+
+        public ScopedDependency(List<string> events)
+        {
+            this.events = events;
+            events.Add("scope created");
+        }
+
+        public ValueTask DisposeAsync()
+        {
+            events.Add("scope disposed");
+            return ValueTask.CompletedTask;
+        }
+    }
+
+    class RecordingCheck(string name, List<string> events) : IStartupCheck
+    {
+        public Task Check(CancellationToken cancellationToken = default)
+        {
+            events.Add(name);
+            return Task.CompletedTask;
+        }
+    }
+
+    class ThrowingCheck : IStartupCheck
+    {
+        public Task Check(CancellationToken cancellationToken = default) => throw new InvalidOperationException("check failed");
+    }
+
+    class TokenCheck(Action<CancellationToken> onCheck) : IStartupCheck
+    {
+        public Task Check(CancellationToken cancellationToken = default)
+        {
+            onCheck(cancellationToken);
+            return Task.CompletedTask;
+        }
+    }
+
+    class LaterService(List<string> events) : IHostedService
+    {
+        public Task StartAsync(CancellationToken cancellationToken = default)
+        {
+            events.Add("later service started");
+            return Task.CompletedTask;
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
+    }
+}

src/ServiceControl.Infrastructure/StartupChecks/IStartupCheck.cs

@@ -0,0 +1,9 @@
+namespace ServiceControl.Infrastructure.StartupChecks;
+
+using System.Threading;
+using System.Threading.Tasks;
+
+public interface IStartupCheck
+{
+    Task Check(CancellationToken cancellationToken = default);
+}

src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs

@@ -6,6 +6,7 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 using Particular.LicensingComponent.Contracts;
 using Particular.LicensingComponent.Persistence;
 using ServiceControl.CustomChecks;
+using ServiceControl.Infrastructure.StartupChecks;
 using ServiceControl.Operations.BodyStorage;
 using ServiceControl.Persistence.EFCore.Implementation;
 using ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
@@ -25,6 +26,8 @@ protected static void RegisterDataStores(IServiceCollection services, EFPersiste
     {
         services.TryAddSingleton(TimeProvider.System);
 
+        services.AddScoped<IStartupCheck, PendingMigrationsStartupCheck>();
+
         services.AddSingleton<MinimumRequiredStorageState>();
 
         services.AddSingleton<IServiceControlSubscriptionStorage, SubscriptionStorage>();
@@ -90,6 +93,7 @@ static void RegisterBodyStorage(IServiceCollection services, EFPersisterSettings
                 services.TryAddSingleton(fileSystem);
                 services.AddSingleton<IBodyStoragePersistence, FileSystemBodyStoragePersistence>();
                 services.AddSingleton<IDriveSpaceProvider, DriveInfoSpaceProvider>();
+                services.AddScoped<IStartupCheck, FileSystemBodyStorageStartupCheck>();
                 services.AddCustomCheck<FileSystemBodyStorageCustomCheck>();
                 break;
             case AzureBlobBodyStorageSettings azureBlob:

src/ServiceControl.Persistence.EFCore/Implementation/BodyStorage/FileSystemBodyStorageStartupCheck.cs

@@ -0,0 +1,26 @@
+namespace ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
+
+using ServiceControl.Infrastructure.StartupChecks;
+using ServiceControl.Persistence.EFCore.Abstractions;
+
+sealed class FileSystemBodyStorageStartupCheck(FileSystemBodyStorageSettings settings) : IStartupCheck
+{
+    public async Task Check(CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var probePath = Path.Combine(settings.StoragePath, $".write-probe-{Guid.NewGuid():N}");
+            await using var probe = new FileStream(probePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.DeleteOnClose);
+            await probe.WriteAsync(new byte[] { 1 }, cancellationToken).ConfigureAwait(false);
+            await probe.FlushAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+        {
+            throw new InvalidOperationException($"Message body storage path '{settings.StoragePath}' is not writable. Ensure the directory exists and the service account can write to it. Run --setup on the normal instance to create the directory.", ex);
+        }
+    }
+}

src/ServiceControl.Persistence.EFCore/Implementation/PendingMigrationsStartupCheck.cs

@@ -0,0 +1,18 @@
+namespace ServiceControl.Persistence.EFCore.Implementation;
+
+using Microsoft.EntityFrameworkCore;
+using ServiceControl.Infrastructure.StartupChecks;
+using ServiceControl.Persistence.EFCore.DbContexts;
+
+sealed class PendingMigrationsStartupCheck(ServiceControlDbContext dbContext) : IStartupCheck
+{
+    public async Task Check(CancellationToken cancellationToken = default)
+    {
+        var pending = (await dbContext.Database.GetPendingMigrationsAsync(cancellationToken).ConfigureAwait(false)).ToList();
+
+        if (pending.Count > 0)
+        {
+            throw new InvalidOperationException($"The database has pending migrations. Run the instance with --setup to apply them.");
+        }
+    }
+}