From 5671f35506f2d7c449a796812fb4d7405645508b Mon Sep 17 00:00:00 2001 From: Cole Robertson Date: Tue, 17 Nov 2026 12:42:36 -0600 Subject: [PATCH] Don't claim jobs for a process that has been pruned A worker normally finds out its process was pruned at its next heartbeat. Until then it keeps claiming jobs with the process id it remembers. Nothing releases those claimed executions while supervisors keep running, so a job that doesn't finish before the worker exits stays claimed until a supervisor boots and fails it with ProcessMissingError. Lock the process row before selecting candidates, and claim nothing if the process is gone. Deregistering deletes that row and releases its claimed executions in one transaction, so a claim either commits first and is released with the rest, or claims nothing. The ReadyExecution tests now claim with a registered process instead of the literal id 42. Co-Authored-By: Claude Opus 5.5 --- app/models/solid_queue/ready_execution.rb | 9 +- .../ready_execution_concurrency_test.rb | 117 ++++++++++++++++++ .../solid_queue/ready_execution_test.rb | 46 ++++--- 3 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 test/models/solid_queue/ready_execution_concurrency_test.rb diff --git a/app/models/solid_queue/ready_execution.rb b/app/models/solid_queue/ready_execution.rb index c0a67ce2..703ea4e6 100644 --- a/app/models/solid_queue/ready_execution.rb +++ b/app/models/solid_queue/ready_execution.rb @@ -24,8 +24,13 @@ def select_and_lock(queue_relation, process_id, limit) return [] if limit <= 0 transaction do - candidates = select_candidates(queue_relation, limit) - lock_candidates(candidates, process_id) + # Lock the process before selecting candidates so deregistration can't miss these claims. + if SolidQueue::Process.lock.exists?(id: process_id) + candidates = select_candidates(queue_relation, limit) + lock_candidates(candidates, process_id) + else + [] + end end end diff --git a/test/models/solid_queue/ready_execution_concurrency_test.rb b/test/models/solid_queue/ready_execution_concurrency_test.rb new file mode 100644 index 00000000..220e4ef0 --- /dev/null +++ b/test/models/solid_queue/ready_execution_concurrency_test.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require "test_helper" + +# These tests exercise the lock ReadyExecution.claim takes on the claiming +# process, so they run on separate connections/threads and skip on SQLite, +# which has no row-level locking. They follow the same pattern as +# SolidQueue::ClaimedExecutionConcurrencyTest. +class SolidQueue::ReadyExecutionConcurrencyTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + setup do + @process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123") + @job = SolidQueue::Job.find_by!(active_job_id: AddToBufferJob.perform_later(1).job_id) + end + + [ true, false ].each do |skip_locked| + test "deregistering a process waits for its claim and releases it (use_skip_locked: #{skip_locked})" do + skip_on_sqlite + + with_skip_locked(skip_locked) do + elapsed = with_claim_in_progress do + @process.deregister + end + + assert_operator elapsed, :>=, 0.3, "deregister should have blocked on the claim (took #{elapsed.round(3)}s)" + assert_not SolidQueue::Process.exists?(@process.id) + assert_equal 0, SolidQueue::ClaimedExecution.count + assert @job.reload.ready? + end + end + end + + test "claiming doesn't deadlock with deregistering another process that releases its claims" do + skip_on_sqlite + + other = SolidQueue::Process.register(kind: "Worker", pid: 43, name: "worker-456") + 10.times { |i| AddToBufferJob.perform_later(i) } + SolidQueue::ReadyExecution.claim("*", 5, other.id) + + errors = Concurrent::Array.new + start = Concurrent::Event.new + + claimers = 3.times.map do + Thread.new do + SolidQueue::Record.connection_pool.with_connection do + start.wait(5) + 5.times { SolidQueue::ReadyExecution.claim("*", 1, @process.id) } + end + rescue => error + errors << error + end + end + + deregisterer = Thread.new do + SolidQueue::Record.connection_pool.with_connection do + start.wait(5) + other.reload.deregister + end + rescue => error + errors << error + end + + start.set + [ *claimers, deregisterer ].each { |thread| assert thread.join(10), "a thread got stuck" } + + assert_empty errors + assert_equal 0, SolidQueue::ClaimedExecution.where(process_id: other.id).count + assert_equal 11, SolidQueue::ReadyExecution.count + SolidQueue::ClaimedExecution.where(process_id: @process.id).count + end + + private + # Claims @job on a separate connection and holds the claim's transaction open + # while the given block runs, then commits. Returns how long the block took. + def with_claim_in_progress + claimed = Concurrent::Event.new + claims = nil + + claimer = Thread.new do + SolidQueue::Record.connection_pool.with_connection do + SolidQueue::Record.transaction do + claims = SolidQueue::ReadyExecution.claim(@job.queue_name, 1, @process.id) + claimed.set + sleep 0.5 + end + end + end + + assert claimed.wait(5), "the claim should have started" + assert_equal [ @job.id ], claims.map(&:job_id) + + started_at = monotonic_now + yield + elapsed = monotonic_now - started_at + + assert claimer.join(5), "the claim should have finished" + elapsed + ensure + claimer&.join(5) + end + + def with_skip_locked(value) + previous = SolidQueue.use_skip_locked + SolidQueue.use_skip_locked = value + yield + ensure + SolidQueue.use_skip_locked = previous + end + + def skip_on_sqlite + skip "Row-level locking not supported on SQLite" if SolidQueue::Record.connection.adapter_name.downcase.include?("sqlite") + end + + def monotonic_now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end +end diff --git a/test/models/solid_queue/ready_execution_test.rb b/test/models/solid_queue/ready_execution_test.rb index ae988b1f..f7168a62 100644 --- a/test/models/solid_queue/ready_execution_test.rb +++ b/test/models/solid_queue/ready_execution_test.rb @@ -2,6 +2,8 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase setup do + @process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-42") + 5.times do |i| AddToBufferJob.set(queue: "backend", priority: 5 - i).perform_later(i) end @@ -11,7 +13,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase test "claim all jobs for existing queue" do assert_claimed_jobs(@jobs.count) do - SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, 42) + SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id) end @jobs.each(&:reload) @@ -19,15 +21,25 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase assert @jobs.all?(&:claimed?) end + test "a process whose registration is gone claims nothing" do + # A worker learns that a supervisor pruned its registration only at its next heartbeat. A claim + # made before that would belong to no process, and nothing but a supervisor's boot recovers it. + @process.destroy! + + assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do + SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id) + end + end + test "claim jobs for queue without jobs at the moment" do assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do - SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, 42) + SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, @process.id) end end test "claim some jobs for existing queue" do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim("backend", 2, 42) + SolidQueue::ReadyExecution.claim("backend", 2, @process.id) end @jobs.first(2).each do |job| @@ -45,7 +57,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, @process.id) end end @@ -53,7 +65,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id) end end @@ -61,7 +73,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, @process.id) end assert @jobs.none?(&:claimed?) @@ -72,25 +84,25 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.set(queue: "mail-ops").perform_later(1) assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id) end AddToBufferJob.set(queue: "mail_ops").perform_later(1) AddToBufferJob.set(queue: "100x_done").perform_later(1) assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id) end assert_equal "mail_ops", SolidQueue::ClaimedExecution.last.job.queue_name assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, @process.id) end AddToBufferJob.set(queue: "100%_done").perform_later(1) assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("100%_*", 1, 42) + SolidQueue::ReadyExecution.claim("100%_*", 1, @process.id) end assert_equal "100%_done", SolidQueue::ClaimedExecution.last.job.queue_name end @@ -101,7 +113,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase SolidQueue::Queue.find_by_name("backend").pause assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id) end @jobs.each(&:reload) @@ -112,7 +124,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, @process.id) end end @@ -120,7 +132,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, @process.id) end end @@ -129,7 +141,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase job = SolidQueue::Job.last assert_claimed_jobs(3) do - SolidQueue::ReadyExecution.claim("*", 3, 42) + SolidQueue::ReadyExecution.claim("*", 3, @process.id) end assert job.reload.claimed? @@ -145,7 +157,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase assert_equal "background", job.queue_name assert_claimed_jobs(3) do - SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, 42) + SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, @process.id) end assert job.reload.claimed? @@ -164,7 +176,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase claimed_jobs = [] 4.times do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, 42) + SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, @process.id) end claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job) @@ -185,7 +197,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase claimed_jobs = [] 5.times do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, 42) + SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, @process.id) end claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job)