diff --git a/app/models/solid_queue/ready_execution.rb b/app/models/solid_queue/ready_execution.rb index c0a67ce2..703ea4e6 100644 --- a/app/models/solid_queue/ready_execution.rb +++ b/app/models/solid_queue/ready_execution.rb @@ -24,8 +24,13 @@ def select_and_lock(queue_relation, process_id, limit) return [] if limit <= 0 transaction do - candidates = select_candidates(queue_relation, limit) - lock_candidates(candidates, process_id) + # Lock the process before selecting candidates so deregistration can't miss these claims. + if SolidQueue::Process.lock.exists?(id: process_id) + candidates = select_candidates(queue_relation, limit) + lock_candidates(candidates, process_id) + else + [] + end end end diff --git a/test/models/solid_queue/ready_execution_concurrency_test.rb b/test/models/solid_queue/ready_execution_concurrency_test.rb new file mode 100644 index 00000000..220e4ef0 --- /dev/null +++ b/test/models/solid_queue/ready_execution_concurrency_test.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require "test_helper" + +# These tests exercise the lock ReadyExecution.claim takes on the claiming +# process, so they run on separate connections/threads and skip on SQLite, +# which has no row-level locking. They follow the same pattern as +# SolidQueue::ClaimedExecutionConcurrencyTest. +class SolidQueue::ReadyExecutionConcurrencyTest < ActiveSupport::TestCase + self.use_transactional_tests = false + + setup do + @process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123") + @job = SolidQueue::Job.find_by!(active_job_id: AddToBufferJob.perform_later(1).job_id) + end + + [ true, false ].each do |skip_locked| + test "deregistering a process waits for its claim and releases it (use_skip_locked: #{skip_locked})" do + skip_on_sqlite + + with_skip_locked(skip_locked) do + elapsed = with_claim_in_progress do + @process.deregister + end + + assert_operator elapsed, :>=, 0.3, "deregister should have blocked on the claim (took #{elapsed.round(3)}s)" + assert_not SolidQueue::Process.exists?(@process.id) + assert_equal 0, SolidQueue::ClaimedExecution.count + assert @job.reload.ready? + end + end + end + + test "claiming doesn't deadlock with deregistering another process that releases its claims" do + skip_on_sqlite + + other = SolidQueue::Process.register(kind: "Worker", pid: 43, name: "worker-456") + 10.times { |i| AddToBufferJob.perform_later(i) } + SolidQueue::ReadyExecution.claim("*", 5, other.id) + + errors = Concurrent::Array.new + start = Concurrent::Event.new + + claimers = 3.times.map do + Thread.new do + SolidQueue::Record.connection_pool.with_connection do + start.wait(5) + 5.times { SolidQueue::ReadyExecution.claim("*", 1, @process.id) } + end + rescue => error + errors << error + end + end + + deregisterer = Thread.new do + SolidQueue::Record.connection_pool.with_connection do + start.wait(5) + other.reload.deregister + end + rescue => error + errors << error + end + + start.set + [ *claimers, deregisterer ].each { |thread| assert thread.join(10), "a thread got stuck" } + + assert_empty errors + assert_equal 0, SolidQueue::ClaimedExecution.where(process_id: other.id).count + assert_equal 11, SolidQueue::ReadyExecution.count + SolidQueue::ClaimedExecution.where(process_id: @process.id).count + end + + private + # Claims @job on a separate connection and holds the claim's transaction open + # while the given block runs, then commits. Returns how long the block took. + def with_claim_in_progress + claimed = Concurrent::Event.new + claims = nil + + claimer = Thread.new do + SolidQueue::Record.connection_pool.with_connection do + SolidQueue::Record.transaction do + claims = SolidQueue::ReadyExecution.claim(@job.queue_name, 1, @process.id) + claimed.set + sleep 0.5 + end + end + end + + assert claimed.wait(5), "the claim should have started" + assert_equal [ @job.id ], claims.map(&:job_id) + + started_at = monotonic_now + yield + elapsed = monotonic_now - started_at + + assert claimer.join(5), "the claim should have finished" + elapsed + ensure + claimer&.join(5) + end + + def with_skip_locked(value) + previous = SolidQueue.use_skip_locked + SolidQueue.use_skip_locked = value + yield + ensure + SolidQueue.use_skip_locked = previous + end + + def skip_on_sqlite + skip "Row-level locking not supported on SQLite" if SolidQueue::Record.connection.adapter_name.downcase.include?("sqlite") + end + + def monotonic_now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end +end diff --git a/test/models/solid_queue/ready_execution_test.rb b/test/models/solid_queue/ready_execution_test.rb index ae988b1f..f7168a62 100644 --- a/test/models/solid_queue/ready_execution_test.rb +++ b/test/models/solid_queue/ready_execution_test.rb @@ -2,6 +2,8 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase setup do + @process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-42") + 5.times do |i| AddToBufferJob.set(queue: "backend", priority: 5 - i).perform_later(i) end @@ -11,7 +13,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase test "claim all jobs for existing queue" do assert_claimed_jobs(@jobs.count) do - SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, 42) + SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id) end @jobs.each(&:reload) @@ -19,15 +21,25 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase assert @jobs.all?(&:claimed?) end + test "a process whose registration is gone claims nothing" do + # A worker learns that a supervisor pruned its registration only at its next heartbeat. A claim + # made before that would belong to no process, and nothing but a supervisor's boot recovers it. + @process.destroy! + + assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do + SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id) + end + end + test "claim jobs for queue without jobs at the moment" do assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do - SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, 42) + SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, @process.id) end end test "claim some jobs for existing queue" do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim("backend", 2, 42) + SolidQueue::ReadyExecution.claim("backend", 2, @process.id) end @jobs.first(2).each do |job| @@ -45,7 +57,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, @process.id) end end @@ -53,7 +65,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id) end end @@ -61,7 +73,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, @process.id) end assert @jobs.none?(&:claimed?) @@ -72,25 +84,25 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.set(queue: "mail-ops").perform_later(1) assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id) end AddToBufferJob.set(queue: "mail_ops").perform_later(1) AddToBufferJob.set(queue: "100x_done").perform_later(1) assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id) end assert_equal "mail_ops", SolidQueue::ClaimedExecution.last.job.queue_name assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, @process.id) end AddToBufferJob.set(queue: "100%_done").perform_later(1) assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("100%_*", 1, 42) + SolidQueue::ReadyExecution.claim("100%_*", 1, @process.id) end assert_equal "100%_done", SolidQueue::ClaimedExecution.last.job.queue_name end @@ -101,7 +113,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase SolidQueue::Queue.find_by_name("backend").pause assert_claimed_jobs(1) do - SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id) end @jobs.each(&:reload) @@ -112,7 +124,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(6) do - SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, @process.id) end end @@ -120,7 +132,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase AddToBufferJob.perform_later("hey") assert_claimed_jobs(0) do - SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, 42) + SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, @process.id) end end @@ -129,7 +141,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase job = SolidQueue::Job.last assert_claimed_jobs(3) do - SolidQueue::ReadyExecution.claim("*", 3, 42) + SolidQueue::ReadyExecution.claim("*", 3, @process.id) end assert job.reload.claimed? @@ -145,7 +157,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase assert_equal "background", job.queue_name assert_claimed_jobs(3) do - SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, 42) + SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, @process.id) end assert job.reload.claimed? @@ -164,7 +176,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase claimed_jobs = [] 4.times do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, 42) + SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, @process.id) end claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job) @@ -185,7 +197,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase claimed_jobs = [] 5.times do assert_claimed_jobs(2) do - SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, 42) + SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, @process.id) end claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job)