Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions app/models/solid_queue/ready_execution.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@ def select_and_lock(queue_relation, process_id, limit)
return [] if limit <= 0

transaction do
candidates = select_candidates(queue_relation, limit)
lock_candidates(candidates, process_id)
# Lock the process before selecting candidates so deregistration can't miss these claims.
if SolidQueue::Process.lock.exists?(id: process_id)
candidates = select_candidates(queue_relation, limit)
lock_candidates(candidates, process_id)
else
[]
end
end
end

Expand Down
117 changes: 117 additions & 0 deletions test/models/solid_queue/ready_execution_concurrency_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# frozen_string_literal: true

require "test_helper"

# These tests exercise the lock ReadyExecution.claim takes on the claiming
# process, so they run on separate connections/threads and skip on SQLite,
# which has no row-level locking. They follow the same pattern as
# SolidQueue::ClaimedExecutionConcurrencyTest.
class SolidQueue::ReadyExecutionConcurrencyTest < ActiveSupport::TestCase
self.use_transactional_tests = false

setup do
@process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123")
@job = SolidQueue::Job.find_by!(active_job_id: AddToBufferJob.perform_later(1).job_id)
end

[ true, false ].each do |skip_locked|
test "deregistering a process waits for its claim and releases it (use_skip_locked: #{skip_locked})" do
skip_on_sqlite

with_skip_locked(skip_locked) do
elapsed = with_claim_in_progress do
@process.deregister
end

assert_operator elapsed, :>=, 0.3, "deregister should have blocked on the claim (took #{elapsed.round(3)}s)"
assert_not SolidQueue::Process.exists?(@process.id)
assert_equal 0, SolidQueue::ClaimedExecution.count
assert @job.reload.ready?
end
end
end

test "claiming doesn't deadlock with deregistering another process that releases its claims" do
skip_on_sqlite

other = SolidQueue::Process.register(kind: "Worker", pid: 43, name: "worker-456")
10.times { |i| AddToBufferJob.perform_later(i) }
SolidQueue::ReadyExecution.claim("*", 5, other.id)

errors = Concurrent::Array.new
start = Concurrent::Event.new

claimers = 3.times.map do
Thread.new do
SolidQueue::Record.connection_pool.with_connection do
start.wait(5)
5.times { SolidQueue::ReadyExecution.claim("*", 1, @process.id) }
end
rescue => error
errors << error
end
end

deregisterer = Thread.new do
SolidQueue::Record.connection_pool.with_connection do
start.wait(5)
other.reload.deregister
end
rescue => error
errors << error
end

start.set
[ *claimers, deregisterer ].each { |thread| assert thread.join(10), "a thread got stuck" }

assert_empty errors
assert_equal 0, SolidQueue::ClaimedExecution.where(process_id: other.id).count
assert_equal 11, SolidQueue::ReadyExecution.count + SolidQueue::ClaimedExecution.where(process_id: @process.id).count
end

private
# Claims @job on a separate connection and holds the claim's transaction open
# while the given block runs, then commits. Returns how long the block took.
def with_claim_in_progress
claimed = Concurrent::Event.new
claims = nil

claimer = Thread.new do
SolidQueue::Record.connection_pool.with_connection do
SolidQueue::Record.transaction do
claims = SolidQueue::ReadyExecution.claim(@job.queue_name, 1, @process.id)
claimed.set
sleep 0.5
end
end
end

assert claimed.wait(5), "the claim should have started"
assert_equal [ @job.id ], claims.map(&:job_id)

started_at = monotonic_now
yield
elapsed = monotonic_now - started_at

assert claimer.join(5), "the claim should have finished"
elapsed
ensure
claimer&.join(5)
end

def with_skip_locked(value)
previous = SolidQueue.use_skip_locked
SolidQueue.use_skip_locked = value
yield
ensure
SolidQueue.use_skip_locked = previous
end

def skip_on_sqlite
skip "Row-level locking not supported on SQLite" if SolidQueue::Record.connection.adapter_name.downcase.include?("sqlite")
end

def monotonic_now
Process.clock_gettime(Process::CLOCK_MONOTONIC)
end
end
46 changes: 29 additions & 17 deletions test/models/solid_queue/ready_execution_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
setup do
@process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-42")

5.times do |i|
AddToBufferJob.set(queue: "backend", priority: 5 - i).perform_later(i)
end
Expand All @@ -11,23 +13,33 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase

test "claim all jobs for existing queue" do
assert_claimed_jobs(@jobs.count) do
SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, 42)
SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id)
end

@jobs.each(&:reload)
assert @jobs.none?(&:ready?)
assert @jobs.all?(&:claimed?)
end

test "a process whose registration is gone claims nothing" do
# A worker learns that a supervisor pruned its registration only at its next heartbeat. A claim
# made before that would belong to no process, and nothing but a supervisor's boot recovers it.
@process.destroy!

assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do
SolidQueue::ReadyExecution.claim("backend", @jobs.count + 1, @process.id)
end
end

test "claim jobs for queue without jobs at the moment" do
assert_no_difference [ -> { SolidQueue::ReadyExecution.count }, -> { SolidQueue::ClaimedExecution.count } ] do
SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, 42)
SolidQueue::ReadyExecution.claim("some_non_existing_queue", 10, @process.id)
end
end

test "claim some jobs for existing queue" do
assert_claimed_jobs(2) do
SolidQueue::ReadyExecution.claim("backend", 2, 42)
SolidQueue::ReadyExecution.claim("backend", 2, @process.id)
end

@jobs.first(2).each do |job|
Expand All @@ -45,23 +57,23 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
AddToBufferJob.perform_later("hey")

assert_claimed_jobs(6) do
SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim(%w[ backend background ], SolidQueue::Job.count + 1, @process.id)
end
end

test "claim jobs using a wildcard" do
AddToBufferJob.perform_later("hey")

assert_claimed_jobs(6) do
SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id)
end
end

test "claim jobs using queue prefixes" do
AddToBufferJob.perform_later("hey")

assert_claimed_jobs(1) do
SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("backgr*", SolidQueue::Job.count + 1, @process.id)
end

assert @jobs.none?(&:claimed?)
Expand All @@ -72,25 +84,25 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
AddToBufferJob.set(queue: "mail-ops").perform_later(1)

assert_claimed_jobs(0) do
SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id)
end

AddToBufferJob.set(queue: "mail_ops").perform_later(1)
AddToBufferJob.set(queue: "100x_done").perform_later(1)

assert_claimed_jobs(1) do
SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("mail_*", SolidQueue::Job.count + 1, @process.id)
end
assert_equal "mail_ops", SolidQueue::ClaimedExecution.last.job.queue_name

assert_claimed_jobs(0) do
SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("100%_*", SolidQueue::Job.count + 1, @process.id)
end

AddToBufferJob.set(queue: "100%_done").perform_later(1)

assert_claimed_jobs(1) do
SolidQueue::ReadyExecution.claim("100%_*", 1, 42)
SolidQueue::ReadyExecution.claim("100%_*", 1, @process.id)
end
assert_equal "100%_done", SolidQueue::ClaimedExecution.last.job.queue_name
end
Expand All @@ -101,7 +113,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
SolidQueue::Queue.find_by_name("backend").pause

assert_claimed_jobs(1) do
SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim("*", SolidQueue::Job.count + 1, @process.id)
end

@jobs.each(&:reload)
Expand All @@ -112,15 +124,15 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
AddToBufferJob.perform_later("hey")

assert_claimed_jobs(6) do
SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim(%w[ backe* background ], SolidQueue::Job.count + 1, @process.id)
end
end

test "claim jobs for queue without jobs at the moment using prefixes" do
AddToBufferJob.perform_later("hey")

assert_claimed_jobs(0) do
SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, 42)
SolidQueue::ReadyExecution.claim(%w[ none* ], SolidQueue::Job.count + 1, @process.id)
end
end

Expand All @@ -129,7 +141,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
job = SolidQueue::Job.last

assert_claimed_jobs(3) do
SolidQueue::ReadyExecution.claim("*", 3, 42)
SolidQueue::ReadyExecution.claim("*", 3, @process.id)
end

assert job.reload.claimed?
Expand All @@ -145,7 +157,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
assert_equal "background", job.queue_name

assert_claimed_jobs(3) do
SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, 42)
SolidQueue::ReadyExecution.claim(%w[ background backend ], 3, @process.id)
end

assert job.reload.claimed?
Expand All @@ -164,7 +176,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
claimed_jobs = []
4.times do
assert_claimed_jobs(2) do
SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, 42)
SolidQueue::ReadyExecution.claim(%w[ queue_b* queue_a* ], 2, @process.id)
end

claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job)
Expand All @@ -185,7 +197,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
claimed_jobs = []
5.times do
assert_claimed_jobs(2) do
SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, 42)
SolidQueue::ReadyExecution.claim(%w[ queue_a2 queue_c1 queue_b* queue_c2 queue_a* ], 2, @process.id)
end

claimed_jobs += SolidQueue::ClaimedExecution.order(:id).last(2).map(&:job)
Expand Down
Loading