From 23d8956966d8c37a8296ecc145a7d96568ab64db Mon Sep 17 00:00:00 2001 From: qianye Date: Tue, 3 Nov 2026 11:40:24 +0800 Subject: [PATCH 1/3] [ISSUE #10225] Avoid waiting on expired receipt handle locks --- .../proxy/common/ReceiptHandleGroup.java | 29 ++++++++++--- .../proxy/common/ReceiptHandleGroupTest.java | 42 ++++++++++++++++++- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java index 005f18025dd..dd14bb4bfb5 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java @@ -102,6 +102,7 @@ public long getOffset() { } public static class HandleData { + private static final long LOCK_EXPIRED_TIME_MS = TimeUnit.SECONDS.toMillis(4); private final Semaphore semaphore = new Semaphore(1); private final AtomicLong lastLockTimeMs = new AtomicLong(-1L); private volatile boolean needRemove = false; @@ -113,17 +114,35 @@ public HandleData(MessageReceiptHandle messageReceiptHandle) { public Long lock(long timeoutMs) { try { - boolean result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS); + // Try non-blocking acquire first, preserving interruption handling. + boolean result = this.semaphore.tryAcquire(0, TimeUnit.MILLISECONDS); long currentTimeMs = System.currentTimeMillis(); + if (result) { + this.lastLockTimeMs.set(currentTimeMs); + return currentTimeMs; + } + // Check expiration before waiting for the semaphore. + if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + synchronized (this) { + if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + log.warn("HandleData lock expired, acquire lock success without blocking " + + "and reset lock time. MessageReceiptHandle={}, lockTime={}", + messageReceiptHandle, currentTimeMs); + this.lastLockTimeMs.set(currentTimeMs); + return currentTimeMs; + } + } + } + result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS); + currentTimeMs = System.currentTimeMillis(); if (result) { this.lastLockTimeMs.set(currentTimeMs); return currentTimeMs; } else { // if the lock is expired, can be acquired again - long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 3; - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { synchronized (this) { - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { log.warn("HandleData lock expired, acquire lock success and reset lock time. " + "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); this.lastLockTimeMs.set(currentTimeMs); @@ -140,7 +159,7 @@ public Long lock(long timeoutMs) { public void unlock(long lockTimeMs) { // if the lock is expired, we don't need to unlock it - if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockTimeoutMsInHandleGroup() * 2) { + if (System.currentTimeMillis() - lockTimeMs > LOCK_EXPIRED_TIME_MS) { log.warn("HandleData lock expired, unlock fail. MessageReceiptHandle={}, lockTime={}, now={}", messageReceiptHandle, lockTimeMs, System.currentTimeMillis()); return; diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java index fdc1c5a3965..2199a855f85 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.proxy.common; +import java.lang.reflect.Field; import java.time.Duration; import java.util.Random; import java.util.concurrent.CompletableFuture; @@ -23,6 +24,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.message.MessageClientIDSetter; @@ -34,6 +36,7 @@ import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -312,4 +315,41 @@ public void testRemoveOne() { private MessageReceiptHandle createMessageReceiptHandle(String handle, String msgID) { return new MessageReceiptHandle(GROUP, TOPIC, 0, handle, msgID, 0, 0); } -} \ No newline at end of file + + @Test + public void testInterruptedLockAcquisition() { + ReceiptHandleGroup.HandleData data = new ReceiptHandleGroup.HandleData(null); + Thread.currentThread().interrupt(); + try { + assertNull(data.lock(0)); + assertFalse(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + assertNotNull(data.lock(0)); + } + + @Test + public void testAcquireExpiredLockWithoutWaiting() throws Exception { + ReceiptHandleGroup.HandleData data = new ReceiptHandleGroup.HandleData(null); + assertNotNull(data.lock(0)); + assertNull(data.lock(0)); + Field field = ReceiptHandleGroup.HandleData.class.getDeclaredField("lastLockTimeMs"); + field.setAccessible(true); + AtomicLong lastLockTimeMs = (AtomicLong) field.get(data); + lastLockTimeMs.set(System.currentTimeMillis() - 3000); + assertNull(data.lock(0)); + long expiredLockTime = System.currentTimeMillis() - 4100; + lastLockTimeMs.set(expiredLockTime); + + long start = System.nanoTime(); + Long lockTime = data.lock(3000); + assertNotNull(lockTime); + assertTrue(Duration.ofNanos(System.nanoTime() - start).toMillis() < 1000); + // The old owner must not release the semaphore after takeover at four seconds. + data.unlock(expiredLockTime); + assertNull(data.lock(0)); + data.unlock(lockTime); + assertNotNull(data.lock(0)); + } +} From d6b28ad2fa7893fb7c8afcdcc7bb204bdf1f20c4 Mon Sep 17 00:00:00 2001 From: qianye Date: Tue, 3 Nov 2026 12:37:42 +0800 Subject: [PATCH 2/3] [ISSUE #10225] Make receipt handle lock expiration configurable --- .../rocketmq/proxy/common/ReceiptHandleGroup.java | 12 ++++++------ .../apache/rocketmq/proxy/config/ProxyConfig.java | 9 +++++++++ .../proxy/common/ReceiptHandleGroupTest.java | 9 ++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java index dd14bb4bfb5..0f21ec0c8a2 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java @@ -102,7 +102,6 @@ public long getOffset() { } public static class HandleData { - private static final long LOCK_EXPIRED_TIME_MS = TimeUnit.SECONDS.toMillis(4); private final Semaphore semaphore = new Semaphore(1); private final AtomicLong lastLockTimeMs = new AtomicLong(-1L); private volatile boolean needRemove = false; @@ -122,9 +121,10 @@ public Long lock(long timeoutMs) { return currentTimeMs; } // Check expiration before waiting for the semaphore. - if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockExpiredTimeMsInHandleGroup(); + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { synchronized (this) { - if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { log.warn("HandleData lock expired, acquire lock success without blocking " + "and reset lock time. MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); @@ -140,9 +140,9 @@ public Long lock(long timeoutMs) { return currentTimeMs; } else { // if the lock is expired, can be acquired again - if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { synchronized (this) { - if (currentTimeMs - this.lastLockTimeMs.get() > LOCK_EXPIRED_TIME_MS) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { log.warn("HandleData lock expired, acquire lock success and reset lock time. " + "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); this.lastLockTimeMs.set(currentTimeMs); @@ -159,7 +159,7 @@ public Long lock(long timeoutMs) { public void unlock(long lockTimeMs) { // if the lock is expired, we don't need to unlock it - if (System.currentTimeMillis() - lockTimeMs > LOCK_EXPIRED_TIME_MS) { + if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockExpiredTimeMsInHandleGroup()) { log.warn("HandleData lock expired, unlock fail. MessageReceiptHandle={}, lockTime={}, now={}", messageReceiptHandle, lockTimeMs, System.currentTimeMillis()); return; diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java index 6ff495449c0..ae95629bc94 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java @@ -249,6 +249,7 @@ public class ProxyConfig implements ConfigFile { private int renewMaxThreadPoolNums = 4; private int renewThreadPoolQueueCapacity = 300; private long lockTimeoutMsInHandleGroup = TimeUnit.SECONDS.toMillis(3); + private long lockExpiredTimeMsInHandleGroup = TimeUnit.SECONDS.toMillis(4); private long renewAheadTimeMillis = TimeUnit.SECONDS.toMillis(10); private long renewMaxTimeMillis = TimeUnit.HOURS.toMillis(3); private long renewSchedulePeriodMillis = TimeUnit.SECONDS.toMillis(5); @@ -1223,6 +1224,14 @@ public void setLockTimeoutMsInHandleGroup(long lockTimeoutMsInHandleGroup) { this.lockTimeoutMsInHandleGroup = lockTimeoutMsInHandleGroup; } + public long getLockExpiredTimeMsInHandleGroup() { + return lockExpiredTimeMsInHandleGroup; + } + + public void setLockExpiredTimeMsInHandleGroup(long lockExpiredTimeMsInHandleGroup) { + this.lockExpiredTimeMsInHandleGroup = lockExpiredTimeMsInHandleGroup; + } + public long getRenewAheadTimeMillis() { return renewAheadTimeMillis; } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java index 2199a855f85..402004d8564 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroupTest.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.common.consumer.ReceiptHandle; import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.utils.FutureUtils; +import org.apache.rocketmq.proxy.config.ConfigurationManager; import org.apache.rocketmq.proxy.config.InitConfigTest; import org.junit.Before; import org.junit.Test; @@ -331,22 +332,24 @@ public void testInterruptedLockAcquisition() { @Test public void testAcquireExpiredLockWithoutWaiting() throws Exception { + assertEquals(4000, ConfigurationManager.getProxyConfig().getLockExpiredTimeMsInHandleGroup()); + ConfigurationManager.getProxyConfig().setLockExpiredTimeMsInHandleGroup(8000); ReceiptHandleGroup.HandleData data = new ReceiptHandleGroup.HandleData(null); assertNotNull(data.lock(0)); assertNull(data.lock(0)); Field field = ReceiptHandleGroup.HandleData.class.getDeclaredField("lastLockTimeMs"); field.setAccessible(true); AtomicLong lastLockTimeMs = (AtomicLong) field.get(data); - lastLockTimeMs.set(System.currentTimeMillis() - 3000); + lastLockTimeMs.set(System.currentTimeMillis() - 7000); assertNull(data.lock(0)); - long expiredLockTime = System.currentTimeMillis() - 4100; + long expiredLockTime = System.currentTimeMillis() - 8100; lastLockTimeMs.set(expiredLockTime); long start = System.nanoTime(); Long lockTime = data.lock(3000); assertNotNull(lockTime); assertTrue(Duration.ofNanos(System.nanoTime() - start).toMillis() < 1000); - // The old owner must not release the semaphore after takeover at four seconds. + // The old owner must not release the semaphore after the configured expiration. data.unlock(expiredLockTime); assertNull(data.lock(0)); data.unlock(lockTime); From f3748ab757366440f8d7e3e874c7dc98eac814bb Mon Sep 17 00:00:00 2001 From: qianye Date: Tue, 3 Nov 2026 16:09:28 +0800 Subject: [PATCH 3/3] [ISSUE #10225] Extract expired receipt handle lock acquisition --- .../proxy/common/ReceiptHandleGroup.java | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java index 0f21ec0c8a2..fee17a1f13c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java @@ -122,41 +122,36 @@ public Long lock(long timeoutMs) { } // Check expiration before waiting for the semaphore. long expiredTimeMs = ConfigurationManager.getProxyConfig().getLockExpiredTimeMsInHandleGroup(); - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { - synchronized (this) { - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { - log.warn("HandleData lock expired, acquire lock success without blocking " + - "and reset lock time. MessageReceiptHandle={}, lockTime={}", - messageReceiptHandle, currentTimeMs); - this.lastLockTimeMs.set(currentTimeMs); - return currentTimeMs; - } - } + Long lockTimeMs = tryAcquireExpiredLock(currentTimeMs, expiredTimeMs); + if (lockTimeMs != null) { + return lockTimeMs; } result = this.semaphore.tryAcquire(timeoutMs, TimeUnit.MILLISECONDS); currentTimeMs = System.currentTimeMillis(); if (result) { this.lastLockTimeMs.set(currentTimeMs); return currentTimeMs; - } else { - // if the lock is expired, can be acquired again - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { - synchronized (this) { - if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { - log.warn("HandleData lock expired, acquire lock success and reset lock time. " + - "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); - this.lastLockTimeMs.set(currentTimeMs); - return currentTimeMs; - } - } - } } - return null; + return tryAcquireExpiredLock(currentTimeMs, expiredTimeMs); } catch (InterruptedException e) { return null; } } + private Long tryAcquireExpiredLock(long currentTimeMs, long expiredTimeMs) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + synchronized (this) { + if (currentTimeMs - this.lastLockTimeMs.get() > expiredTimeMs) { + log.warn("HandleData lock expired, acquire lock success and reset lock time. " + + "MessageReceiptHandle={}, lockTime={}", messageReceiptHandle, currentTimeMs); + this.lastLockTimeMs.set(currentTimeMs); + return currentTimeMs; + } + } + } + return null; + } + public void unlock(long lockTimeMs) { // if the lock is expired, we don't need to unlock it if (System.currentTimeMillis() - lockTimeMs > ConfigurationManager.getProxyConfig().getLockExpiredTimeMsInHandleGroup()) {