From 73719321c1bbd03a1cc4687a932eba5f4cade963 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 07:20:22 -0600 Subject: [PATCH 1/6] feat(loopback): reject an exhausted pool with its holders instead of waiting forever getNextAvailableLoopbackAddress looped silently when every address was held, so a wedged run looked like a hang. It now waits up to HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS (default 10 minutes, past the unusable-pool quarantine) and throws LoopbackPoolExhaustedError naming the pool range and the pids holding each address, with a warning at 30 seconds. Refs #13 Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: integration-testing-loopback-exhaustion-diagnostic --- CONTRIBUTING.md | 2 +- README.md | 8 +- src/index.ts | 1 + src/loopbackAddressPool.ts | 79 +++++++++++++++++- test/loopbackAddressPoolExhaustion.test.ts | 96 ++++++++++++++++++++++ 5 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 test/loopbackAddressPoolExhaustion.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f19300c..615d0cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ Tests live in `test/` (separate from `src/`) and run on the built-in Node.js tes Internal-only helpers may be exported from their modules for testing (e.g. `runHarperCommand` in `harperLifecycle.ts`) but are deliberately **not** re-exported from `src/index.ts`, keeping them out of the public API. -`loopbackAddressPool.ts` reads its pool file location, address range, and canary ports from the environment once, when it is imported, so `test/loopbackAddressPool.test.ts` sets them before a dynamic import: a private pool file in its own temp directory, and `127.0.0.1`, the one address every OS has without aliases. Its foreign-listener cases need an OS that lets an exclusive bind of that address succeed beside a listener on all interfaces, as macOS and Windows do; where the bind conflicts instead (Linux), the conflict canary refuses the address first and those cases skip themselves. That is why CI includes a macOS job alongside Linux and Windows. +`loopbackAddressPool.ts` reads its pool file location, address range, and canary ports from the environment once, when it is imported, so `test/loopbackAddressPool.test.ts` sets them before a dynamic import: a private pool file in its own temp directory, and `127.0.0.1`, the one address every OS has without aliases. Its foreign-listener cases need an OS that lets an exclusive bind of that address succeed beside a listener on all interfaces, as macOS and Windows do; where the bind conflicts instead (Linux), the conflict canary refuses the address first and those cases skip themselves. That is why CI includes a macOS job alongside Linux and Windows. `test/loopbackAddressPoolExhaustion.test.ts` is separate because it needs its own timeout and pool size, which are read the same way: it fills the pool with live pids and expects the allocation to reject naming them. ## Releases diff --git a/README.md b/README.md index 4ef1fb9..ae30fa2 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,10 @@ try { Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when another process already accepts connections on the operations or HTTP port of a newly allocated address (see [Other services listening on Harper's ports](#other-services-listening-on-harpers-ports)). It extends `Error` with `name` set to `'ForeignListenerError'`, plus `loopbackAddress` and `port` properties naming what was reached. The address is returned to the pool before it is thrown. Set `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS=1` (or `true`) to get a warning instead. +### `LoopbackPoolExhaustedError` + +Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when no address became free within `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`, 10 minutes). It extends `Error` with `name` set to `'LoopbackPoolExhaustedError'`; the message gives the pool range and, for each holding pid, the addresses it holds, plus any unclaimed address a Harper node still has bound. The same listing is logged as a `[loopback-pool]` warning once a wait reaches 30 seconds (or a quarter of the timeout, if shorter). See [Loopback Pool Utilities](#loopback-pool-utilities). + ### `targz(dirPath)` Packs and compresses a directory into a base64-encoded tar.gz string. Useful for `deploy_component` Operations API calls. @@ -298,12 +302,14 @@ Packs and compresses a directory into a base64-encoded tar.gz string. Useful for These are used internally by `startHarper` and `teardownHarper`, but are exported for advanced use cases or custom runner integrations. - `validateLoopbackAddressPool(): Promise<{ successful: string[]; failed: { loopbackAddress: string; error: Error }[] }>` - Validates all pool addresses can be bound to -- `getNextAvailableLoopbackAddress(): Promise` - Allocates an address from the pool; throws a `ForeignListenerError` if another process already accepts connections on its operations or HTTP port (see [Other services listening on Harper's ports](#other-services-listening-on-harpers-ports)) +- `getNextAvailableLoopbackAddress(): Promise` - Allocates an address from the pool, waiting for one to be released if all are held; throws a `ForeignListenerError` if another process already accepts connections on its operations or HTTP port (see [Other services listening on Harper's ports](#other-services-listening-on-harpers-ports)), or a `LoopbackPoolExhaustedError` if the wait outlasts `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` - `releaseLoopbackAddress(address: string): Promise` - Returns an address to the pool - `releaseAllLoopbackAddressesForCurrentProcess(): Promise` - Releases all addresses held by this process The pool is one file shared by every process on the machine: `${TMPDIR}/harper-integration-test-loopback-pool.json`, a JSON array with one slot per address holding the PID of the process that reserved it (or `null`), guarded by the `harper-integration-test-loopback-pool.lock` mutex beside it. Updates are published by renaming a complete `*.pending` file over it, so a process killed mid-update leaves the previous pool intact (and possibly an inert `*.pending` file). Releasing an address clears its slot only if this process holds it. +Addresses held by dead processes are reclaimed each time the pool is found full. When every address is held by a live process, allocation waits up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`) and then throws a [`LoopbackPoolExhaustedError`](#loopbackpoolexhaustederror) naming the holders, rather than waiting forever. The default outlasts the quarantine below. Raise it if suites legitimately queue for longer than that. + A pool file that is unparseable, not an array, or an empty array can still be produced by a writer that doesn't publish by rename (an older version of this package killed mid-write) or by an outside edit. Such a file is either about to be completed or has lost reservations whose holders may not have bound their address yet. Allocation does refuse an address that a Harper node is already listening on, but that check cannot see a reservation nothing is listening on yet. So the file is quarantined rather than reset: - `getNextAvailableLoopbackAddress()` waits, and both release functions leave the file untouched, until it has gone 6 minutes without modification. That outlasts `startHarper`'s default startup ceiling (2 minutes, 5 under CI), so a `startHarper` that reserved an address before the file broke has bound it or given up by then. diff --git a/src/index.ts b/src/index.ts index a3f8628..45810b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,7 @@ export { releaseLoopbackAddress, releaseAllLoopbackAddressesForCurrentProcess, ForeignListenerError, + LoopbackPoolExhaustedError, } from './loopbackAddressPool.ts'; export { targz } from './targz.ts'; diff --git a/src/loopbackAddressPool.ts b/src/loopbackAddressPool.ts index b4c2a64..7675207 100644 --- a/src/loopbackAddressPool.ts +++ b/src/loopbackAddressPool.ts @@ -40,6 +40,16 @@ let lockTokenCounter = 0; // reserve-to-bind time: its startup ceiling (DEFAULT_STARTUP_MAX_MS) plus the setup before it. const UNUSABLE_POOL_QUARANTINE_MS = 360000; +// How long getNextAvailableLoopbackAddress waits for a free address before throwing. It must outlast +// UNUSABLE_POOL_QUARANTINE_MS, which a quarantined pool file can make every caller wait out. +const DEFAULT_ACQUIRE_TIMEOUT_MS = 600000; +const ACQUIRE_TIMEOUT_MS = (() => { + const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS || '', 10); + return Number.isNaN(parsed) || parsed < 1 ? DEFAULT_ACQUIRE_TIMEOUT_MS : parsed; +})(); +// A wait this long is no longer a sibling suite about to finish, so it gets one warning naming the holders. +const WAIT_NOTICE_MS = Math.min(30000, ACQUIRE_TIMEOUT_MS / 4); + // Port used as a conflict canary when allocating an address. This MUST be a port that // Harper binds WITHOUT SO_REUSEPORT (the operations API — `OPERATIONS_API_PORT` in // harperLifecycle.ts — does exactly that, main-thread-only). Because that port is @@ -108,6 +118,20 @@ class InvalidLoopbackAddressError extends Error { } } +/** + * Thrown by {@link getNextAvailableLoopbackAddress} when no address became free within + * `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS`. The message names the pool range and the pids + * holding its addresses. + */ +export class LoopbackPoolExhaustedError extends Error { + constructor(waitedMs: number, poolState: string) { + super( + `No loopback address became free within ${Math.round(waitedMs / 1000)}s. ${poolState}\nIf that much queuing is expected, raise HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS (now ${ACQUIRE_TIMEOUT_MS}).` + ); + this.name = 'LoopbackPoolExhaustedError'; + } +} + const ALLOW_FOREIGN_LISTENERS_ENV = 'HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS'; /** @@ -483,6 +507,39 @@ export async function validateLoopbackAddressPool(): Promise<{ ); } +/** + * Says who holds the pool, for a caller that is waiting on it. `conflictPorts` maps a free slot's index + * to the canary port another Harper node was last found binding there. + */ +function describePoolState(pool: LoopbackPool | null, conflictPorts: ReadonlyMap): string { + if (!pool) { + return `The pool file ${HARPER_LOOPBACK_POOL_PATH} is unusable and quarantined (see the earlier [loopback-pool] warning).`; + } + const address = (index: number) => `127.0.0.${index + HARPER_LOOPBACK_POOL_START}`; + const addressesByPid = new Map(); + const occupiedFree: string[] = []; + pool.forEach((pid, index) => { + if (pid !== null) { + addressesByPid.set(pid, [...(addressesByPid.get(pid) ?? []), address(index)]); + } else if (conflictPorts.has(index)) { + occupiedFree.push(`${address(index)} (port ${conflictPorts.get(index)})`); + } + }); + const lines = [ + `Pool ${address(0)}-${address(pool.length - 1)} (${pool.length} addresses) in ${HARPER_LOOPBACK_POOL_PATH}:`, + ...[...addressesByPid].map( + ([pid, addresses]) => ` pid ${pid}${pid === process.pid ? ' (this process)' : ''} holds ${addresses.join(', ')}` + ), + ]; + if (occupiedFree.length > 0) { + lines.push(` unclaimed but bound by another Harper node: ${occupiedFree.join(', ')}`); + } + lines.push( + 'Each pid is alive; `ps -p ` shows what it is, and `lsof -nP -iTCP: -sTCP:LISTEN` what holds a port. Stop what leaked.' + ); + return lines.join('\n'); +} + /** * Retrieves the next available loopback address from the pool using a file-based * locking mechanism to safely allocate addresses across concurrent test processes. @@ -497,7 +554,8 @@ export async function validateLoopbackAddressPool(): Promise<{ * 7. Returns the loopback address (e.g., "127.0.0.2") * * If no addresses are available, or the pool file is quarantined (see readPoolFile), waits and - * retries until one becomes available. + * retries until one becomes available, for up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` + * (default 10 minutes). A wait of 30 seconds or more logs one warning naming the holders. * * **Pool file location:** `${tmpdir()}/harper-integration-test-loopback-pool.json` * **Lock file location:** `${tmpdir()}/harper-integration-test-loopback-pool.lock` @@ -508,6 +566,7 @@ export async function validateLoopbackAddressPool(): Promise<{ * @throws {ForeignListenerError} If another process accepts connections on the address's canary * ports and `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS` is not set (the slot is released) * @throws {LockOwnershipLostError} If ownership is lost on the initial attempt and all three retries + * @throws {LoopbackPoolExhaustedError} If no address became free within the timeout */ export async function getNextAvailableLoopbackAddress(): Promise { // Each index maps to a different loopback address (index 0 -> 127.0.0.2, index 1 -> 127.0.0.3, etc.) @@ -532,10 +591,17 @@ export async function getNextAvailableLoopbackAddress(): Promise { // them within this attempt; the set is cleared before each wait so they can be re-probed once // the lingering node has had time to exit. const triedIndices = new Set(); + // What the exhaustion diagnostic reports: the pool as the last attempt saw it (null while quarantined) + // and the canary port that made each skipped address unusable. + const waitStartedAt = performance.now(); + let noticed = false; + let lastPool: LoopbackPool | null = null; + const conflictPorts = new Map(); while (true) { const assignedIndex = await withLock(async (lockToken) => { // Read the pool file const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); + lastPool = loopbackPool; if (!loopbackPool) return null; // Find the first available index we haven't already found in-use this attempt @@ -624,6 +690,7 @@ export async function getNextAvailableLoopbackAddress(): Promise { // not to re-try it this attempt; loop to claim the next available one. await releaseLoopbackAddress(loopbackAddress); triedIndices.add(assignedIndex); + conflictPorts.set(assignedIndex, conflictingPort); console.warn( `[loopback-pool] ${loopbackAddress} is still in use by another Harper node (port ${conflictingPort} bound); skipping to avoid an SO_REUSEPORT co-bind.` ); @@ -633,6 +700,16 @@ export async function getNextAvailableLoopbackAddress(): Promise { // No available addresses; clear the per-attempt skip set so poisoned addresses can be // re-probed (their lingering node may have exited during the wait), then wait and retry. triedIndices.clear(); + const waitedMs = performance.now() - waitStartedAt; + if (waitedMs >= ACQUIRE_TIMEOUT_MS) { + throw new LoopbackPoolExhaustedError(waitedMs, describePoolState(lastPool, conflictPorts)); + } + if (!noticed && waitedMs >= WAIT_NOTICE_MS) { + noticed = true; + console.warn( + `[loopback-pool] Still waiting for a free loopback address after ${Math.round(waitedMs / 1000)}s; giving up after ${Math.round(ACQUIRE_TIMEOUT_MS / 1000)}s. ${describePoolState(lastPool, conflictPorts)}` + ); + } await sleep(RETRY_DELAY_MS); } } diff --git a/test/loopbackAddressPoolExhaustion.test.ts b/test/loopbackAddressPoolExhaustion.test.ts new file mode 100644 index 0000000..22c38a5 --- /dev/null +++ b/test/loopbackAddressPoolExhaustion.test.ts @@ -0,0 +1,96 @@ +import { test, after, mock } from 'node:test'; +import { match, ok, strictEqual } from 'node:assert'; +import { spawn } from 'node:child_process'; +import { createServer, type AddressInfo } from 'node:net'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(port)); + }); + }); +} + +// The pool reads these once, when it is imported. 127.0.0.1 and 127.0.0.2 are held in every case below, +// so no address is ever bound except 127.0.0.1 in the case that frees it. +const poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); +const canaryPort = await freePort(); +process.env.TMPDIR = process.env.TEMP = poolDir; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '2'; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS = '2500'; +process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(canaryPort); +process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT = String(canaryPort); +delete process.env.HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS; +const { getNextAvailableLoopbackAddress, releaseLoopbackAddress, LoopbackPoolExhaustedError } = await import( + '../src/loopbackAddressPool.ts' +); + +const holder = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e6)'], { stdio: 'ignore' }); +ok(holder.pid, 'the holder process started'); +const holderPid = holder.pid; + +after(() => { + holder.kill(); + rmSync(poolDir, { recursive: true, force: true }); +}); + +function writePool(pool: (number | null)[]) { + writeFileSync(join(poolDir, 'harper-integration-test-loopback-pool.json'), JSON.stringify(pool)); +} + +test('an exhausted pool rejects with the pool range and the pids holding it, instead of waiting forever', async () => { + writePool([process.pid, holderPid]); + const warn = mock.method(console, 'warn', () => {}); + try { + const error = await getNextAvailableLoopbackAddress().then( + () => { + throw new Error('an address was handed out from a full pool'); + }, + (rejection) => rejection + ); + ok(error instanceof LoopbackPoolExhaustedError); + strictEqual(error.name, 'LoopbackPoolExhaustedError'); + match(error.message, /127\.0\.0\.1-127\.0\.0\.2 \(2 addresses\)/); + match(error.message, new RegExp(`pid ${process.pid} \\(this process\\) holds 127\\.0\\.0\\.1`)); + match(error.message, new RegExp(`pid ${holderPid} holds 127\\.0\\.0\\.2`)); + match(error.message, /HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS/); + + const notices = warn.mock.calls.map((call) => String(call.arguments[0])).filter((text) => text.includes('Still waiting')); + strictEqual(notices.length, 1, 'one warning names the holders before the wait ends'); + match(notices[0], new RegExp(`pid ${holderPid} holds 127\\.0\\.0\\.2`)); + } finally { + warn.mock.restore(); + writePool([null, null]); + } +}); + +test('an address freed during the wait is handed out rather than rejected', async () => { + writePool([process.pid, holderPid]); + const acquiring = getNextAvailableLoopbackAddress(); + setTimeout(() => releaseLoopbackAddress('127.0.0.1'), 300); + const address = await acquiring; + try { + strictEqual(address, '127.0.0.1'); + } finally { + await releaseLoopbackAddress(address); + } +}); + +test('a pool holding only dead pids is reclaimed, not reported as exhausted', async () => { + const dead = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' }); + await new Promise((resolve) => dead.once('exit', resolve)); + writePool([dead.pid!, dead.pid!]); + const address = await getNextAvailableLoopbackAddress(); + try { + match(address, /^127\.0\.0\.[12]$/); + } finally { + await releaseLoopbackAddress(address); + } +}); From 938965fe1618f0c0f5a4be68b78ea068792c82e0 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 07:31:27 -0600 Subject: [PATCH 2/6] fix(loopback): claim a slot freed from dead holders instead of waiting or rejecting The pass that freed dead holders' slots slept a second before looking again, and past the deadline it threw instead of claiming the slot it had just freed. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: integration-testing-loopback-exhaustion-diagnostic --- src/loopbackAddressPool.ts | 15 +++++++---- test/loopbackAddressPoolExhaustion.test.ts | 29 ++++++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/loopbackAddressPool.ts b/src/loopbackAddressPool.ts index 7675207..4110a4b 100644 --- a/src/loopbackAddressPool.ts +++ b/src/loopbackAddressPool.ts @@ -47,7 +47,6 @@ const ACQUIRE_TIMEOUT_MS = (() => { const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS || '', 10); return Number.isNaN(parsed) || parsed < 1 ? DEFAULT_ACQUIRE_TIMEOUT_MS : parsed; })(); -// A wait this long is no longer a sibling suite about to finish, so it gets one warning naming the holders. const WAIT_NOTICE_MS = Math.min(30000, ACQUIRE_TIMEOUT_MS / 4); // Port used as a conflict canary when allocating an address. This MUST be a port that @@ -591,13 +590,12 @@ export async function getNextAvailableLoopbackAddress(): Promise { // them within this attempt; the set is cleared before each wait so they can be re-probed once // the lingering node has had time to exit. const triedIndices = new Set(); - // What the exhaustion diagnostic reports: the pool as the last attempt saw it (null while quarantined) - // and the canary port that made each skipped address unusable. const waitStartedAt = performance.now(); let noticed = false; let lastPool: LoopbackPool | null = null; const conflictPorts = new Map(); while (true) { + let reclaimed = false; const assignedIndex = await withLock(async (lockToken) => { // Read the pool file const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); @@ -615,7 +613,7 @@ export async function getNextAvailableLoopbackAddress(): Promise { if (index === null) { // Nothing available - remove any dead processes from the pool and wait for one to become available - removeDeadProcessesFromPool(loopbackPool); + reclaimed = removeDeadProcessesFromPool(loopbackPool); } else { // Assign the process PID to that index to mark it as used loopbackPool[index] = process.pid; @@ -697,6 +695,9 @@ export async function getNextAvailableLoopbackAddress(): Promise { continue; } + // Dead holders' slots were just freed and published, so claim one now rather than waiting out the delay or the deadline. + if (reclaimed) continue; + // No available addresses; clear the per-attempt skip set so poisoned addresses can be // re-probed (their lingering node may have exited during the wait), then wait and retry. triedIndices.clear(); @@ -717,16 +718,20 @@ export async function getNextAvailableLoopbackAddress(): Promise { /** * Removes any dead processes from the loopback pool. * @param loopbackPool + * @returns Whether any slot was freed */ -function removeDeadProcessesFromPool(loopbackPool: LoopbackPool) { +function removeDeadProcessesFromPool(loopbackPool: LoopbackPool): boolean { + let reclaimed = false; loopbackPool.forEach((pid, index) => { if (pid === null) return; try { process.kill(pid, 0); } catch { loopbackPool[index] = null; + reclaimed = true; } }); + return reclaimed; } /** diff --git a/test/loopbackAddressPoolExhaustion.test.ts b/test/loopbackAddressPoolExhaustion.test.ts index 22c38a5..6c673a8 100644 --- a/test/loopbackAddressPoolExhaustion.test.ts +++ b/test/loopbackAddressPoolExhaustion.test.ts @@ -5,6 +5,7 @@ import { createServer, type AddressInfo } from 'node:net'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; function freePort(): Promise { return new Promise((resolve, reject) => { @@ -17,8 +18,7 @@ function freePort(): Promise { }); } -// The pool reads these once, when it is imported. 127.0.0.1 and 127.0.0.2 are held in every case below, -// so no address is ever bound except 127.0.0.1 in the case that frees it. +// The pool reads these once, when it is imported. const poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); const canaryPort = await freePort(); process.env.TMPDIR = process.env.TEMP = poolDir; @@ -73,9 +73,24 @@ test('an exhausted pool rejects with the pool range and the pids holding it, ins test('an address freed during the wait is handed out rather than rejected', async () => { writePool([process.pid, holderPid]); - const acquiring = getNextAvailableLoopbackAddress(); - setTimeout(() => releaseLoopbackAddress('127.0.0.1'), 300); - const address = await acquiring; + const [address] = await Promise.all([ + getNextAvailableLoopbackAddress(), + sleep(300).then(() => releaseLoopbackAddress('127.0.0.1')), + ]); + try { + strictEqual(address, '127.0.0.1'); + } finally { + await releaseLoopbackAddress(address); + } +}); + +test('a holder that dies after the deadline is reclaimed rather than reported', async () => { + const dying = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e6)'], { stdio: 'ignore' }); + writePool([dying.pid!, holderPid]); + const [address] = await Promise.all([ + getNextAvailableLoopbackAddress(), + sleep(2500).then(() => dying.kill()), + ]); try { strictEqual(address, '127.0.0.1'); } finally { @@ -86,10 +101,10 @@ test('an address freed during the wait is handed out rather than rejected', asyn test('a pool holding only dead pids is reclaimed, not reported as exhausted', async () => { const dead = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' }); await new Promise((resolve) => dead.once('exit', resolve)); - writePool([dead.pid!, dead.pid!]); + writePool([dead.pid!, holderPid]); const address = await getNextAvailableLoopbackAddress(); try { - match(address, /^127\.0\.0\.[12]$/); + strictEqual(address, '127.0.0.1'); } finally { await releaseLoopbackAddress(address); } From 93891ad60264f1ec0a79fdb1f4039f1067431801 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 07:39:45 -0600 Subject: [PATCH 3/6] fix(loopback): claim a reclaimed slot in the pass that frees it Search again after dropping dead holders, under the same lock, rather than carrying a flag to the next pass. Also tighten the exhaustion tests and trim comments. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: integration-testing-loopback-exhaustion-diagnostic --- src/loopbackAddressPool.ts | 39 ++++++++-------------- test/loopbackAddressPoolExhaustion.test.ts | 30 +++++++++++------ 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/loopbackAddressPool.ts b/src/loopbackAddressPool.ts index 4110a4b..b79ba80 100644 --- a/src/loopbackAddressPool.ts +++ b/src/loopbackAddressPool.ts @@ -40,8 +40,7 @@ let lockTokenCounter = 0; // reserve-to-bind time: its startup ceiling (DEFAULT_STARTUP_MAX_MS) plus the setup before it. const UNUSABLE_POOL_QUARANTINE_MS = 360000; -// How long getNextAvailableLoopbackAddress waits for a free address before throwing. It must outlast -// UNUSABLE_POOL_QUARANTINE_MS, which a quarantined pool file can make every caller wait out. +// Must outlast UNUSABLE_POOL_QUARANTINE_MS, which a quarantined pool file can make every caller wait out. const DEFAULT_ACQUIRE_TIMEOUT_MS = 600000; const ACQUIRE_TIMEOUT_MS = (() => { const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS || '', 10); @@ -506,10 +505,7 @@ export async function validateLoopbackAddressPool(): Promise<{ ); } -/** - * Says who holds the pool, for a caller that is waiting on it. `conflictPorts` maps a free slot's index - * to the canary port another Harper node was last found binding there. - */ +/** `conflictPorts` maps a free slot's index to the canary port another Harper node was last found binding there. */ function describePoolState(pool: LoopbackPool | null, conflictPorts: ReadonlyMap): string { if (!pool) { return `The pool file ${HARPER_LOOPBACK_POOL_PATH} is unusable and quarantined (see the earlier [loopback-pool] warning).`; @@ -595,7 +591,6 @@ export async function getNextAvailableLoopbackAddress(): Promise { let lastPool: LoopbackPool | null = null; const conflictPorts = new Map(); while (true) { - let reclaimed = false; const assignedIndex = await withLock(async (lockToken) => { // Read the pool file const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); @@ -603,18 +598,19 @@ export async function getNextAvailableLoopbackAddress(): Promise { if (!loopbackPool) return null; // Find the first available index we haven't already found in-use this attempt - let index: number | null = null; - for (let i = 0; i < loopbackPool.length; i++) { - if (loopbackPool[i] === null && !triedIndices.has(i)) { - index = i; - break; + const findAvailableIndex = (): number | null => { + for (let i = 0; i < loopbackPool.length; i++) { + if (loopbackPool[i] === null && !triedIndices.has(i)) return i; } - } - + return null; + }; + let index = findAvailableIndex(); if (index === null) { - // Nothing available - remove any dead processes from the pool and wait for one to become available - reclaimed = removeDeadProcessesFromPool(loopbackPool); - } else { + // Nothing available - remove any dead processes from the pool and claim a slot they held, else wait for one to become available + removeDeadProcessesFromPool(loopbackPool); + index = findAvailableIndex(); + } + if (index !== null) { // Assign the process PID to that index to mark it as used loopbackPool[index] = process.pid; } @@ -695,9 +691,6 @@ export async function getNextAvailableLoopbackAddress(): Promise { continue; } - // Dead holders' slots were just freed and published, so claim one now rather than waiting out the delay or the deadline. - if (reclaimed) continue; - // No available addresses; clear the per-attempt skip set so poisoned addresses can be // re-probed (their lingering node may have exited during the wait), then wait and retry. triedIndices.clear(); @@ -718,20 +711,16 @@ export async function getNextAvailableLoopbackAddress(): Promise { /** * Removes any dead processes from the loopback pool. * @param loopbackPool - * @returns Whether any slot was freed */ -function removeDeadProcessesFromPool(loopbackPool: LoopbackPool): boolean { - let reclaimed = false; +function removeDeadProcessesFromPool(loopbackPool: LoopbackPool) { loopbackPool.forEach((pid, index) => { if (pid === null) return; try { process.kill(pid, 0); } catch { loopbackPool[index] = null; - reclaimed = true; } }); - return reclaimed; } /** diff --git a/test/loopbackAddressPoolExhaustion.test.ts b/test/loopbackAddressPoolExhaustion.test.ts index 6c673a8..3800b44 100644 --- a/test/loopbackAddressPoolExhaustion.test.ts +++ b/test/loopbackAddressPoolExhaustion.test.ts @@ -1,6 +1,7 @@ import { test, after, mock } from 'node:test'; import { match, ok, strictEqual } from 'node:assert'; -import { spawn } from 'node:child_process'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { once } from 'node:events'; import { createServer, type AddressInfo } from 'node:net'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -18,7 +19,6 @@ function freePort(): Promise { }); } -// The pool reads these once, when it is imported. const poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); const canaryPort = await freePort(); process.env.TMPDIR = process.env.TEMP = poolDir; @@ -41,6 +41,12 @@ after(() => { rmSync(poolDir, { recursive: true, force: true }); }); +async function kill(child: ChildProcess) { + const exited = once(child, 'exit'); + child.kill('SIGKILL'); + await exited; +} + function writePool(pool: (number | null)[]) { writeFileSync(join(poolDir, 'harper-integration-test-loopback-pool.json'), JSON.stringify(pool)); } @@ -75,7 +81,7 @@ test('an address freed during the wait is handed out rather than rejected', asyn writePool([process.pid, holderPid]); const [address] = await Promise.all([ getNextAvailableLoopbackAddress(), - sleep(300).then(() => releaseLoopbackAddress('127.0.0.1')), + sleep(500).then(() => releaseLoopbackAddress('127.0.0.1')), ]); try { strictEqual(address, '127.0.0.1'); @@ -86,15 +92,19 @@ test('an address freed during the wait is handed out rather than rejected', asyn test('a holder that dies after the deadline is reclaimed rather than reported', async () => { const dying = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e6)'], { stdio: 'ignore' }); - writePool([dying.pid!, holderPid]); - const [address] = await Promise.all([ - getNextAvailableLoopbackAddress(), - sleep(2500).then(() => dying.kill()), - ]); try { - strictEqual(address, '127.0.0.1'); + writePool([dying.pid!, holderPid]); + const [address] = await Promise.all([ + getNextAvailableLoopbackAddress(), + sleep(2500).then(() => kill(dying)), + ]); + try { + strictEqual(address, '127.0.0.1'); + } finally { + await releaseLoopbackAddress(address); + } } finally { - await releaseLoopbackAddress(address); + dying.kill('SIGKILL'); } }); From 06197a7e5459ff5fd7b6ad3430193606e4c6ce6c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 09:04:47 -0600 Subject: [PATCH 4/6] fix(loopback): hold an address by listening on its claim port, not by storing a PID The pool recorded each holder's PID in a JSON file under os.tmpdir(). A runner in a PID namespace stores a PID that names an unrelated process on the host, so its slots never came back, and a host runner's PID looks dead from inside a namespace, so its address was handed out twice. Runs with different TMPDIRs kept separate pool files and handed out the same addresses. A process now holds an address by listening on
: (default 9924, HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT). The kernel allows one such listener per network namespace and closes it when its process exits, so a dead holder's address is free at once, whatever its PID namespace or TMPDIR. The pool file, its lock, the quarantine and the PID sweep are gone, and with them #44's deferred lock-wait finding. Allocation also skips an address whose HTTPS, MQTT or MQTTS port is still bound on it, besides the operations and HTTP ports. On Linux, a conflict that a bind of an address outside the pool shares comes from a listener on all interfaces: that now throws ForeignListenerError at once instead of waiting out the timeout. The exhaustion message names each holder from its claim's reply: pid, PID namespace and host name. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: integration-testing-44-pool-liveness-root-fix --- .github/workflows/verify-pr.yaml | 7 + CONTRIBUTING.md | 4 +- README.md | 22 +- src/harperLifecycle.ts | 10 +- src/loopbackAddressPool.ts | 846 ++++++++----------- test/fixtures/loopbackHolder.ts | 14 + test/loopback-lock-fencing.test.ts | 383 --------- test/loopbackAddressPool.test.ts | 344 +++----- test/loopbackAddressPoolExhaustion.test.ts | 222 +++-- test/loopbackAddressPoolProbeFailure.test.ts | 29 +- test/loopbackHolder.ts | 77 ++ 11 files changed, 761 insertions(+), 1197 deletions(-) create mode 100644 test/fixtures/loopbackHolder.ts delete mode 100644 test/loopback-lock-fencing.test.ts create mode 100644 test/loopbackHolder.ts diff --git a/.github/workflows/verify-pr.yaml b/.github/workflows/verify-pr.yaml index 782b6bc..b8fb5df 100644 --- a/.github/workflows/verify-pr.yaml +++ b/.github/workflows/verify-pr.yaml @@ -38,8 +38,15 @@ jobs: run: npm run check - name: Build run: npm run build + # The loopback pool's PID-namespace test runs its holder under `unshare --user`, which Ubuntu's + # AppArmor default blocks for unprivileged users. + - name: Allow unprivileged user namespaces + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true # Tests run the TypeScript sources directly via Node's native type stripping (Node >= 22.18, # enforced by devEngines). They spawn short-lived node processes and bind loopback ports, so # no real Harper instance or loopback-pool setup is required. - name: Run tests run: npm test + env: + LOOPBACK_POOL_TEST_REQUIRE_PID_NAMESPACE: ${{ runner.os == 'Linux' && '1' || '' }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 615d0cd..2cae221 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Contributors are encouraged to communicate with maintainers in issues or other c Source files are located in `src/`. These are built to the `dist/` directory. The published package includes `dist/`, `scripts/`, and the regular npm metadata and documentation files. -The `src/index.ts` is the source for the main export. This is the public re-export of all the various utilities from `src/harperLifecycle.ts`, `targz.ts`, and more. The `src/run.ts` is the source for the `harper-integration-test-run` bin script. The internal `src/harperInstanceRegistry.ts` publishes each running Harper instance to a shared on-disk registry, and `src/harperMonitor.ts` is the singleton monitor process that reads it and reaps instances whose test runner died without cleaning up (see README's *Orphaned Instance Monitor* section). A registry record covers a whole process group, so the monitor — the only half that can see when that group is finished — is also the only writer that removes one; the lifecycle side registers and never deregisters. The pool behind the loopback utilities, `src/loopbackAddressPool.ts`, never writes an unusable pool file back as an empty pool before its quarantine ends (see README's *Loopback Pool Utilities*): every path that rewrites the file returns early when `readPoolFile` reports it quarantined (`null`), so a new write path must do the same. Harper's `restart` operation replaces the process the harness spawned with a detached one the harness never tracked, so `killHarper` identifies the node by `/hdb.pid` once the spawned process has exited (covered by the relaunch tests in `test/harperLifecycle.test.ts`); the runner-exit handlers and the monitor still key on the spawned process and do not follow a restart. And the `scripts/setup-loopback.sh` is the source for the `harper-integration-test-setup-loopback` bin script. +The `src/index.ts` is the source for the main export. This is the public re-export of all the various utilities from `src/harperLifecycle.ts`, `targz.ts`, and more. The `src/run.ts` is the source for the `harper-integration-test-run` bin script. The internal `src/harperInstanceRegistry.ts` publishes each running Harper instance to a shared on-disk registry, and `src/harperMonitor.ts` is the singleton monitor process that reads it and reaps instances whose test runner died without cleaning up (see README's *Orphaned Instance Monitor* section). A registry record covers a whole process group, so the monitor — the only half that can see when that group is finished — is also the only writer that removes one; the lifecycle side registers and never deregisters. The pool behind the loopback utilities, `src/loopbackAddressPool.ts`, has one source of truth for who holds an address: a listening socket on that address's claim port, which the kernel keeps unique per network namespace and closes when its process dies (see README's *Loopback Pool Utilities*). Do not add a second record of holders, such as a file of PIDs: a PID means nothing outside its own PID namespace, and a file is shared only by processes that see the same filesystem, which is not the scope in which addresses collide. A claim joins the process-wide map only once allocation succeeds, and every failure after the bind closes it. Harper's `restart` operation replaces the process the harness spawned with a detached one the harness never tracked, so `killHarper` identifies the node by `/hdb.pid` once the spawned process has exited (covered by the relaunch tests in `test/harperLifecycle.test.ts`); the runner-exit handlers and the monitor still key on the spawned process and do not follow a restart. And the `scripts/setup-loopback.sh` is the source for the `harper-integration-test-setup-loopback` bin script. The package is `"type": "module"` — all source files are ESM by default. @@ -29,7 +29,7 @@ Tests live in `test/` (separate from `src/`) and run on the built-in Node.js tes Internal-only helpers may be exported from their modules for testing (e.g. `runHarperCommand` in `harperLifecycle.ts`) but are deliberately **not** re-exported from `src/index.ts`, keeping them out of the public API. -`loopbackAddressPool.ts` reads its pool file location, address range, and canary ports from the environment once, when it is imported, so `test/loopbackAddressPool.test.ts` sets them before a dynamic import: a private pool file in its own temp directory, and `127.0.0.1`, the one address every OS has without aliases. Its foreign-listener cases need an OS that lets an exclusive bind of that address succeed beside a listener on all interfaces, as macOS and Windows do; where the bind conflicts instead (Linux), the conflict canary refuses the address first and those cases skip themselves. That is why CI includes a macOS job alongside Linux and Windows. `test/loopbackAddressPoolExhaustion.test.ts` is separate because it needs its own timeout and pool size, which are read the same way: it fills the pool with live pids and expects the allocation to reject naming them. +`loopbackAddressPool.ts` reads its address range, claim port, and probe ports from the environment once, when it is imported, so `test/loopbackAddressPool.test.ts` sets them before a dynamic import. It uses `127.0.0.1`, the one address every OS has without aliases, and ephemeral ports, so neither a local Harper nor another test file running at the same time affects it. A listener on all interfaces is found by binding on Linux and by connecting on macOS and Windows, so the foreign-listener cases take a different path on each, which is why CI runs all three. `test/loopbackAddressPoolExhaustion.test.ts` is separate because it needs its own timeout, which is read the same way. Its holders are child processes (`test/fixtures/loopbackHolder.ts`), each with its own `TMPDIR`; one runs in its own PID namespace through `unshare`. That case skips itself where unprivileged user namespaces are blocked (Ubuntu's AppArmor default), so the Linux CI job lifts that restriction and sets `LOOPBACK_POOL_TEST_REQUIRE_PID_NAMESPACE` to turn the skip into a failure. ## Releases diff --git a/README.md b/README.md index ae30fa2..dc1c102 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ npm install --save-dev @harperfast/integration-testing ## Framework Agnosticism -The core lifecycle and loopback pool APIs in this package — `startHarper`, `teardownHarper`, `killHarper`, and the loopback utilities — are **not tied to any specific test runner or framework**. They spawn and manage Harper child processes and allocate loopback addresses safely across concurrent processes using a file-based locking mechanism. This means they can be used with any testing framework that supports async setup and teardown: Node.js `node:test`, Vitest, Playwright, or anything else. +The core lifecycle and loopback pool APIs in this package — `startHarper`, `teardownHarper`, `killHarper`, and the loopback utilities — are **not tied to any specific test runner or framework**. They spawn and manage Harper child processes and allocate loopback addresses safely across concurrent processes, whatever their PID namespace or `TMPDIR`. This means they can be used with any testing framework that supports async setup and teardown: Node.js `node:test`, Vitest, Playwright, or anything else. The key responsibility these APIs place on you is **thinking critically about Harper instance scope**. Starting a Harper instance has real cost — it spawns a process, allocates a loopback address from the pool, and creates a temporary install directory. How you scope instances depends on your framework and what you're testing. A single instance shared across many assertions in one logical test block may be perfectly appropriate; spinning up a fresh instance for every individual assertion would be wasteful. What matters is that concurrently executing scopes each have their own instance and loopback address — the pool handles cross-process allocation safely, but your setup and teardown hooks determine the boundaries. @@ -84,7 +84,7 @@ Each node binds its fixed ports on its own loopback address, so nodes on differe `getNextAvailableLoopbackAddress`, and therefore `startHarper`, refuses an address when another process already accepts connections on its operations or HTTP port, throwing a `ForeignListenerError` that names the port. Stop the other process or bind it to a specific address (`lsof -nP -iTCP:9926 -sTCP:LISTEN` names it). To run anyway, set `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS=1`, which turns the error into a warning. -On Linux, binding a pool address beside such a listener already fails, so the pool reports each address as still in use by another node instead. +On Linux, such a listener also stops anything else from binding that port on a single address, Harper included, so there the same error covers Harper's HTTPS, MQTT and MQTTS ports too. With the variable set on Linux, the address is handed out even though a Harper node bound to that address under the other listener cannot be ruled out. ## API @@ -287,11 +287,11 @@ try { ### `ForeignListenerError` -Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when another process already accepts connections on the operations or HTTP port of a newly allocated address (see [Other services listening on Harper's ports](#other-services-listening-on-harpers-ports)). It extends `Error` with `name` set to `'ForeignListenerError'`, plus `loopbackAddress` and `port` properties naming what was reached. The address is returned to the pool before it is thrown. Set `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS=1` (or `true`) to get a warning instead. +Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when another process already accepts connections on the operations or HTTP port of a newly allocated address, or on Linux on any of its Harper ports through a listener on all interfaces (see [Other services listening on Harper's ports](#other-services-listening-on-harpers-ports)). It extends `Error` with `name` set to `'ForeignListenerError'`, plus `loopbackAddress` and `port` properties naming what was reached. The address is released before the error is thrown. Set `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS=1` (or `true`) to get a warning instead. ### `LoopbackPoolExhaustedError` -Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when no address became free within `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`, 10 minutes). It extends `Error` with `name` set to `'LoopbackPoolExhaustedError'`; the message gives the pool range and, for each holding pid, the addresses it holds, plus any unclaimed address a Harper node still has bound. The same listing is logged as a `[loopback-pool]` warning once a wait reaches 30 seconds (or a quarter of the timeout, if shorter). See [Loopback Pool Utilities](#loopback-pool-utilities). +Thrown by `getNextAvailableLoopbackAddress`, and so by `startHarper`, when no address became free within `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`, 10 minutes). It extends `Error` with `name` set to `'LoopbackPoolExhaustedError'`. The message gives the pool range and, for each holding process, the addresses it holds: its pid, the PID namespace when that differs from this process's, and the host name when that differs. It also lists any address held by a process that did not identify itself, and any unclaimed address a Harper node still has bound. The same listing is logged as a `[loopback-pool]` warning once a wait reaches 30 seconds (or a quarter of the timeout, if shorter). See [Loopback Pool Utilities](#loopback-pool-utilities). ### `targz(dirPath)` @@ -306,19 +306,13 @@ These are used internally by `startHarper` and `teardownHarper`, but are exporte - `releaseLoopbackAddress(address: string): Promise` - Returns an address to the pool - `releaseAllLoopbackAddressesForCurrentProcess(): Promise` - Releases all addresses held by this process -The pool is one file shared by every process on the machine: `${TMPDIR}/harper-integration-test-loopback-pool.json`, a JSON array with one slot per address holding the PID of the process that reserved it (or `null`), guarded by the `harper-integration-test-loopback-pool.lock` mutex beside it. Updates are published by renaming a complete `*.pending` file over it, so a process killed mid-update leaves the previous pool intact (and possibly an inert `*.pending` file). Releasing an address clears its slot only if this process holds it. +A process holds an address by listening on its **claim port** (`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT`, default `9924`). The kernel allows one listener per address and port, so while one process holds an address, no other process can claim it. The kernel also closes the listener when its process exits, however it exits. This covers every process in one network namespace, which is where two Harper nodes on one address would collide, whatever their PID namespace or `TMPDIR`. Processes in separate network namespaces, such as most containers, have separate loopback interfaces and cannot collide. Every run on a machine must use the same claim port. Nothing is written to disk. -Addresses held by dead processes are reclaimed each time the pool is found full. When every address is held by a live process, allocation waits up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`) and then throws a [`LoopbackPoolExhaustedError`](#loopbackpoolexhaustederror) naming the holders, rather than waiting forever. The default outlasts the quarantine below. Raise it if suites legitimately queue for longer than that. +Allocation takes the first address whose claim port is free, so an address left by a process that died is reused at once. It then skips an address on which Harper's operations, HTTP, HTTPS, MQTT or MQTTS port is still bound, such as by a node that outlived a killed test runner, and tries the next. Replication is not checked: harper-pro's default replication port has no host, so every node binds it on all interfaces. Releasing an address only frees a claim this process holds. -A pool file that is unparseable, not an array, or an empty array can still be produced by a writer that doesn't publish by rename (an older version of this package killed mid-write) or by an outside edit. Such a file is either about to be completed or has lost reservations whose holders may not have bound their address yet. Allocation does refuse an address that a Harper node is already listening on, but that check cannot see a reservation nothing is listening on yet. So the file is quarantined rather than reset: +When every address is held, allocation waits up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default `600000`), then throws a [`LoopbackPoolExhaustedError`](#loopbackpoolexhaustederror) naming the holders, rather than waiting forever. Raise the timeout if suites legitimately queue for longer than that. To name its holder, a claim answers a connection with one JSON line: its process's pid, PID namespace and host name. Allocation itself never connects to a claim. -- `getNextAvailableLoopbackAddress()` waits, and both release functions leave the file untouched, until it has gone 6 minutes without modification. That outlasts `startHarper`'s default startup ceiling (2 minutes, 5 under CI), so a `startHarper` that reserved an address before the file broke has bound it or given up by then. -- After that, the pool is reinitialized with every address free and a warning is logged. From then on, an address still in use is protected only by that listening check. -- Deleting the file skips the wait, but only do that once every integration-test run on the machine — including ones using older versions of this package — has stopped. - -The quarantine bounds, rather than eliminates, the risk of handing out an address twice: a caller that holds an address for longer than that without binding it (for example, with a `startupMaxMs` above 5 minutes) is not covered. - -If other processes take over the pool lock as stale (held over 10 seconds) before this process publishes its update, the allocate and release functions retry from the current pool state with a `[loopback-pool]` warning; after four takeovers in a row they reject with an error named `LockOwnershipLostError`. +Versions 1.0.5 and earlier kept the pool as a file of process IDs in `${TMPDIR}/harper-integration-test-loopback-pool.json`. They do not see claims, and this version does not read that file. Until every run on a machine uses this version, the two can hand out one address to two runs before either has started its node. Let runs on the old version finish before switching. The old file can then be deleted. ## Node.js Test Runner diff --git a/src/harperLifecycle.ts b/src/harperLifecycle.ts index f8f23cc..68e979f 100644 --- a/src/harperLifecycle.ts +++ b/src/harperLifecycle.ts @@ -1112,11 +1112,11 @@ export async function teardownHarper(ctx: StartedHarperTestContext): Promise HARPER_LOOPBACK_POOL_MAX) { throw new Error(`HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT must be between 1 and ${HARPER_LOOPBACK_POOL_MAX}`); } -const HARPER_LOOPBACK_POOL_PATH = join(tmpdir(), 'harper-integration-test-loopback-pool.json'); -const HARPER_LOOPBACK_POOL_LOCK_PATH = join(tmpdir(), 'harper-integration-test-loopback-pool.lock'); -// Constants for timeouts and retries -const LOCK_STALE_TIMEOUT_MS = 10000; -const RETRY_DELAY_MS = 1000; -// A one-off stall past LOCK_STALE_TIMEOUT_MS loses the lock once; losing it repeatedly means sections keep -// outlasting it (a stalling event loop, or clock or mtime skew), which retrying cannot fix. -const LOCK_OWNERSHIP_RETRIES = 3; - -let lockTokenCounter = 0; +/** + * An address is held exactly while a process listens on this port of it. The kernel enforces one listener per + * address and port within a network namespace, which is also the scope in which two Harper nodes on one address + * collide, and closes the listener when its process dies, whatever PID namespace or TMPDIR that process has. + * Every cooperating runner must use the same value. + */ +const CLAIM_PORT = (() => { + const value = process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT; + if (!value) return 9924; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error(`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT must be an integer between 1 and 65535 (got ${value})`); + } + return parsed; +})(); -// How long an unusable pool file must go unmodified before it is reinitialized. The reservations it -// lost may not be bound yet, which the conflict canary can't see, so this must outlast startHarper's -// reserve-to-bind time: its startup ceiling (DEFAULT_STARTUP_MAX_MS) plus the setup before it. -const UNUSABLE_POOL_QUARANTINE_MS = 360000; +const RETRY_DELAY_MS = 1000; -// Must outlast UNUSABLE_POOL_QUARANTINE_MS, which a quarantined pool file can make every caller wait out. const DEFAULT_ACQUIRE_TIMEOUT_MS = 600000; const ACQUIRE_TIMEOUT_MS = (() => { const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS || '', 10); @@ -48,46 +47,46 @@ const ACQUIRE_TIMEOUT_MS = (() => { })(); const WAIT_NOTICE_MS = Math.min(30000, ACQUIRE_TIMEOUT_MS / 4); -// Port used as a conflict canary when allocating an address. This MUST be a port that -// Harper binds WITHOUT SO_REUSEPORT (the operations API — `OPERATIONS_API_PORT` in -// harperLifecycle.ts — does exactly that, main-thread-only). Because that port is -// exclusive, a stale/overlapping Harper node still holding the address makes a plain -// (non-reusePort) bind here fail with EADDRINUSE — which is precisely the signal we -// need. Without this check, SO_REUSEPORT on the shared HTTP/replication ports would let -// a freshly-assigned node silently co-bind an address another node still owns, corrupting -// both suites. Override via HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT if the operations -// port ever changes. (Hardcoded rather than imported from harperLifecycle to avoid a -// circular import between these two modules.) -const CONFLICT_PROBE_PORT = (() => { - const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT || '', 10); - // Fall back to the default if unset/non-numeric/out-of-range, so an invalid override - // can't turn into a `server.listen(NaN)` crash during allocation. - return Number.isNaN(parsed) || parsed < 0 || parsed > 65535 ? 9925 : parsed; -})(); +function probePortFromEnv(name: string, fallback: number): number { + const parsed = parseInt(process.env[name] || '', 10); + // An invalid override must not turn into a `server.listen(NaN)` crash during allocation. + return Number.isNaN(parsed) || parsed < 0 || parsed > 65535 ? fallback : parsed; +} -// Second conflict-canary port: Harper's HTTP listener (`HTTP_PORT` in harperLifecycle.ts). -// The operations port above is bound by the MAIN thread without SO_REUSEPORT, so it catches -// a fully-alive (or recently-signalled) node. But the most common contamination shape after -// a kill/teardown is different: the main thread has already exited (freeing the operations -// port) while the HTTP WORKER threads still linger holding the address with SO_REUSEPORT. -// The operations-port probe alone reports such an address as free and hands it out, producing -// a silent SO_REUSEPORT co-bind on the HTTP port. A plain (non-reusePort) bind still fails -// with EADDRINUSE against a lingering SO_REUSEPORT socket (Linux disallows mixing reusePort -// and non-reusePort binders on the same address:port), so probing the HTTP port here closes -// that blind spot. (This detection is Linux-effective — macOS/BSD SO_REUSEPORT semantics differ -// and may not surface EADDRINUSE here, but macOS also doesn't exhibit the silent co-bind in the -// first place, and CI runs on Linux.) Override via HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT -// if HTTP_PORT ever changes. (Hardcoded rather than imported from harperLifecycle to avoid a circular import.) -const HTTP_CONFLICT_PROBE_PORT = (() => { - const parsed = parseInt(process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT || '', 10); - return Number.isNaN(parsed) || parsed < 0 || parsed > 65535 ? 9926 : parsed; +// Harper's operations API (`OPERATIONS_API_PORT` in harperLifecycle.ts) and HTTP (`HTTP_PORT`) ports. Besides the +// leftover-node probe below, these two are checked for another process accepting connections on them (see +// ForeignListenerError). Hardcoded rather than imported from harperLifecycle to avoid a circular import. +const CONFLICT_PROBE_PORT = probePortFromEnv('HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT', 9925); +const HTTP_CONFLICT_PROBE_PORT = probePortFromEnv('HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT', 9926); +const FOREIGN_LISTENER_PROBE_PORTS = [...new Set([CONFLICT_PROBE_PORT, HTTP_CONFLICT_PROBE_PORT])]; + +// Harper's HTTPS, MQTT and MQTTS ports: with the two above, every port startHarper binds on the node's address +// (`ALL_HARPER_PORTS` in harperLifecycle.ts). Not replication: harper-pro's default `replication.securePort` has no +// host, so every node binds it on all interfaces, where it cannot be told apart from a leftover node's. +const OTHER_CONFLICT_PROBE_PORTS = (() => { + const value = process.env.HARPER_INTEGRATION_TEST_OTHER_CONFLICT_PROBE_PORTS; + const defaults = [9927, 1883, 8883]; + if (value === undefined) return defaults; + const ports = value + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + .map(Number); + return ports.every((port) => Number.isInteger(port) && port >= 0 && port <= 65535) ? ports : defaults; })(); -// The set of ports the conflict canary probes, de-duplicated in case the two overrides collide. -const CONFLICT_PROBE_PORTS = [...new Set([CONFLICT_PROBE_PORT, HTTP_CONFLICT_PROBE_PORT])]; +const CONFLICT_PROBE_PORTS = [...new Set([...FOREIGN_LISTENER_PROBE_PORTS, ...OTHER_CONFLICT_PROBE_PORTS])]; +if (CONFLICT_PROBE_PORTS.includes(CLAIM_PORT)) { + throw new Error(`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT (${CLAIM_PORT}) must not be one of Harper's ports`); +} + +// Linux routes all of 127.0.0.0/8 to lo; this address is outside the pool's 127.0.0.x range. +const CONTROL_ADDRESS = '127.255.255.254'; -// Type definitions -type LoopbackPool = (number | null)[]; +const IDENTITY_PROTOCOL = 'harper-integration-test-loopback-claim/1'; +const IDENTITY_SERVE_MS = 2000; +const IDENTITY_QUERY_MS = 1000; +const IDENTITY_MAX_BYTES = 1024; interface LoopbackAddressError extends Error { loopbackAddress: string; @@ -118,7 +117,7 @@ class InvalidLoopbackAddressError extends Error { /** * Thrown by {@link getNextAvailableLoopbackAddress} when no address became free within - * `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS`. The message names the pool range and the pids + * `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS`. The message names the pool range and the processes * holding its addresses. */ export class LoopbackPoolExhaustedError extends Error { @@ -134,7 +133,7 @@ const ALLOW_FOREIGN_LISTENERS_ENV = 'HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTE /** * Thrown by {@link getNextAvailableLoopbackAddress} when another process accepts connections on one of - * a newly allocated address's canary ports before any Harper node runs there. The usual cause is a + * a newly allocated address's Harper ports before any Harper node runs there. The usual cause is a * service listening on all interfaces, which covers every pool address, so another address would not * help. */ @@ -157,232 +156,149 @@ function foreignListenersAllowed(): boolean { return value === '1' || value?.toLowerCase() === 'true'; } -/** A failure to return a refused address must not replace the error that says why it was refused. */ -async function releaseRefusedAddress(address: string): Promise { - try { - await releaseLoopbackAddress(address); - } catch (error) { - console.warn(`[loopback-pool] Could not return ${address} to the pool; it is reclaimed when this process exits:`, error); - } +interface Claim { + server: Server; + /** Identity queries in flight; release resets them so they cannot hold the claim port open. */ + connections: Set; } -class LockOwnershipLostError extends Error { - constructor(cause?: unknown) { - super( - `Another process took over the loopback pool lock as stale (older than ${LOCK_STALE_TIMEOUT_MS}ms) before this process published its update`, - { cause } - ); - this.name = 'LockOwnershipLostError'; - } -} +// Shared by every copy of this module in the process, so release-all keeps meaning "this process". +const CLAIMS_KEY = Symbol.for('@harperfast/integration-testing/loopback-claims'); +const claims: Map = ((globalThis as Record)[CLAIMS_KEY] ??= new Map()) as Map< + string, + Claim +>; -/** - * Acquires a file-based lock by creating the lock file. This enables safe concurrent - * access to the loopback pool across multiple test processes. - * - * Uses the 'wx' file flag which atomically fails if the file already exists, providing - * a simple but effective cross-process mutex. Handles stale locks by removing lock files - * older than LOCK_STALE_TIMEOUT_MS (10 seconds). - */ -async function acquireLock(): Promise { - const token = `${process.pid}-${++lockTokenCounter}-${randomBytes(8).toString('hex')}`; - while (true) { - try { - // The 'wx' flag causes the open to fail if the file already exists - const lockFileHandle = await open(HARPER_LOOPBACK_POOL_LOCK_PATH, 'wx'); - // The write may have outlasted the stale timeout, so the path may no longer be ours to unlink. - try { - await lockFileHandle.writeFile(token); - } finally { - await lockFileHandle.close(); - } - return token; - } catch (error) { - // If the lock file already exists, it's either stale or we wait for it to be released - if ((error as NodeJS.ErrnoException).code === 'EEXIST') { - try { - const lockFileStat = await stat(HARPER_LOOPBACK_POOL_LOCK_PATH); - // If the lock file is older than the timeout, consider it stale and remove it - if (Date.now() - lockFileStat.mtimeMs > LOCK_STALE_TIMEOUT_MS) { - await unlink(HARPER_LOOPBACK_POOL_LOCK_PATH); - } - } catch { - // Lock file may have been removed by another process, continue - } - - await sleep(RETRY_DELAY_MS); - continue; - } - - // Rethrow other errors - throw error; - } - } +function claimKey(address: string): string { + return `${address}:${CLAIM_PORT}`; } -/** The read and unlink are not atomic; a takeover between them can still remove a successor's lock. */ -async function releaseLock(token: string): Promise { - try { - if ((await readFile(HARPER_LOOPBACK_POOL_LOCK_PATH, 'utf-8')) === token) { - await unlink(HARPER_LOOPBACK_POOL_LOCK_PATH); - } - } catch {} +function addressAt(index: number): string { + return `127.0.0.${index + HARPER_LOOPBACK_POOL_START}`; } -async function assertLockHeld(token: string): Promise { - let currentToken: string; - try { - currentToken = await readFile(HARPER_LOOPBACK_POOL_LOCK_PATH, 'utf-8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') throw new LockOwnershipLostError(error); - throw error; - } - if (currentToken !== token) throw new LockOwnershipLostError(); +interface HolderIdentity { + pid: number; + pidNamespace: string | null; + hostname: string | null; } -/** Reruns the callback after lock ownership is lost, up to LOCK_OWNERSHIP_RETRIES times; callers must reread shared state. */ -export async function withLock(callback: (lockToken: string) => Promise): Promise { - for (let attempt = 1; ; attempt++) { - const token = await acquireLock(); - try { - return await callback(token); - } catch (error) { - if (!(error instanceof LockOwnershipLostError) || attempt > LOCK_OWNERSHIP_RETRIES) throw error; - console.warn( - `[loopback-pool] ${error.message}; retrying from the current pool state (retry ${attempt} of ${LOCK_OWNERSHIP_RETRIES}).` - ); - } finally { - await releaseLock(token); +let ownIdentity: HolderIdentity | undefined; +function getOwnIdentity(): HolderIdentity { + if (!ownIdentity) { + let pidNamespace: string | null = null; + if (process.platform === 'linux') { + try { + pidNamespace = readlinkSync('/proc/self/ns/pid'); + } catch {} } + ownIdentity = { pid: process.pid, pidNamespace, hostname: hostname() }; } + return ownIdentity; } -const quarantinedPoolPaths = new Set(); +let identityLine: string | undefined; + +/** The client closes first, so the claim port never sits in TIME_WAIT on the holder's side. */ +function answerIdentity(connections: Set, socket: Socket): void { + connections.add(socket); + socket.on('error', () => {}); + socket.unref(); + const timer = setTimeout(() => socket.resetAndDestroy(), IDENTITY_SERVE_MS).unref(); + socket.once('close', () => { + clearTimeout(timer); + connections.delete(socket); + }); + socket.resume(); + socket.write((identityLine ??= `${JSON.stringify({ protocol: IDENTITY_PROTOCOL, ...getOwnIdentity() })}\n`)); +} + +/** Resolves the claim, or `null` when another listener already holds the address's claim port. */ +function claimAddress(address: string): Promise { + return new Promise((resolve, reject) => { + const connections = new Set(); + const server = createServer((socket) => answerIdentity(connections, socket)); + const onError = (error: NodeJS.ErrnoException) => { + if (error.code === 'EADDRINUSE') { + resolve(null); + return; + } + reject(new LoopbackAddressValidationError(address, error)); + }; + server.once('error', onError); + // `exclusive` keeps a cluster worker from sharing the primary's handle with a sibling worker. + server.listen({ host: address, port: CLAIM_PORT, exclusive: true }, () => { + server.off('error', onError); + // An accept failure (EMFILE) must not crash the holder; the claim stays bound. + server.on('error', () => {}); + server.unref(); + resolve({ server, connections }); + }); + }); +} -function emptyPool(): LoopbackPool { - return Array(HARPER_LOOPBACK_POOL_COUNT).fill(null); +function closeClaim(claim: Claim): Promise { + return new Promise((resolve) => { + claim.server.close(() => resolve()); + for (const socket of claim.connections) socket.resetAndDestroy(); + }); } -/** - * Reads the loopback pool from the pool file. The pool is a JSON array where each - * index represents a loopback address (127.0.0.2, 127.0.0.3, etc.) and the value - * is either null (available) or a process PID (in use). - * - * An unusable file (unparseable, not an array, or an empty array) may still be mid-write by a - * writer that doesn't publish by rename, or may have lost reservations whose holders haven't bound - * their address yet, so it reads as `null` until it has gone UNUSABLE_POOL_QUARANTINE_MS - * unmodified, and only then is replaced with an empty pool. A non-empty array of another length - * is trusted as-is: two processes configured with different counts would otherwise keep wiping - * each other's reservations. - * - * @param poolPath The pool file path (overridable for tests; defaults to the shared pool file) - * @param lockToken The held lock token used if an aged unusable shared pool needs reinitializing - * @returns The loopback pool array, or `null` while the pool file is quarantined - */ -export async function readPoolFile( - poolPath: string = HARPER_LOOPBACK_POOL_PATH, - lockToken?: string -): Promise { - let problem: string; - try { - const parsed: unknown = JSON.parse(await readFile(poolPath, 'utf-8')); - if (Array.isArray(parsed) && parsed.length > 0) { - quarantinedPoolPaths.delete(poolPath); - return parsed as LoopbackPool; - } - problem = Array.isArray(parsed) ? 'empty array' : `JSON ${parsed === null ? 'null' : typeof parsed} instead of an array`; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return emptyPool(); - if (!(error instanceof SyntaxError)) throw error; - problem = error.message; - } +type HolderReport = HolderIdentity | 'unidentified' | 'released'; - let modifiedMs: number; +function parseIdentity(line: string): HolderIdentity | null { + let reply: unknown; try { - modifiedMs = (await stat(poolPath)).mtimeMs; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return emptyPool(); - throw error; - } - const unmodifiedMs = Date.now() - modifiedMs; - if (unmodifiedMs < UNUSABLE_POOL_QUARANTINE_MS) { - if (!quarantinedPoolPaths.has(poolPath)) { - quarantinedPoolPaths.add(poolPath); - console.warn( - `[loopback-pool] ${poolPath} is unusable (${problem}); allocation waits until it has gone ${UNUSABLE_POOL_QUARANTINE_MS / 1000}s unmodified, then reinitializes it. Deleting it skips the wait, but only once every integration-test run on this machine has stopped.` - ); - } + reply = JSON.parse(line); + } catch { return null; } - quarantinedPoolPaths.delete(poolPath); - const pool = emptyPool(); - await writePoolFile(pool, poolPath, lockToken); - console.warn(`[loopback-pool] ${poolPath} has been unusable for ${Math.round(unmodifiedMs / 1000)}s (${problem}); reinitialized.`); - return pool; + if (typeof reply !== 'object' || reply === null) return null; + const { protocol, pid, pidNamespace, hostname } = reply as Record; + if (protocol !== IDENTITY_PROTOCOL || !Number.isSafeInteger(pid) || (pid as number) < 1) return null; + return { + pid: pid as number, + pidNamespace: typeof pidNamespace === 'string' && /^pid:\[\d+\]$/.test(pidNamespace) ? pidNamespace : null, + hostname: typeof hostname === 'string' && /^[\w.-]{1,253}$/.test(hostname) ? hostname : null, + }; } -let pendingWriteCounter = 0; - -/** - * Writes the loopback pool to the pool file as JSON. - * - * Publishes by write-to-temp-then-rename, so readers only ever see the old complete file or - * the new complete file. The pending name must be unpredictable, not just unique: `wx` refuses - * a file already at that name, so a guessable one could be pre-planted to make every write fail. - * The ownership check must stay after the pending file closes and immediately before rename. It narrows, - * but does not close, the window in which a takeover lets a stale holder publish over its successor. - * - * @param pool The loopback pool array to persist - * @param poolPath The pool file path (overridable for tests; defaults to the shared pool file) - * @param lockToken The held lock token required to publish the shared pool - */ -export async function writePoolFile( - pool: LoopbackPool, - poolPath: string = HARPER_LOOPBACK_POOL_PATH, - lockToken?: string -): Promise { - const pendingPath = `${poolPath}.${process.pid}.${++pendingWriteCounter}.${randomBytes(8).toString('hex')}.pending`; - try { - const pendingFileHandle = await open(pendingPath, 'wx'); - try { - await pendingFileHandle.writeFile(JSON.stringify(pool)); - } finally { - await pendingFileHandle.close(); - } - if (poolPath === HARPER_LOOPBACK_POOL_PATH) { - if (!lockToken) throw new Error('A lock token is required to publish the shared loopback pool'); - await assertLockHeld(lockToken); - } - await rename(pendingPath, poolPath); - } catch (error) { - await unlink(pendingPath).catch(() => {}); - throw error; - } +/** Asks a claim's holder who it is. Whatever answers is untrusted, and only ever described. */ +function queryHolder(address: string): Promise { + return new Promise((resolve) => { + const socket = connect({ host: address, port: CLAIM_PORT }); + let reply = ''; + let settled = false; + const settle = (report: HolderReport) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(report); + }; + const timer = setTimeout(() => settle('unidentified'), IDENTITY_QUERY_MS); + socket.setEncoding('utf8'); + socket.on('data', (chunk: string) => { + reply += chunk; + const newline = reply.indexOf('\n'); + if (newline !== -1) settle(parseIdentity(reply.slice(0, newline)) ?? 'unidentified'); + else if (reply.length > IDENTITY_MAX_BYTES) settle('unidentified'); + }); + socket.on('error', (error: NodeJS.ErrnoException) => settle(error.code === 'ECONNREFUSED' ? 'released' : 'unidentified')); + socket.on('end', () => settle('unidentified')); + }); } -/** - * Validates the format of a loopback address and extracts the pool index. - * - * Expects addresses in the format "127.0.0.X" where X is between the pool start - * (127.0.0.2) and the pool start + pool count - 1. The returned index is 0-based - * (e.g., "127.0.0.2" returns index 0). - * - * @param address The loopback address to parse (e.g., "127.0.0.2") - * @returns The 0-based pool index for this address - * @throws {InvalidLoopbackAddressError} If the address format is invalid or out of range - */ -function parseLoopbackAddress(address: string): number { - const parts = address.split('.'); - if (parts.length !== 4 || parts[0] !== '127' || parts[1] !== '0' || parts[2] !== '0') { - throw new InvalidLoopbackAddressError(address); +function describeHolder(identity: HolderIdentity): string { + const own = getOwnIdentity(); + let description = `pid ${identity.pid}`; + if (identity.pidNamespace && own.pidNamespace && identity.pidNamespace !== own.pidNamespace) { + description += ` in another PID namespace (${identity.pidNamespace})`; } - - const index = parseInt(parts[3], 10) - HARPER_LOOPBACK_POOL_START; - if (isNaN(index) || index < 0 || index >= HARPER_LOOPBACK_POOL_COUNT) { - throw new InvalidLoopbackAddressError(address); + if (identity.hostname && own.hostname && identity.hostname !== own.hostname) { + description += ` on host ${identity.hostname}`; } - - return index; + return description; } /** @@ -447,27 +363,41 @@ function isPortInUseOnAddress(port: number, loopbackAddress: string): Promise { + if (process.platform !== 'linux') return false; + try { + return await isPortInUseOnAddress(port, CONTROL_ADDRESS); + } catch { + return false; + } +} + +interface PortConflict { + port: number; + /** `address`: a listener on the pool address itself, such as a Harper node that outlived its runner. */ + scope: 'address' | 'all-interfaces'; +} + +/** + * Finds a port of {@link CONFLICT_PROBE_PORTS} already bound on `loopbackAddress`, preferring one held on the + * address itself. Two shapes of leftover node are caught: one whose main thread still holds the exclusive + * operations port, and one whose main thread exited while SO_REUSEPORT workers still hold the HTTP port. A plain + * bind fails against either, which prevents a co-bind that would split connections between two nodes. * - * Returns the first in-use port number, or `null` if the address is free. * Re-throws any non-EADDRINUSE bind error (a real configuration problem). - * - * @param loopbackAddress The loopback IP address to probe (e.g., "127.0.0.2") */ -async function findConflictingPort(loopbackAddress: string): Promise { +async function findConflictingPort(loopbackAddress: string): Promise { + let allInterfacesConflict: PortConflict | null = null; for (const port of CONFLICT_PROBE_PORTS) { - if (await isPortInUseOnAddress(port, loopbackAddress)) { - return port; - } + if (!(await isPortInUseOnAddress(port, loopbackAddress))) continue; + if (!(await isHeldOnAllInterfaces(port))) return { port, scope: 'address' }; + allInterfacesConflict ??= { port, scope: 'all-interfaces' }; } - return null; + return allInterfacesConflict; } /** @@ -485,9 +415,7 @@ export async function validateLoopbackAddressPool(): Promise<{ failed: { loopbackAddress: string; error: Error }[]; }> { return Promise.allSettled( - Array.from({ length: HARPER_LOOPBACK_POOL_COUNT }, (_, i) => - validateLoopbackAddress(`127.0.0.${i + HARPER_LOOPBACK_POOL_START}`) - ) + Array.from({ length: HARPER_LOOPBACK_POOL_COUNT }, (_, i) => validateLoopbackAddress(addressAt(i))) ).then((results) => results.reduce<{ successful: string[]; failed: { loopbackAddress: string; error: Error }[] }>( (acc, result) => { @@ -505,203 +433,108 @@ export async function validateLoopbackAddressPool(): Promise<{ ); } -/** `conflictPorts` maps a free slot's index to the canary port another Harper node was last found binding there. */ -function describePoolState(pool: LoopbackPool | null, conflictPorts: ReadonlyMap): string { - if (!pool) { - return `The pool file ${HARPER_LOOPBACK_POOL_PATH} is unusable and quarantined (see the earlier [loopback-pool] warning).`; - } - const address = (index: number) => `127.0.0.${index + HARPER_LOOPBACK_POOL_START}`; - const addressesByPid = new Map(); - const occupiedFree: string[] = []; - pool.forEach((pid, index) => { - if (pid !== null) { - addressesByPid.set(pid, [...(addressesByPid.get(pid) ?? []), address(index)]); - } else if (conflictPorts.has(index)) { - occupiedFree.push(`${address(index)} (port ${conflictPorts.get(index)})`); +/** + * @param claimed Indices whose claim port was already bound at the last pass + * @param conflictPorts Maps an unclaimed index to the Harper port another node was last found binding there + */ +async function describePoolState(claimed: readonly number[], conflictPorts: ReadonlyMap): Promise { + const isOwn = (index: number) => claims.has(claimKey(addressAt(index))); + const ownAddresses = claimed.filter(isOwn).map(addressAt); + const others = claimed.filter((index) => !isOwn(index)); + const reports = await Promise.all(others.map((index) => queryHolder(addressAt(index)))); + const addressesByHolder = new Map(); + const unidentified: string[] = []; + others.forEach((index, i) => { + const report = reports[i]; + if (report === 'released') return; + if (report === 'unidentified') { + unidentified.push(addressAt(index)); + return; } + const holder = describeHolder(report); + addressesByHolder.set(holder, [...(addressesByHolder.get(holder) ?? []), addressAt(index)]); }); const lines = [ - `Pool ${address(0)}-${address(pool.length - 1)} (${pool.length} addresses) in ${HARPER_LOOPBACK_POOL_PATH}:`, - ...[...addressesByPid].map( - ([pid, addresses]) => ` pid ${pid}${pid === process.pid ? ' (this process)' : ''} holds ${addresses.join(', ')}` - ), + `Pool ${addressAt(0)}-${addressAt(HARPER_LOOPBACK_POOL_COUNT - 1)} (${HARPER_LOOPBACK_POOL_COUNT} addresses), each held by a listener on its port ${CLAIM_PORT}:`, ]; - if (occupiedFree.length > 0) { - lines.push(` unclaimed but bound by another Harper node: ${occupiedFree.join(', ')}`); + if (ownAddresses.length > 0) lines.push(` pid ${process.pid} (this process) holds ${ownAddresses.join(', ')}`); + for (const [holder, addresses] of addressesByHolder) lines.push(` ${holder} holds ${addresses.join(', ')}`); + if (unidentified.length > 0) { + lines.push(` held by a process that did not identify itself: ${unidentified.join(', ')}`); + if (await isHeldOnAllInterfaces(CLAIM_PORT)) { + lines.push( + ` port ${CLAIM_PORT} is bound on all interfaces by another program, which holds every address; set HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT to a free port for every run` + ); + } } + const occupied = [...conflictPorts].map(([index, port]) => `${addressAt(index)} (port ${port})`); + if (occupied.length > 0) lines.push(` unclaimed but bound by another Harper node: ${occupied.join(', ')}`); lines.push( - 'Each pid is alive; `ps -p ` shows what it is, and `lsof -nP -iTCP: -sTCP:LISTEN` what holds a port. Stop what leaked.' + `Each holder is alive: its listener closes when it exits. \`lsof -nP -iTCP@
:${CLAIM_PORT} -sTCP:LISTEN\` names a holder by its pid on this host, and \`lsof -nP -iTCP@
-sTCP:LISTEN\` what binds an address's Harper ports (\`netstat -ano | findstr
:\` on Windows). Stop what leaked.` ); return lines.join('\n'); } /** - * Retrieves the next available loopback address from the pool using a file-based - * locking mechanism to safely allocate addresses across concurrent test processes. + * Retrieves the next available loopback address from the pool. Safe across concurrent processes, + * whatever their PID namespace or TMPDIR, as long as they share a network namespace. * * **How it works:** - * 1. Acquires a file-based lock to prevent race conditions with other processes - * 2. Reads the pool state (a JSON array of process IDs, with null for available slots) - * 3. Finds the first available (null) slot and assigns the current process PID to it - * 4. Writes the updated pool back to disk and releases the lock - * 5. Validates that the allocated address can actually be bound to - * 6. Checks that no other process accepts connections on the address's canary ports - * 7. Returns the loopback address (e.g., "127.0.0.2") - * - * If no addresses are available, or the pool file is quarantined (see readPoolFile), waits and - * retries until one becomes available, for up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` - * (default 10 minutes). A wait of 30 seconds or more logs one warning naming the holders. + * 1. Claims the first address whose claim port (`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT`, default + * 9924) no other process listens on, by listening there itself until the address is released or + * this process exits + * 2. Skips an address whose Harper ports (operations, HTTP, HTTPS, MQTT, MQTTS) are already bound + * on it, such as by a Harper node that outlived its runner, and tries the next one + * 3. Checks that no other process accepts connections on the address's operations and HTTP ports + * 4. Returns the loopback address (e.g., "127.0.0.2") * - * **Pool file location:** `${tmpdir()}/harper-integration-test-loopback-pool.json` - * **Lock file location:** `${tmpdir()}/harper-integration-test-loopback-pool.lock` + * If every address is held, waits and retries until one becomes available, for up to + * `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default 10 minutes). A wait of 30 seconds or + * more logs one warning naming the holders. * * @returns A promise that resolves with an allocated loopback address - * @throws {LoopbackAddressValidationError} If the allocated address cannot be bound to, or cannot be - * checked for another process's listener (the slot is released in that case) - * @throws {ForeignListenerError} If another process accepts connections on the address's canary - * ports and `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS` is not set (the slot is released) - * @throws {LockOwnershipLostError} If ownership is lost on the initial attempt and all three retries + * @throws {LoopbackAddressValidationError} If the address cannot be bound to, or cannot be checked for + * another process's listener + * @throws {ForeignListenerError} If another process accepts connections on the address's Harper ports and + * `HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS` is not set * @throws {LoopbackPoolExhaustedError} If no address became free within the timeout */ export async function getNextAvailableLoopbackAddress(): Promise { - // Each index maps to a different loopback address (index 0 -> 127.0.0.2, index 1 -> 127.0.0.3, etc.) - // So if the first test process number is 42, it would be assigned to index 0 associated with address 127.0.0.2 - // [42, null, null, ...]; - // Then the next process (call is 43) gets the next available, so index 1 -> 127.0.0.3 - // [42, 43, null, ...]; - // And so on... - // As processes exit and release their loopback addresses, those addresses become available for new processes to use - // [42, null, 44, ...]; - // Next process (45) gets index 1 again -> - // [42, 45, 44, ...]; - // This continues until all loopback addresses are used, at which point new processes will wait until an address becomes available - - // Since multiple processes may be trying to get a loopback address at the same time, we need to implement a simple file-based locking mechanism to prevent race conditions - - // Indices found in-use by the conflict canary during THIS call. Tracked locally (not by - // parking them in the shared pool) so we never deadlock ourselves: if we parked every - // poisoned slot under our own live PID, the pool could fill with our PID, the free-slot search - // would find nothing, removeDeadProcessesFromPool couldn't reclaim them (we're alive), and we'd - // spin forever. Instead we release poisoned slots back to the pool and just avoid re-trying - // them within this attempt; the set is cleared before each wait so they can be re-probed once - // the lingering node has had time to exit. - const triedIndices = new Set(); const waitStartedAt = performance.now(); + const warnedConflicts = new Set(); let noticed = false; - let lastPool: LoopbackPool | null = null; - const conflictPorts = new Map(); while (true) { - const assignedIndex = await withLock(async (lockToken) => { - // Read the pool file - const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); - lastPool = loopbackPool; - if (!loopbackPool) return null; - - // Find the first available index we haven't already found in-use this attempt - const findAvailableIndex = (): number | null => { - for (let i = 0; i < loopbackPool.length; i++) { - if (loopbackPool[i] === null && !triedIndices.has(i)) return i; - } - return null; - }; - let index = findAvailableIndex(); - if (index === null) { - // Nothing available - remove any dead processes from the pool and claim a slot they held, else wait for one to become available - removeDeadProcessesFromPool(loopbackPool); - index = findAvailableIndex(); - } - if (index !== null) { - // Assign the process PID to that index to mark it as used - loopbackPool[index] = process.pid; - } - // Write the updated pool back to the file - await writePoolFile(loopbackPool, HARPER_LOOPBACK_POOL_PATH, lockToken); - - return index; - }); - - // If we got an index, validate and return the address - if (assignedIndex !== null) { - const loopbackAddress = `127.0.0.${assignedIndex + HARPER_LOOPBACK_POOL_START}`; - try { - await validateLoopbackAddress(loopbackAddress); - } catch (error) { - // Validation failed - throw a proper error instead of breaking - throw new LoopbackAddressValidationError(loopbackAddress, error as Error); + const claimed: number[] = []; + const conflictPorts = new Map(); + for (let index = 0; index < HARPER_LOOPBACK_POOL_COUNT; index++) { + const loopbackAddress = addressAt(index); + const claim = await claimAddress(loopbackAddress); + if (!claim) { + claimed.push(index); + continue; } - - // Conflict canary: a stale or still-running Harper node from an overlapping - // suite may already hold this address (e.g. the pool slot was freed while its - // node lingered — including the common shape where the main thread exited but the - // HTTP workers still hold the address). SO_REUSEPORT on Harper's shared HTTP/ - // replication ports would otherwise let our node silently co-bind it, splitting - // connections between two nodes and corrupting both suites. Detect that here by - // probing both the exclusive operations port and the worker-held HTTP port, and - // skip the poisoned address. - let conflictingPort: number | null; + let accepted: boolean; try { - conflictingPort = await findConflictingPort(loopbackAddress); + accepted = await checkClaimedAddress(loopbackAddress, index, conflictPorts, warnedConflicts); } catch (error) { - throw new LoopbackAddressValidationError(loopbackAddress, error as Error); + await closeClaim(claim); + throw error; } - if (conflictingPort === null) { - // macOS and Windows let the canary's exclusive bind succeed beside a listener on all interfaces; Linux refuses it. - if (process.platform === 'linux') { - return loopbackAddress; - } - let shadowedPort: number | null; - try { - shadowedPort = await findAcceptingPort(loopbackAddress, CONFLICT_PROBE_PORTS); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - if (foreignListenersAllowed()) { - console.warn( - `[loopback-pool] Could not check ${loopbackAddress} for another process's listener (${reason}); continuing because ${ALLOW_FOREIGN_LISTENERS_ENV} is set.` - ); - return loopbackAddress; - } - await releaseRefusedAddress(loopbackAddress); - throw new LoopbackAddressValidationError( - loopbackAddress, - error as Error, - `Could not check whether another process accepts connections on ${loopbackAddress}'s Harper ports (${reason}). Set ${ALLOW_FOREIGN_LISTENERS_ENV}=1 to skip this check.` - ); - } - if (shadowedPort === null) { - return loopbackAddress; - } - if (foreignListenersAllowed()) { - console.warn( - `[loopback-pool] Another process accepts connections on ${loopbackAddress}:${shadowedPort}; continuing because ${ALLOW_FOREIGN_LISTENERS_ENV} is set. This node's clients reach that process whenever the node has no listener of its own there.` - ); - return loopbackAddress; - } - await releaseRefusedAddress(loopbackAddress); - throw new ForeignListenerError(loopbackAddress, shadowedPort); + if (accepted) { + claims.set(claimKey(loopbackAddress), claim); + return loopbackAddress; } - - // Release the slot back to the pool (so it isn't leaked under our PID) and remember - // not to re-try it this attempt; loop to claim the next available one. - await releaseLoopbackAddress(loopbackAddress); - triedIndices.add(assignedIndex); - conflictPorts.set(assignedIndex, conflictingPort); - console.warn( - `[loopback-pool] ${loopbackAddress} is still in use by another Harper node (port ${conflictingPort} bound); skipping to avoid an SO_REUSEPORT co-bind.` - ); - continue; + await closeClaim(claim); } - // No available addresses; clear the per-attempt skip set so poisoned addresses can be - // re-probed (their lingering node may have exited during the wait), then wait and retry. - triedIndices.clear(); const waitedMs = performance.now() - waitStartedAt; if (waitedMs >= ACQUIRE_TIMEOUT_MS) { - throw new LoopbackPoolExhaustedError(waitedMs, describePoolState(lastPool, conflictPorts)); + throw new LoopbackPoolExhaustedError(waitedMs, await describePoolState(claimed, conflictPorts)); } if (!noticed && waitedMs >= WAIT_NOTICE_MS) { noticed = true; console.warn( - `[loopback-pool] Still waiting for a free loopback address after ${Math.round(waitedMs / 1000)}s; giving up after ${Math.round(ACQUIRE_TIMEOUT_MS / 1000)}s. ${describePoolState(lastPool, conflictPorts)}` + `[loopback-pool] Still waiting for a free loopback address after ${Math.round(waitedMs / 1000)}s; giving up after ${Math.round(ACQUIRE_TIMEOUT_MS / 1000)}s. ${await describePoolState(claimed, conflictPorts)}` ); } await sleep(RETRY_DELAY_MS); @@ -709,68 +542,109 @@ export async function getNextAvailableLoopbackAddress(): Promise { } /** - * Removes any dead processes from the loopback pool. - * @param loopbackPool + * Decides whether a just-claimed address can be handed out. Resolves `false` to skip it for this pass. */ -function removeDeadProcessesFromPool(loopbackPool: LoopbackPool) { - loopbackPool.forEach((pid, index) => { - if (pid === null) return; - try { - process.kill(pid, 0); - } catch { - loopbackPool[index] = null; +async function checkClaimedAddress( + loopbackAddress: string, + index: number, + conflictPorts: Map, + warnedConflicts: Set +): Promise { + let conflict: PortConflict | null; + try { + conflict = await findConflictingPort(loopbackAddress); + } catch (error) { + throw new LoopbackAddressValidationError(loopbackAddress, error as Error); + } + if (conflict?.scope === 'address') { + conflictPorts.set(index, conflict.port); + const warning = `${loopbackAddress}:${conflict.port}`; + if (!warnedConflicts.has(warning)) { + warnedConflicts.add(warning); + console.warn( + `[loopback-pool] ${loopbackAddress} is still in use by another Harper node (port ${conflict.port} bound); skipping to avoid an SO_REUSEPORT co-bind.` + ); } - }); + return false; + } + if (conflict) { + if (!foreignListenersAllowed()) throw new ForeignListenerError(loopbackAddress, conflict.port); + console.warn( + `[loopback-pool] Another process listens on ${loopbackAddress}:${conflict.port} through a listener on all interfaces; continuing because ${ALLOW_FOREIGN_LISTENERS_ENV} is set. This node's clients reach that process whenever the node has no listener of its own there, and a Harper node still bound to ${loopbackAddress} itself cannot be ruled out.` + ); + return true; + } + // macOS and Windows let the exclusive bind above succeed beside a listener on all interfaces; Linux refuses it. + if (process.platform === 'linux') return true; + + let shadowedPort: number | null; + try { + shadowedPort = await findAcceptingPort(loopbackAddress, FOREIGN_LISTENER_PROBE_PORTS); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + if (foreignListenersAllowed()) { + console.warn( + `[loopback-pool] Could not check ${loopbackAddress} for another process's listener (${reason}); continuing because ${ALLOW_FOREIGN_LISTENERS_ENV} is set.` + ); + return true; + } + throw new LoopbackAddressValidationError( + loopbackAddress, + error as Error, + `Could not check whether another process accepts connections on ${loopbackAddress}'s Harper ports (${reason}). Set ${ALLOW_FOREIGN_LISTENERS_ENV}=1 to skip this check.` + ); + } + if (shadowedPort === null) return true; + if (!foreignListenersAllowed()) throw new ForeignListenerError(loopbackAddress, shadowedPort); + console.warn( + `[loopback-pool] Another process accepts connections on ${loopbackAddress}:${shadowedPort}; continuing because ${ALLOW_FOREIGN_LISTENERS_ENV} is set. This node's clients reach that process whenever the node has no listener of its own there.` + ); + return true; } /** - * Releases a loopback address back to the pool, making it available for other processes. + * Validates the format of a loopback address and extracts the pool index. * - * Only a slot this process holds is cleared: once the pool has been reinitialized, the same slot - * may already be another runner's reservation. Nothing is written while the pool file is - * quarantined (see readPoolFile): the reservation is already gone from it, and rewriting it would - * end the quarantine early. + * @param address The loopback address to parse (e.g., "127.0.0.2") + * @returns The 0-based pool index for this address + * @throws {InvalidLoopbackAddressError} If the address format is invalid or out of range + */ +function parseLoopbackAddress(address: string): number { + const parts = address.split('.'); + if (parts.length !== 4 || parts[0] !== '127' || parts[1] !== '0' || parts[2] !== '0') { + throw new InvalidLoopbackAddressError(address); + } + + const index = parseInt(parts[3], 10) - HARPER_LOOPBACK_POOL_START; + if (isNaN(index) || index < 0 || index >= HARPER_LOOPBACK_POOL_COUNT) { + throw new InvalidLoopbackAddressError(address); + } + + return index; +} + +/** + * Releases a loopback address back to the pool, making it available for other processes. Only an + * address this process holds is released. * * @param address The loopback address to release (e.g., "127.0.0.2") * @throws InvalidLoopbackAddressError if the address format is invalid */ export async function releaseLoopbackAddress(address: string): Promise { - // Validate and parse the address - const index = parseLoopbackAddress(address); - - await withLock(async (lockToken) => { - // Read the pool file - const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); - if (!loopbackPool || loopbackPool[index] !== process.pid) return; - - // Release the address by setting it to null - loopbackPool[index] = null; - - // Write the updated pool back to the file - await writePoolFile(loopbackPool, HARPER_LOOPBACK_POOL_PATH, lockToken); - }); + parseLoopbackAddress(address); + const key = claimKey(address); + const claim = claims.get(key); + if (!claim) return; + claims.delete(key); + await closeClaim(claim); } /** - * Releases all loopback addresses assigned to the current process. + * Releases all loopback addresses held by the current process. * Useful for cleanup during graceful shutdown. */ export async function releaseAllLoopbackAddressesForCurrentProcess(): Promise { - await withLock(async (lockToken) => { - // Read the pool file - const loopbackPool = await readPoolFile(HARPER_LOOPBACK_POOL_PATH, lockToken); - if (!loopbackPool) return; - - // Find and release all addresses assigned to this process - let released = false; - for (let i = 0; i < loopbackPool.length; i++) { - if (loopbackPool[i] === process.pid) { - loopbackPool[i] = null; - released = true; - } - } - - // Write the updated pool back to the file - if (released) await writePoolFile(loopbackPool, HARPER_LOOPBACK_POOL_PATH, lockToken); - }); + const held = [...claims.values()]; + claims.clear(); + await Promise.all(held.map(closeClaim)); } diff --git a/test/fixtures/loopbackHolder.ts b/test/fixtures/loopbackHolder.ts new file mode 100644 index 0000000..ee98ce9 --- /dev/null +++ b/test/fixtures/loopbackHolder.ts @@ -0,0 +1,14 @@ +// A pool client in its own process. Prints `{"ready":true}` once the pool is loaded, waits for a line on stdin when +// run with --wait-for-go, then prints the allocation's outcome as one JSON line and holds it until stdin ends. +import { createInterface } from 'node:readline'; +import { getNextAvailableLoopbackAddress } from '../../src/loopbackAddressPool.ts'; + +const lines = createInterface({ input: process.stdin }); +lines.once('close', () => process.exit(0)); +console.log(JSON.stringify({ ready: true })); +if (process.argv.includes('--wait-for-go')) await new Promise((resolve) => lines.once('line', resolve)); +try { + console.log(JSON.stringify({ address: await getNextAvailableLoopbackAddress(), pid: process.pid })); +} catch (error) { + console.log(JSON.stringify({ error: (error as Error).name, pid: process.pid })); +} diff --git a/test/loopback-lock-fencing.test.ts b/test/loopback-lock-fencing.test.ts deleted file mode 100644 index 305278e..0000000 --- a/test/loopback-lock-fencing.test.ts +++ /dev/null @@ -1,383 +0,0 @@ -import test, { mock } from 'node:test'; -import { deepStrictEqual, notStrictEqual, rejects, strictEqual } from 'node:assert'; -import { EventEmitter } from 'node:events'; -import * as fsPromises from 'node:fs/promises'; -import { mkdtemp, readFile, readdir, rm, unlink, utimes, writeFile } from 'node:fs/promises'; -import * as net from 'node:net'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; - -let freshImportCounter = 0; - -function freshModuleUrl(): string { - return `../src/loopbackAddressPool.ts?lock-fencing=${++freshImportCounter}`; -} - -function deferred(): { promise: Promise; resolve: () => void } { - let resolve!: () => void; - const promise = new Promise((res) => { - resolve = res; - }); - return { promise, resolve }; -} - -function restoreEnv(name: string, value: string | undefined): void { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; -} - -function createAvailableServer() { - const emitter = new EventEmitter(); - const server = { - once(event: string, listener: (...args: unknown[]) => void) { - emitter.once(event, listener); - return server; - }, - off(event: string, listener: (...args: unknown[]) => void) { - emitter.off(event, listener); - return server; - }, - listen(...args: unknown[]) { - const listener = args.at(-1) as () => void; - queueMicrotask(listener); - return server; - }, - close(listener: () => void) { - queueMicrotask(listener); - return server; - }, - }; - return server; -} - -function createRefusedSocket(): net.Socket { - const socket = new EventEmitter() as unknown as net.Socket; - socket.setTimeout = () => socket; - socket.destroy = () => socket; - queueMicrotask(() => socket.emit('error', Object.assign(new Error('Connection refused'), { code: 'ECONNREFUSED' }))); - return socket; -} - -function mockTakeoverAfterPendingClose(poolPath: string, lockPath: string, successorPool: (number | null)[]): { - fsMock: ReturnType; - getRenameCount: () => number; -} { - let tookOver = false; - let renameCount = 0; - const fsMock = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - open: async (path: string, flags: string) => { - const handle = await fsPromises.open(path, flags); - if (!path.startsWith(`${poolPath}.`) || !path.endsWith('.pending')) return handle; - return { - writeFile: (data: string) => handle.writeFile(data), - close: async () => { - await handle.close(); - if (tookOver) return; - tookOver = true; - await writeFile(poolPath, JSON.stringify(successorPool)); - await unlink(lockPath); - await writeFile(lockPath, 'successor-holder', { flag: 'wx' }); - const staleTime = new Date(Date.now() - 20000); - await utimes(lockPath, staleTime, staleTime); - }, - }; - }, - rename: async (from: string, to: string) => { - renameCount++; - await fsPromises.rename(from, to); - }, - }, - }); - return { fsMock, getRenameCount: () => renameCount }; -} - -test('a writer superseded after reading cannot erase the successor claim', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-fencing-')); - const poolPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.json'); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousEnv = new Map( - [ - 'TMPDIR', - 'TMP', - 'TEMP', - 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START', - 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT', - ].map((name) => [name, process.env[name]]) - ); - process.env.TMPDIR = isolatedTmpDir; - process.env.TMP = isolatedTmpDir; - process.env.TEMP = isolatedTmpDir; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '2'; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '3'; - - const staleSnapshotRead = deferred(); - const resumeStaleWriter = deferred(); - let pauseFirstPoolRead = true; - const fsMock = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - readFile: async (path: string, encoding: BufferEncoding) => { - const contents = await readFile(path, encoding); - if (path === poolPath && pauseFirstPoolRead) { - pauseFirstPoolRead = false; - staleSnapshotRead.resolve(); - await resumeStaleWriter.promise; - } - return contents; - }, - }, - }); - const netMock = mock.module('node:net', { - namedExports: { - ...net, - createServer: createAvailableServer, - connect: createRefusedSocket, - }, - }); - const warn = mock.method(console, 'warn', () => {}); - - let staleWriter: Promise | undefined; - try { - await writeFile(poolPath, JSON.stringify([null, null, null])); - const { getNextAvailableLoopbackAddress } = await import(freshModuleUrl()); - staleWriter = getNextAvailableLoopbackAddress(); - await staleSnapshotRead.promise; - - await writeFile(poolPath, JSON.stringify([process.ppid, null, null])); - const staleTime = new Date(Date.now() - 20000); - await utimes(lockPath, staleTime, staleTime); - - const successorAddress = await getNextAvailableLoopbackAddress(); - strictEqual(successorAddress, '127.0.0.3'); - await writeFile(lockPath, 'replacement-holder', { flag: 'wx' }); - await utimes(lockPath, staleTime, staleTime); - resumeStaleWriter.resolve(); - - const staleWriterAddress = await staleWriter; - notStrictEqual(staleWriterAddress, successorAddress); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), [process.ppid, process.pid, process.pid]); - strictEqual(warn.mock.callCount(), 1); - } finally { - resumeStaleWriter.resolve(); - await staleWriter?.catch(() => {}); - fsMock.restore(); - netMock.restore(); - warn.mock.restore(); - for (const [name, value] of previousEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('a takeover during pending preparation prevents the stale rename and retries from the successor pool', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-pending-takeover-')); - const poolPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.json'); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousEnv = new Map( - [ - 'TMPDIR', - 'TMP', - 'TEMP', - 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START', - 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT', - ].map((name) => [name, process.env[name]]) - ); - process.env.TMPDIR = isolatedTmpDir; - process.env.TMP = isolatedTmpDir; - process.env.TEMP = isolatedTmpDir; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '2'; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '3'; - const successorPool = [process.ppid, null, null]; - const takeover = mockTakeoverAfterPendingClose(poolPath, lockPath, successorPool); - const netMock = mock.module('node:net', { - namedExports: { - ...net, - createServer: createAvailableServer, - connect: createRefusedSocket, - }, - }); - const warn = mock.method(console, 'warn', () => {}); - try { - await writeFile(poolPath, JSON.stringify([null, null, null])); - const { getNextAvailableLoopbackAddress } = await import(freshModuleUrl()); - - strictEqual(await getNextAvailableLoopbackAddress(), '127.0.0.3'); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), [process.ppid, process.pid, null]); - strictEqual(takeover.getRenameCount(), 1, 'only the retry should rename a pending file'); - strictEqual(warn.mock.callCount(), 1); - deepStrictEqual(await readdir(isolatedTmpDir), ['harper-integration-test-loopback-pool.json']); - } finally { - takeover.fsMock.restore(); - netMock.restore(); - warn.mock.restore(); - for (const [name, value] of previousEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('read-time repair passes its lock token through pending preparation', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-repair-takeover-')); - const poolPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.json'); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousTmpEnv = new Map( - ['TMPDIR', 'TMP', 'TEMP', 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START', 'HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT'] - .map((name) => [name, process.env[name]]) - ); - process.env.TMPDIR = isolatedTmpDir; - process.env.TMP = isolatedTmpDir; - process.env.TEMP = isolatedTmpDir; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '2'; - process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '3'; - const successorPool = [process.ppid, null, null]; - const takeover = mockTakeoverAfterPendingClose(poolPath, lockPath, successorPool); - const netMock = mock.module('node:net', { - namedExports: { - ...net, - createServer: createAvailableServer, - connect: createRefusedSocket, - }, - }); - const warn = mock.method(console, 'warn', () => {}); - try { - await writeFile(poolPath, '['); - const staleTime = new Date(Date.now() - 10 * 60 * 1000); - await utimes(poolPath, staleTime, staleTime); - const { getNextAvailableLoopbackAddress } = await import(freshModuleUrl()); - - strictEqual(await getNextAvailableLoopbackAddress(), '127.0.0.3'); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), [process.ppid, process.pid, null]); - strictEqual(takeover.getRenameCount(), 1, 'only allocation after repair retry should rename a pending file'); - strictEqual(warn.mock.callCount(), 1, 'only the lost-ownership retry should warn before succeeding'); - deepStrictEqual(await readdir(isolatedTmpDir), ['harper-integration-test-loopback-pool.json']); - } finally { - takeover.fsMock.restore(); - netMock.restore(); - warn.mock.restore(); - for (const [name, value] of previousTmpEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('a superseded holder does not unlink the replacement lock', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-release-')); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousTmpEnv = new Map(['TMPDIR', 'TMP', 'TEMP'].map((name) => [name, process.env[name]])); - for (const name of previousTmpEnv.keys()) process.env[name] = isolatedTmpDir; - const sectionEntered = deferred(); - const resumeSection = deferred(); - try { - const { withLock } = await import(freshModuleUrl()); - const staleHolder = withLock(async () => { - sectionEntered.resolve(); - await resumeSection.promise; - }); - await sectionEntered.promise; - await unlink(lockPath); - await writeFile(lockPath, 'replacement-holder', { flag: 'wx' }); - resumeSection.resolve(); - await staleHolder; - - strictEqual(await readFile(lockPath, 'utf-8'), 'replacement-holder'); - } finally { - resumeSection.resolve(); - for (const [name, value] of previousTmpEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('a failed token write does not delete a successor lock', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-write-failure-')); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousTmpEnv = new Map(['TMPDIR', 'TMP', 'TEMP'].map((name) => [name, process.env[name]])); - for (const name of previousTmpEnv.keys()) process.env[name] = isolatedTmpDir; - const fsMock = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - open: async (path: string, flags: string) => { - if (path !== lockPath) return fsPromises.open(path, flags); - const handle = await fsPromises.open(path, flags); - return { - close: () => handle.close(), - writeFile: async () => { - await unlink(lockPath); - await writeFile(lockPath, 'replacement-holder', { flag: 'wx' }); - throw new Error('simulated token write failure'); - }, - }; - }, - }, - }); - try { - const { withLock } = await import(freshModuleUrl()); - await rejects(withLock(async () => {}), /simulated token write failure/); - strictEqual(await readFile(lockPath, 'utf-8'), 'replacement-holder'); - } finally { - fsMock.restore(); - for (const [name, value] of previousTmpEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('a failed token write leaves its empty lock for stale reclamation', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-empty-write-failure-')); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousTmpEnv = new Map(['TMPDIR', 'TMP', 'TEMP'].map((name) => [name, process.env[name]])); - for (const name of previousTmpEnv.keys()) process.env[name] = isolatedTmpDir; - const fsMock = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - open: async (path: string, flags: string) => { - if (path !== lockPath) return fsPromises.open(path, flags); - const handle = await fsPromises.open(path, flags); - return { - close: () => handle.close(), - writeFile: async () => { - throw new Error('simulated ENOSPC'); - }, - }; - }, - }, - }); - try { - const { withLock } = await import(freshModuleUrl()); - await rejects(withLock(async () => {}), /simulated ENOSPC/); - strictEqual(await readFile(lockPath, 'utf-8'), ''); - } finally { - fsMock.restore(); - for (const [name, value] of previousTmpEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); - -test('a holder that keeps losing the lock gives up instead of retrying forever', async () => { - const isolatedTmpDir = await mkdtemp(join(tmpdir(), 'loopback-lock-persistent-loss-')); - const poolPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.json'); - const lockPath = join(isolatedTmpDir, 'harper-integration-test-loopback-pool.lock'); - const previousTmpEnv = new Map(['TMPDIR', 'TMP', 'TEMP'].map((name) => [name, process.env[name]])); - for (const name of previousTmpEnv.keys()) process.env[name] = isolatedTmpDir; - const fsMock = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - readFile: async (path: string, encoding: BufferEncoding) => { - const contents = await readFile(path, encoding); - if (path === poolPath) await unlink(lockPath); - return contents; - }, - }, - }); - const warn = mock.method(console, 'warn', () => {}); - try { - const pool = JSON.stringify([process.pid, null, null]); - await writeFile(poolPath, pool); - const { releaseAllLoopbackAddressesForCurrentProcess } = await import(freshModuleUrl()); - await rejects(releaseAllLoopbackAddressesForCurrentProcess(), { name: 'LockOwnershipLostError' }); - strictEqual(warn.mock.callCount(), 3); - strictEqual(await readFile(poolPath, 'utf-8'), pool); - } finally { - fsMock.restore(); - warn.mock.restore(); - for (const [name, value] of previousTmpEnv) restoreEnv(name, value); - await rm(isolatedTmpDir, { recursive: true, force: true }); - } -}); diff --git a/test/loopbackAddressPool.test.ts b/test/loopbackAddressPool.test.ts index 0c3ada6..f1a4c61 100644 --- a/test/loopbackAddressPool.test.ts +++ b/test/loopbackAddressPool.test.ts @@ -1,9 +1,7 @@ -import { test, after, mock } from 'node:test'; +import { test, after } from 'node:test'; import { deepStrictEqual, match, ok, rejects, strictEqual } from 'node:assert'; -import { createServer, type AddressInfo, type Server } from 'node:net'; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { mkdtemp, readdir, readFile, rm, utimes, writeFile } from 'node:fs/promises'; -import * as fsPromises from 'node:fs/promises'; +import { connect, createServer, type AddressInfo, type Server } from 'node:net'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; @@ -25,49 +23,45 @@ function close(server: Server): Promise { return new Promise((resolve) => server.close(() => resolve())); } -/** Two distinct free ports: both are held at once, so the OS cannot hand out the same one twice. */ -async function twoFreePorts(): Promise<[number, number]> { - const servers = await Promise.all([listen(HOST, 0), listen(HOST, 0)]); +/** Distinct free ports: all are held at once, so the OS cannot hand out the same one twice. */ +async function freePorts(count: number): Promise { + const servers = await Promise.all(Array.from({ length: count }, () => listen(HOST, 0))); const ports = servers.map((server) => (server.address() as AddressInfo).port); await Promise.all(servers.map(close)); - return [ports[0], ports[1]]; + return ports; } -// The pool reads these once, when it is imported. -const poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-test-')); -const [operationsPort, httpPort] = await twoFreePorts(); -// The pool keeps its state in os.tmpdir(), which prefers TMPDIR on POSIX and TEMP on Windows. -process.env.TMPDIR = process.env.TEMP = poolDir; +// The pool reads these once, when it is imported. Every port is ephemeral, so neither a real Harper nor a +// concurrent run on this machine can make the result depend on the host. +const scratchDir = mkdtempSync(join(tmpdir(), 'loopback-pool-test-')); +const [claimPort, operationsPort, httpPort, mqttPort] = await freePorts(4); +process.env.TMPDIR = process.env.TEMP = scratchDir; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '1'; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(operationsPort); process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT = String(httpPort); +process.env.HARPER_INTEGRATION_TEST_OTHER_CONFLICT_PROBE_PORTS = String(mqttPort); delete process.env[ALLOW_ENV]; -const { getNextAvailableLoopbackAddress, releaseLoopbackAddress, readPoolFile, writePoolFile } = await import('../src/loopbackAddressPool.ts'); +const { getNextAvailableLoopbackAddress, releaseLoopbackAddress, releaseAllLoopbackAddressesForCurrentProcess } = + await import('../src/loopbackAddressPool.ts'); const { startHarper, setupHarperWithFixture, publishHarperNode, createHarperContext } = await import( '../src/harperLifecycle.ts' ); -after(() => rmSync(poolDir, { recursive: true, force: true })); +after(() => rmSync(scratchDir, { recursive: true, force: true })); -function readPool(): unknown { - return JSON.parse(readFileSync(join(poolDir, 'harper-integration-test-loopback-pool.json'), 'utf-8')); +async function claimIsFree(): Promise { + return isPortFree(HOST, claimPort); } -const CANARY_CATCHES_IT = - 'an exclusive bind of the pool address conflicts with a listener on all interfaces here, so the conflict canary refuses the address before this check runs'; - -/** Where an exclusive bind of the pool address conflicts with it (Linux), resolves a reason to skip instead. */ +/** Resolves a reason to skip where this host cannot listen on all interfaces. */ async function listenOnAllInterfaces(host: string, port: number): Promise { - let server: Server; try { - server = await listen(host, port); + return await listen(host, port); } catch (error) { return `cannot listen on ${host}: ${(error as NodeJS.ErrnoException).code}`; } - if (await isPortFree(HOST, port)) return server; - await close(server); - return CANARY_CATCHES_IT; } /** The allocation's rejection; an address handed out instead goes back first, so the one-slot pool cannot wedge. */ @@ -82,61 +76,53 @@ async function refusal(): Promise { throw new Error(`expected getNextAvailableLoopbackAddress to reject, but it handed out ${address}`); } -let freshImportCounter = 0; -/** Monotonic cache-busting suffix — millisecond-resolution timestamps can collide. */ -function freshModuleUrl(): string { - return `../src/loopbackAddressPool.ts?fresh=${++freshImportCounter}`; -} +test('hands out an address and holds its claim port until released', async () => { + const address = await getNextAvailableLoopbackAddress(); + strictEqual(address, HOST); + ok(!(await claimIsFree()), 'the claim port is held while the address is'); + await releaseLoopbackAddress(address); + ok(await claimIsFree(), 'releasing frees the claim port'); + await releaseLoopbackAddress(address); +}); -async function withTempDir(body: (dir: string) => Promise): Promise { - const dir = await mkdtemp(join(tmpdir(), 'loopback-pool-test-')); +test('releasing an address another process holds leaves its claim alone', async () => { + const otherHolder = await listen(HOST, claimPort); try { - await body(dir); + await releaseLoopbackAddress(HOST); + ok(otherHolder.listening); + ok(!(await claimIsFree())); } finally { - await rm(dir, { recursive: true, force: true }); + await close(otherHolder); } -} - -const POOL_FILE_NAME = 'harper-integration-test-loopback-pool.json'; +}); -async function ageBeyondQuarantine(path: string): Promise { - const anHourAgo = new Date(Date.now() - 3600000); - await utimes(path, anHourAgo, anHourAgo); -} +test('a release that overlaps the next allocation of the same address does not release the new claim', async () => { + const first = await getNextAvailableLoopbackAddress(); + const releasing = releaseLoopbackAddress(first); + const second = await getNextAvailableLoopbackAddress(); + await releasing; + strictEqual(second, HOST); + ok(!(await claimIsFree()), 'the second claim is still held'); + await releaseLoopbackAddress(second); + ok(await claimIsFree()); +}); -/** - * A fresh copy of the pool module whose pool and lock live in `dir` and which hands out only - * 127.0.0.1. Both canary ports are chosen ephemerally, so a real Harper on this machine can't - * make the result depend on the host. - */ -async function importIsolatedPool(dir: string): Promise { - const [operationsPort, httpPort] = await twoFreePorts(); - const overrides: Record = { - TMPDIR: dir, - TMP: dir, - TEMP: dir, - HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START: '1', - HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT: '1', - HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT: String(operationsPort), - HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT: String(httpPort), - }; - const saved = Object.fromEntries(Object.keys(overrides).map((name) => [name, process.env[name]])); - Object.assign(process.env, overrides); - try { - return await import(freshModuleUrl()); - } finally { - for (const [name, value] of Object.entries(saved)) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - } -} +test('releaseAllLoopbackAddressesForCurrentProcess frees every claim', async () => { + await getNextAvailableLoopbackAddress(); + await releaseAllLoopbackAddressesForCurrentProcess(); + ok(await claimIsFree()); +}); -test('hands out an address when no other process listens on its ports', async () => { +test('a release completes while a client that asked for the holder stays connected', { timeout: 5000 }, async () => { const address = await getNextAvailableLoopbackAddress(); - strictEqual(address, HOST); + const client = connect({ host: HOST, port: claimPort }); + client.on('error', () => {}); + const reply: string = await new Promise((resolve) => client.once('data', (chunk) => resolve(String(chunk)))); + match(reply, /"protocol":"harper-integration-test-loopback-claim\/1"/); + match(reply, new RegExp(`"pid":${process.pid}`)); await releaseLoopbackAddress(address); - deepStrictEqual(readPool(), [null]); + ok(await claimIsFree()); + client.destroy(); }); for (const wildcard of ['0.0.0.0', '::']) { @@ -158,11 +144,30 @@ for (const wildcard of ['0.0.0.0', '::']) { strictEqual(error.port, port); match(error.message, new RegExp(`${HOST}:${port}`)); match(error.message, new RegExp(`${ALLOW_ENV}=1`)); - deepStrictEqual(readPool(), [null], 'the refused address must go back to the pool'); + ok(await claimIsFree(), 'the refused address must go back to the pool'); }); } } +test('a listener on all interfaces on an MQTT-class port refuses the address only where it blocks binding it (Linux)', async (t) => { + const listener = await listenOnAllInterfaces('0.0.0.0', mqttPort); + if (typeof listener === 'string') return t.skip(listener); + try { + if (process.platform === 'linux') { + const error = await refusal(); + strictEqual(error.name, 'ForeignListenerError'); + strictEqual(error.port, mqttPort); + } else { + const address = await getNextAvailableLoopbackAddress(); + strictEqual(address, HOST); + await releaseLoopbackAddress(address); + } + } finally { + await close(listener); + } + ok(await claimIsFree()); +}); + for (const value of ['1', 'true', 'TRUE']) { test(`hands the address out with a warning when ${ALLOW_ENV}=${value}`, async (t) => { const listener = await listenOnAllInterfaces('0.0.0.0', httpPort); @@ -201,24 +206,33 @@ for (const value of ['0', 'false', '']) { }); } -test('waits out a listener bound to the address itself instead of refusing it', { timeout: 15000 }, async (t) => { - const warn = t.mock.method(console, 'warn', () => {}); - const canaryWarned = () => - warn.mock.calls.some((call) => String(call.arguments[0]).includes('still in use by another Harper node')); - const lingering = await listen(HOST, httpPort); - t.after(() => (lingering.listening ? close(lingering) : undefined)); - const allocation = getNextAvailableLoopbackAddress(); - while (!canaryWarned()) await sleep(10); - await close(lingering); - strictEqual(await allocation, HOST); - await releaseLoopbackAddress(HOST); -}); +for (const [portName, port] of [ + ['operations', operationsPort], + ['HTTP', httpPort], + ['MQTT-class', mqttPort], +] as const) { + test(`waits out a listener bound to the address's ${portName} port instead of refusing it`, { timeout: 15000 }, async (t) => { + const warn = t.mock.method(console, 'warn', () => {}); + const canaryWarned = () => + warn.mock.calls.some((call) => String(call.arguments[0]).includes(`still in use by another Harper node (port ${port} bound)`)); + const lingering = await listen(HOST, port); + t.after(() => (lingering.listening ? close(lingering) : undefined)); + const allocation = getNextAvailableLoopbackAddress(); + while (!canaryWarned()) await sleep(10); + ok(await claimIsFree(), 'the skipped address is not held while it waits'); + await close(lingering); + strictEqual(await allocation, HOST); + await releaseLoopbackAddress(HOST); + }); +} test('refuses an address whose port answers no handshake, as one that could not be checked', async (t) => { if (process.platform === 'win32') return t.skip('stalling a handshake needs SIGSTOP'); const listener = await startChildListener('0.0.0.0', httpPort); try { - if (!(await isPortFree(HOST, httpPort))) return t.skip(CANARY_CATCHES_IT); + if (!(await isPortFree(HOST, httpPort))) { + return t.skip('a listener on all interfaces blocks binding the address here, so the bind check refuses it first'); + } if (!(await listener.stall())) return t.skip(RESETS_INSTEAD_OF_STALLING); const error = await refusal(); strictEqual(error.name, 'LoopbackAddressValidationError'); @@ -227,22 +241,22 @@ test('refuses an address whose port answers no handshake, as one that could not } finally { await listener.close(); } - deepStrictEqual(readPool(), [null], 'the refused address must go back to the pool'); + ok(await claimIsFree(), 'the refused address must go back to the pool'); }); test('startHarper and setupHarperWithFixture remove the install directory they created when allocation refuses the address', async (t) => { const listener = await listenOnAllInterfaces('0.0.0.0', httpPort); if (typeof listener === 'string') return t.skip(listener); - const installParent = join(poolDir, 'installs'); + const installParent = join(scratchDir, 'installs'); mkdirSync(installParent); process.env.HARPER_INTEGRATION_TEST_INSTALL_PARENT_DIR = installParent; // Were the address handed out, startHarper would go on to launch Harper; make that fail instead. - process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = join(poolDir, 'no-such-harper.js'); + process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = join(scratchDir, 'no-such-harper.js'); try { await rejects(startHarper(createHarperContext('refused')), { name: 'ForeignListenerError' }); deepStrictEqual(readdirSync(installParent), []); - const fixture = mkdtempSync(join(poolDir, 'fixture-')); + const fixture = mkdtempSync(join(scratchDir, 'fixture-')); writeFileSync(join(fixture, 'config.yaml'), ''); const fixtureCtx = createHarperContext('refused-fixture'); await rejects(setupHarperWithFixture(fixtureCtx, fixture), { name: 'ForeignListenerError' }); @@ -261,146 +275,24 @@ test('startHarper and setupHarperWithFixture remove the install directory they c } }); -test('readPoolFile quarantines an unusable pool file, then reinitializes it once it has gone unmodified long enough', async (t) => { - t.mock.method(console, 'warn', () => {}); - await withTempDir(async (dir) => { - const poolPath = join(dir, 'pool.json'); - for (const content of ['[1,2,nul', '', 'null', '{}', '[]']) { - await writeFile(poolPath, content); - strictEqual(await readPoolFile(poolPath), null, `fresh ${JSON.stringify(content)} should be quarantined`); +let freshImportCounter = 0; - await ageBeyondQuarantine(poolPath); - const pool = await readPoolFile(poolPath); - ok(pool && pool.length > 0 && pool.every((slot) => slot === null), `aged ${JSON.stringify(content)} should reinitialize`); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), pool, 'the reinitialized pool should be published'); +for (const value of ['0', '65536', '1.5', 'nine']) { + test(`refuses HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT=${value} when imported`, async () => { + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = value; + try { + await rejects(import(`../src/loopbackAddressPool.ts?claim-port=${++freshImportCounter}`), /CLAIM_PORT must be an integer/); + } finally { + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); } }); -}); +} -test('readPoolFile reads a pool file deleted between its read and its stat as missing', async () => { - const m = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - readFile: async () => '[1,2,nul', - stat: async () => { - const error = new Error('simulated ENOENT') as NodeJS.ErrnoException; - error.code = 'ENOENT'; - throw error; - }, - }, - }); +test("refuses a claim port that is one of Harper's ports when imported", async () => { + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(mqttPort); try { - const { readPoolFile: freshReadPoolFile } = await import(freshModuleUrl()); - const pool = await freshReadPoolFile('unused'); - ok(pool.length > 0 && pool.every((slot: number | null) => slot === null)); + await rejects(import(`../src/loopbackAddressPool.ts?claim-port=${++freshImportCounter}`), /must not be one of Harper's ports/); } finally { - m.restore(); + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); } }); - -test('a reservation lost to an unusable pool file is not reissued until the quarantine ends', async (t) => { - const warn = t.mock.method(console, 'warn', () => {}); - await withTempDir(async (dir) => { - const pool = await importIsolatedPool(dir); - const poolPath = join(dir, POOL_FILE_NAME); - strictEqual(await pool.getNextAvailableLoopbackAddress(), '127.0.0.1'); - - // A writer that doesn't publish by rename dies mid-write, losing that reservation. - await writeFile(poolPath, '['); - const reissued = pool.getNextAvailableLoopbackAddress(); - try { - strictEqual(await Promise.race([reissued.then(() => 'reissued'), sleep(2500).then(() => 'waiting')]), 'waiting'); - - await pool.releaseLoopbackAddress('127.0.0.1'); - await pool.releaseAllLoopbackAddressesForCurrentProcess(); - strictEqual(await readFile(poolPath, 'utf-8'), '[', 'releasing must not end the quarantine'); - - await ageBeyondQuarantine(poolPath); - strictEqual(await Promise.race([reissued, sleep(5000).then(() => 'still waiting')]), '127.0.0.1'); - } finally { - // Unblocks the allocation if an assertion above failed, so the test can't hang. - await rm(poolPath, { force: true }); - await reissued; - } - strictEqual(warn.mock.calls.filter((call) => String(call.arguments[0]).includes('is unusable')).length, 1); - }); -}); - -test('releaseLoopbackAddress clears only a slot this process holds', async () => { - await withTempDir(async (dir) => { - const pool = await importIsolatedPool(dir); - const poolPath = join(dir, POOL_FILE_NAME); - - strictEqual(await pool.getNextAvailableLoopbackAddress(), '127.0.0.1'); - await pool.releaseLoopbackAddress('127.0.0.1'); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), [null]); - - const otherRunnerPid = process.pid + 1; - await writeFile(poolPath, JSON.stringify([otherRunnerPid])); - await pool.releaseLoopbackAddress('127.0.0.1'); - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), [otherRunnerPid]); - }); -}); - -test('readPoolFile still rethrows errors unrelated to a missing/corrupt file', async () => { - await withTempDir(async (dir) => { - const poolPath = join(dir, 'pool.json'); - const m = mock.module('node:fs/promises', { - // @types/node 22.0.0 only knows `namedExports`; `exports` isn't in its types yet. - namedExports: { - ...fsPromises, - readFile: async () => { - const error = new Error('simulated EACCES') as NodeJS.ErrnoException; - error.code = 'EACCES'; - throw error; - }, - }, - }); - try { - const { readPoolFile: freshReadPoolFile } = await import(freshModuleUrl()); - await rejects(() => freshReadPoolFile(poolPath), /simulated EACCES/); - } finally { - m.restore(); - } - }); -}); - -test('writePoolFile round-trips through readPoolFile and leaves no leftover pending file', async () => { - await withTempDir(async (dir) => { - const poolPath = join(dir, 'pool.json'); - const pool = [null, 123, null, 456]; - - await writePoolFile(pool, poolPath); - - deepStrictEqual(await readPoolFile(poolPath), pool); - deepStrictEqual(await readdir(dir), ['pool.json']); - }); -}); - -test('writePoolFile never leaves a partially-written file visible at the pool path', async () => { - await withTempDir(async (dir) => { - const poolPath = join(dir, 'pool.json'); - const original = [null, null, null]; - await writeFile(poolPath, JSON.stringify(original)); - - // A hard kill in this same window (vs. this mocked rejection) bypasses catch/finally and - // orphans the pending file — the pool has no reaper for that, unlike the instance registry. - const m = mock.module('node:fs/promises', { - namedExports: { - ...fsPromises, - rename: async () => { - throw new Error('simulated rename failure'); - }, - }, - }); - try { - const { writePoolFile: freshWritePoolFile } = await import(freshModuleUrl()); - await rejects(() => freshWritePoolFile([1, 2, 3], poolPath), /simulated rename failure/); - } finally { - m.restore(); - } - - deepStrictEqual(JSON.parse(await readFile(poolPath, 'utf-8')), original); - deepStrictEqual(await readdir(dir), ['pool.json']); - }); -}); diff --git a/test/loopbackAddressPoolExhaustion.test.ts b/test/loopbackAddressPoolExhaustion.test.ts index 3800b44..709c4fe 100644 --- a/test/loopbackAddressPoolExhaustion.test.ts +++ b/test/loopbackAddressPoolExhaustion.test.ts @@ -1,121 +1,221 @@ import { test, after, mock } from 'node:test'; import { match, ok, strictEqual } from 'node:assert'; -import { spawn, type ChildProcess } from 'node:child_process'; -import { once } from 'node:events'; -import { createServer, type AddressInfo } from 'node:net'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer, type AddressInfo, type Socket } from 'node:net'; +import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { setTimeout as sleep } from 'node:timers/promises'; +import { pidNamespaceUnavailable, startHolder, type Holder } from './loopbackHolder.ts'; + +const HOST = '127.0.0.1'; function freePort(): Promise { return new Promise((resolve, reject) => { const server = createServer(); server.once('error', reject); - server.listen(0, '127.0.0.1', () => { + server.listen(0, HOST, () => { const { port } = server.address() as AddressInfo; server.close(() => resolve(port)); }); }); } -const poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); +const scratchDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); +const claimPort = await freePort(); const canaryPort = await freePort(); -process.env.TMPDIR = process.env.TEMP = poolDir; +// The pool reads these once, when it is imported; the holder processes inherit them. process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; -process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '2'; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS = '2500'; +process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(canaryPort); process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT = String(canaryPort); +process.env.HARPER_INTEGRATION_TEST_OTHER_CONFLICT_PROBE_PORTS = ''; delete process.env.HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS; const { getNextAvailableLoopbackAddress, releaseLoopbackAddress, LoopbackPoolExhaustedError } = await import( '../src/loopbackAddressPool.ts' ); -const holder = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e6)'], { stdio: 'ignore' }); -ok(holder.pid, 'the holder process started'); -const holderPid = holder.pid; - -after(() => { - holder.kill(); - rmSync(poolDir, { recursive: true, force: true }); -}); +after(() => rmSync(scratchDir, { recursive: true, force: true })); -async function kill(child: ChildProcess) { - const exited = once(child, 'exit'); - child.kill('SIGKILL'); - await exited; +/** A holder with a TMPDIR of its own, so nothing but the claim itself can tell it apart from this process. */ +async function holdingHolder(options: Parameters[0] = {}): Promise { + const ownTmp = mkdtempSync(join(scratchDir, 'holder-tmp-')); + const holder = startHolder({ ...options, env: { TMPDIR: ownTmp, TMP: ownTmp, TEMP: ownTmp, ...options.env } }); + const outcome = await holder.outcome; + strictEqual(outcome.address, HOST, `the holder should hold ${HOST}, got ${JSON.stringify(outcome)}`); + return holder; } -function writePool(pool: (number | null)[]) { - writeFileSync(join(poolDir, 'harper-integration-test-loopback-pool.json'), JSON.stringify(pool)); +async function exhaustion(): Promise { + const error = await getNextAvailableLoopbackAddress().then( + async (address) => { + await releaseLoopbackAddress(address); + throw new Error(`an address was handed out from a full pool: ${address}`); + }, + (rejection) => rejection + ); + ok(error instanceof LoopbackPoolExhaustedError, `expected LoopbackPoolExhaustedError, got ${error}`); + return error; } -test('an exhausted pool rejects with the pool range and the pids holding it, instead of waiting forever', async () => { - writePool([process.pid, holderPid]); +test('an exhausted pool rejects naming the holding process, instead of waiting forever', async () => { + const holder = await holdingHolder(); const warn = mock.method(console, 'warn', () => {}); try { - const error = await getNextAvailableLoopbackAddress().then( - () => { - throw new Error('an address was handed out from a full pool'); - }, - (rejection) => rejection - ); - ok(error instanceof LoopbackPoolExhaustedError); + const error = await exhaustion(); strictEqual(error.name, 'LoopbackPoolExhaustedError'); - match(error.message, /127\.0\.0\.1-127\.0\.0\.2 \(2 addresses\)/); - match(error.message, new RegExp(`pid ${process.pid} \\(this process\\) holds 127\\.0\\.0\\.1`)); - match(error.message, new RegExp(`pid ${holderPid} holds 127\\.0\\.0\\.2`)); + match(error.message, /127\.0\.0\.1-127\.0\.0\.1 \(1 addresses\), each held by a listener on its port \d+/); + match(error.message, new RegExp(`pid ${holder.child.pid} holds 127\\.0\\.0\\.1`)); match(error.message, /HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS/); const notices = warn.mock.calls.map((call) => String(call.arguments[0])).filter((text) => text.includes('Still waiting')); strictEqual(notices.length, 1, 'one warning names the holders before the wait ends'); - match(notices[0], new RegExp(`pid ${holderPid} holds 127\\.0\\.0\\.2`)); + match(notices[0], new RegExp(`pid ${holder.child.pid} holds 127\\.0\\.0\\.1`)); } finally { warn.mock.restore(); - writePool([null, null]); + await holder.stop(); } }); -test('an address freed during the wait is handed out rather than rejected', async () => { - writePool([process.pid, holderPid]); - const [address] = await Promise.all([ - getNextAvailableLoopbackAddress(), - sleep(500).then(() => releaseLoopbackAddress('127.0.0.1')), - ]); +test("this process's own claim is listed as this process", async () => { + const address = await getNextAvailableLoopbackAddress(); + const warn = mock.method(console, 'warn', () => {}); try { - strictEqual(address, '127.0.0.1'); + const error = await exhaustion(); + match(error.message, new RegExp(`pid ${process.pid} \\(this process\\) holds 127\\.0\\.0\\.1`)); } finally { + warn.mock.restore(); await releaseLoopbackAddress(address); } }); -test('a holder that dies after the deadline is reclaimed rather than reported', async () => { - const dying = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1e6)'], { stdio: 'ignore' }); +test('an address released during the wait is handed out rather than rejected', async () => { + const held = await getNextAvailableLoopbackAddress(); + const [address] = await Promise.all([getNextAvailableLoopbackAddress(), sleep(500).then(() => releaseLoopbackAddress(held))]); try { - writePool([dying.pid!, holderPid]); - const [address] = await Promise.all([ - getNextAvailableLoopbackAddress(), - sleep(2500).then(() => kill(dying)), - ]); - try { - strictEqual(address, '127.0.0.1'); - } finally { - await releaseLoopbackAddress(address); - } + strictEqual(address, HOST); } finally { - dying.kill('SIGKILL'); + await releaseLoopbackAddress(address); + } +}); + +test('a holder that exits during the wait frees its address at once', async () => { + const holder = await holdingHolder(); + const [address] = await Promise.all([getNextAvailableLoopbackAddress(), sleep(500).then(() => holder.stop())]); + try { + strictEqual(address, HOST); + } finally { + await releaseLoopbackAddress(address); } }); -test('a pool holding only dead pids is reclaimed, not reported as exhausted', async () => { - const dead = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' }); - await new Promise((resolve) => dead.once('exit', resolve)); - writePool([dead.pid!, holderPid]); +test("a SIGKILLed holder's address is handed out next, without the pool filling first", async () => { + const twoSlots = { HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT: '2' }; + const holder = await holdingHolder({ env: twoSlots }); + await holder.kill(); + const saved = process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT; + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = twoSlots.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT; + const twoSlotPoolUrl = '../src/loopbackAddressPool.ts?two-slots'; + let pool: typeof import('../src/loopbackAddressPool.ts'); + try { + pool = await import(twoSlotPoolUrl); + } finally { + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = saved; + } + // 127.0.0.2 is free (and is not an address at all on macOS without aliases), so a pool that reclaimed dead + // holders only when full would hand it out instead. + const address = await pool.getNextAvailableLoopbackAddress(); + try { + strictEqual(address, HOST); + } finally { + await pool.releaseLoopbackAddress(address); + } +}); + +test('processes with different TMPDIRs that allocate at once never hold one address together', async () => { + const contenders = Array.from({ length: 4 }, () => { + const ownTmp = mkdtempSync(join(scratchDir, 'contender-tmp-')); + return startHolder({ waitForGo: true, env: { TMPDIR: ownTmp, TMP: ownTmp, TEMP: ownTmp } }); + }); + try { + await Promise.all(contenders.map((contender) => contender.ready)); + for (const contender of contenders) contender.go(); + const outcomes = await Promise.all(contenders.map((contender) => contender.outcome)); + strictEqual(outcomes.filter((outcome) => outcome.address === HOST).length, 1, JSON.stringify(outcomes)); + strictEqual(outcomes.filter((outcome) => outcome.error === 'LoopbackPoolExhaustedError').length, 3, JSON.stringify(outcomes)); + } finally { + await Promise.all(contenders.map((contender) => contender.stop())); + } +}); + +test('a holder in another PID namespace holds its address while it runs, and frees it when it exits', async (t) => { + const unavailable = pidNamespaceUnavailable(); + if (unavailable) { + if (process.env.LOOPBACK_POOL_TEST_REQUIRE_PID_NAMESPACE) throw new Error(unavailable); + console.log(`# skipping the PID namespace case: ${unavailable}`); + return t.skip(unavailable); + } + const holder = await holdingHolder({ pidNamespace: true }); + const warn = mock.method(console, 'warn', () => {}); + try { + const { pid } = await holder.outcome; + ok(pid !== holder.child.pid, 'the holder runs in a PID namespace of its own'); + const error = await exhaustion(); + match(error.message, new RegExp(`pid ${pid} in another PID namespace \\(pid:\\[\\d+\\]\\) holds 127\\.0\\.0\\.1`)); + } finally { + warn.mock.restore(); + await holder.stop(); + } const address = await getNextAvailableLoopbackAddress(); try { - strictEqual(address, '127.0.0.1'); + strictEqual(address, HOST); } finally { await releaseLoopbackAddress(address); } }); + +/** A listener on the claim port that is not a pool holder. Closing it resets its connections, leaving no TIME_WAIT. */ +async function listenOnClaimPort(onConnection: (socket: Socket) => void): Promise<{ close(): Promise }> { + const sockets = new Set(); + const server = createServer((socket) => { + sockets.add(socket); + socket.on('error', () => {}); + socket.once('close', () => sockets.delete(socket)); + onConnection(socket); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(claimPort, HOST, () => resolve()); + }); + return { + close: () => + new Promise((resolve) => { + server.close(() => resolve()); + for (const socket of sockets) socket.resetAndDestroy(); + }), + }; +} + +const strangers: [string, (socket: Socket) => void][] = [ + ['answers with something else', (socket) => socket.write('HTTP/1.1 400 Bad Request\r\n\r\n')], + ['answers with an unknown protocol', (socket) => socket.write('{"protocol":"other","pid":1}\n')], + ['sends a reply too long to be one', (socket) => socket.write('x'.repeat(64 * 1024))], + ['never answers', () => {}], +]; +for (const [behavior, onConnection] of strangers) { + test(`a claim port held by a process that ${behavior} is listed as unidentified, within the deadline`, async () => { + const stranger = await listenOnClaimPort(onConnection); + const warn = mock.method(console, 'warn', () => {}); + const startedAt = performance.now(); + try { + const error = await exhaustion(); + match(error.message, /held by a process that did not identify itself: 127\.0\.0\.1/); + const elapsedMs = performance.now() - startedAt; + ok(elapsedMs < 6000, `the diagnosis took ${Math.round(elapsedMs)}ms`); + } finally { + warn.mock.restore(); + await stranger.close(); + } + }); +} diff --git a/test/loopbackAddressPoolProbeFailure.test.ts b/test/loopbackAddressPoolProbeFailure.test.ts index f4dbba7..336b738 100644 --- a/test/loopbackAddressPoolProbeFailure.test.ts +++ b/test/loopbackAddressPoolProbeFailure.test.ts @@ -1,10 +1,7 @@ -import { after, before, describe, test } from 'node:test'; -import { deepStrictEqual, match, ok, strictEqual } from 'node:assert'; +import { before, describe, test } from 'node:test'; +import { match, ok, strictEqual } from 'node:assert'; import { createServer, type AddressInfo } from 'node:net'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { acceptsConnections } from '../src/portUtils.ts'; +import { acceptsConnections, isPortFree } from '../src/portUtils.ts'; const HOST = '127.0.0.1'; const ALLOW_ENV = 'HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS'; @@ -26,7 +23,7 @@ async function freePort(): Promise { * (EADDRNOTAVAIL on macOS). Resolves why they cannot run here, or false when they can. */ async function skipReason(): Promise { - if (process.platform === 'linux') return 'Linux skips the probe'; + if (process.platform === 'linux') return 'Linux checks for a listener on all interfaces by binding, not by connecting'; const probeFails = await acceptsConnections(HOST, 0).then( () => false, () => true @@ -35,30 +32,22 @@ async function skipReason(): Promise { } describe('an address that cannot be checked', { skip: await skipReason() }, () => { - let poolDir: string | undefined; + let claimPort: number; let pool: typeof import('../src/loopbackAddressPool.ts'); before(async () => { - poolDir = mkdtempSync(join(tmpdir(), 'loopback-pool-probe-test-')); - // The pool keeps its state in os.tmpdir(), which prefers TMPDIR on POSIX and TEMP on Windows. - process.env.TMPDIR = process.env.TEMP = poolDir; + claimPort = await freePort(); process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '1'; + process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(await freePort()); process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT = '0'; + process.env.HARPER_INTEGRATION_TEST_OTHER_CONFLICT_PROBE_PORTS = ''; delete process.env[ALLOW_ENV]; // The pool reads these once, when it is imported. pool = await import('../src/loopbackAddressPool.ts'); }); - after(() => { - if (poolDir) rmSync(poolDir, { recursive: true, force: true }); - }); - - function readPool(): unknown { - return JSON.parse(readFileSync(join(poolDir!, 'harper-integration-test-loopback-pool.json'), 'utf-8')); - } - test('is refused, with the reason, and goes back to the pool', async () => { let error: any; try { @@ -71,7 +60,7 @@ describe('an address that cannot be checked', { skip: await skipReason() }, () = match(error.message, /Could not check whether another process accepts connections on 127\.0\.0\.1/); match(error.message, new RegExp(`${ALLOW_ENV}=1`)); ok(error.cause, 'the probe error is the cause'); - deepStrictEqual(readPool(), [null]); + ok(await isPortFree(HOST, claimPort), 'the refused address goes back to the pool'); }); test(`is handed out with a warning when ${ALLOW_ENV} is set`, async (t) => { diff --git a/test/loopbackHolder.ts b/test/loopbackHolder.ts new file mode 100644 index 0000000..312523e --- /dev/null +++ b/test/loopbackHolder.ts @@ -0,0 +1,77 @@ +import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; +import { once } from 'node:events'; +import { createInterface } from 'node:readline'; +import { fileURLToPath } from 'node:url'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/loopbackHolder.ts', import.meta.url)); +const PID_NAMESPACE_COMMAND = ['unshare', '--user', '--map-root-user', '--pid', '--fork', '--mount-proc']; + +export interface HolderOutcome { + address?: string; + error?: string; + pid: number; +} + +export interface Holder { + child: ChildProcess; + /** Resolves once the child has loaded the pool; rejects if it exits first. */ + ready: Promise; + /** The allocation's outcome; rejects if the child exits without one. */ + outcome: Promise; + /** Starts the allocation of a holder started with `waitForGo`. */ + go(): void; + /** Ends stdin, which makes the holder exit, and waits for it. */ + stop(): Promise; + /** SIGKILLs the holder and waits for it. */ + kill(): Promise; +} + +export interface HolderOptions { + env?: Record; + waitForGo?: boolean; + /** Run the holder in its own PID namespace (Linux; see {@link pidNamespaceUnavailable}). */ + pidNamespace?: boolean; +} + +/** A process that allocates from the pool with the parent's pool settings plus `env`. */ +export function startHolder({ env = {}, waitForGo = false, pidNamespace = false }: HolderOptions = {}): Holder { + const args = [FIXTURE, ...(waitForGo ? ['--wait-for-go'] : [])]; + const [command, ...commandArgs] = pidNamespace ? [...PID_NAMESPACE_COMMAND, process.execPath, ...args] : [process.execPath, ...args]; + const child = spawn(command, commandArgs, { env: { ...process.env, ...env }, stdio: ['pipe', 'pipe', 'inherit'] }); + const exited = once(child, 'exit'); + let resolveReady!: () => void; + let resolveOutcome!: (outcome: HolderOutcome) => void; + const ready = new Promise((resolve) => (resolveReady = resolve)); + const outcome = new Promise((resolve) => (resolveOutcome = resolve)); + createInterface({ input: child.stdout! }).on('line', (line) => { + const message = JSON.parse(line); + if (message.ready) resolveReady(); + else resolveOutcome(message); + }); + const exitedEarly = exited.then(([code]) => { + throw new Error(`the holder exited (code ${code}) before reporting`); + }); + return { + child, + ready: Promise.race([ready, exitedEarly]), + outcome: Promise.race([outcome, exitedEarly]), + go: () => child.stdin!.write('go\n'), + async stop() { + child.stdin!.end(); + if (child.exitCode === null && child.signalCode === null) await exited; + }, + async kill() { + child.kill('SIGKILL'); + if (child.exitCode === null && child.signalCode === null) await exited; + }, + }; +} + +/** Why a holder cannot run in its own PID namespace here, or false when it can. */ +export function pidNamespaceUnavailable(): string | false { + if (process.platform !== 'linux') return 'PID namespaces are Linux-only'; + const probe = spawnSync(PID_NAMESPACE_COMMAND[0], [...PID_NAMESPACE_COMMAND.slice(1), 'true'], { encoding: 'utf8' }); + if (probe.error) return `cannot run unshare: ${probe.error.message}`; + if (probe.status !== 0) return `unshare cannot create a PID namespace here: ${probe.stderr.trim()}`; + return false; +} From b17ded0db9b7ab6f460babf4afc593ece7fc1e78 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 09:26:41 -0600 Subject: [PATCH 5/6] fix(loopback): keep pool diagnostics accurate under concurrent allocations - Each pool address gets its own control address (127.255.0.x), which only that address's claim holder binds, so two allocations probing different addresses cannot mistake each other's probe for a listener on all interfaces. - A claim reports whether its allocation is still checking the address. The exhaustion message no longer names such a process as a holder, and lists the Harper port that made it give the address up. - A hostname lookup failure no longer escapes the claim's connection handler, and the identity size limit also applies to a reply that ends in a newline. - Tests release during a wait only after the allocation reports waiting, clean up on every exit, and pick their ports while holding them all. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: integration-testing-44-pool-liveness-root-fix --- src/loopbackAddressPool.ts | 109 +++++++++++-------- test/loopbackAddressPool.test.ts | 1 - test/loopbackAddressPoolExhaustion.test.ts | 88 ++++++++++++--- test/loopbackAddressPoolProbeFailure.test.ts | 29 +++-- test/loopbackHolder.ts | 2 - 5 files changed, 153 insertions(+), 76 deletions(-) diff --git a/src/loopbackAddressPool.ts b/src/loopbackAddressPool.ts index c8de756..d343253 100644 --- a/src/loopbackAddressPool.ts +++ b/src/loopbackAddressPool.ts @@ -80,13 +80,19 @@ if (CONFLICT_PROBE_PORTS.includes(CLAIM_PORT)) { throw new Error(`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT (${CLAIM_PORT}) must not be one of Harper's ports`); } -// Linux routes all of 127.0.0.0/8 to lo; this address is outside the pool's 127.0.0.x range. -const CONTROL_ADDRESS = '127.255.255.254'; +/** + * An address outside the pool (Linux routes all of 127.0.0.0/8 to lo) for telling a listener on all interfaces from + * one on a pool address. Each pool address gets its own, which only that address's claim holder binds, so concurrent + * probes of different addresses cannot mistake each other for such a listener. + */ +function controlAddressFor(loopbackAddress: string): string { + return loopbackAddress.replace(/^127\.0\.0\./, '127.255.0.'); +} const IDENTITY_PROTOCOL = 'harper-integration-test-loopback-claim/1'; const IDENTITY_SERVE_MS = 2000; const IDENTITY_QUERY_MS = 1000; -const IDENTITY_MAX_BYTES = 1024; +const IDENTITY_MAX_CHARACTERS = 1024; interface LoopbackAddressError extends Error { loopbackAddress: string; @@ -160,6 +166,8 @@ interface Claim { server: Server; /** Identity queries in flight; release resets them so they cannot hold the claim port open. */ connections: Set; + /** False while allocation still checks the address's Harper ports, and may yet let it go. */ + accepted: boolean; } // Shared by every copy of this module in the process, so release-all keeps meaning "this process". @@ -181,6 +189,7 @@ interface HolderIdentity { pid: number; pidNamespace: string | null; hostname: string | null; + checking?: boolean; } let ownIdentity: HolderIdentity | undefined; @@ -192,15 +201,18 @@ function getOwnIdentity(): HolderIdentity { pidNamespace = readlinkSync('/proc/self/ns/pid'); } catch {} } - ownIdentity = { pid: process.pid, pidNamespace, hostname: hostname() }; + let ownHostname: string | null = null; + try { + ownHostname = hostname(); + } catch {} + ownIdentity = { pid: process.pid, pidNamespace, hostname: ownHostname }; } return ownIdentity; } -let identityLine: string | undefined; - /** The client closes first, so the claim port never sits in TIME_WAIT on the holder's side. */ -function answerIdentity(connections: Set, socket: Socket): void { +function answerIdentity(claim: Claim, socket: Socket): void { + const { connections } = claim; connections.add(socket); socket.on('error', () => {}); socket.unref(); @@ -210,14 +222,14 @@ function answerIdentity(connections: Set, socket: Socket): void { connections.delete(socket); }); socket.resume(); - socket.write((identityLine ??= `${JSON.stringify({ protocol: IDENTITY_PROTOCOL, ...getOwnIdentity() })}\n`)); + socket.write(`${JSON.stringify({ protocol: IDENTITY_PROTOCOL, ...getOwnIdentity(), checking: !claim.accepted })}\n`); } /** Resolves the claim, or `null` when another listener already holds the address's claim port. */ function claimAddress(address: string): Promise { return new Promise((resolve, reject) => { - const connections = new Set(); - const server = createServer((socket) => answerIdentity(connections, socket)); + const server = createServer((socket) => answerIdentity(claim, socket)); + const claim: Claim = { server, connections: new Set(), accepted: false }; const onError = (error: NodeJS.ErrnoException) => { if (error.code === 'EADDRINUSE') { resolve(null); @@ -232,7 +244,7 @@ function claimAddress(address: string): Promise { // An accept failure (EMFILE) must not crash the holder; the claim stays bound. server.on('error', () => {}); server.unref(); - resolve({ server, connections }); + resolve(claim); }); }); } @@ -254,12 +266,13 @@ function parseIdentity(line: string): HolderIdentity | null { return null; } if (typeof reply !== 'object' || reply === null) return null; - const { protocol, pid, pidNamespace, hostname } = reply as Record; + const { protocol, pid, pidNamespace, hostname, checking } = reply as Record; if (protocol !== IDENTITY_PROTOCOL || !Number.isSafeInteger(pid) || (pid as number) < 1) return null; return { pid: pid as number, pidNamespace: typeof pidNamespace === 'string' && /^pid:\[\d+\]$/.test(pidNamespace) ? pidNamespace : null, hostname: typeof hostname === 'string' && /^[\w.-]{1,253}$/.test(hostname) ? hostname : null, + checking: checking === true, }; } @@ -281,8 +294,8 @@ function queryHolder(address: string): Promise { socket.on('data', (chunk: string) => { reply += chunk; const newline = reply.indexOf('\n'); - if (newline !== -1) settle(parseIdentity(reply.slice(0, newline)) ?? 'unidentified'); - else if (reply.length > IDENTITY_MAX_BYTES) settle('unidentified'); + if ((newline === -1 ? reply.length : newline) > IDENTITY_MAX_CHARACTERS) settle('unidentified'); + else if (newline !== -1) settle(parseIdentity(reply.slice(0, newline)) ?? 'unidentified'); }); socket.on('error', (error: NodeJS.ErrnoException) => settle(error.code === 'ECONNREFUSED' ? 'released' : 'unidentified')); socket.on('end', () => settle('unidentified')); @@ -291,6 +304,9 @@ function queryHolder(address: string): Promise { function describeHolder(identity: HolderIdentity): string { const own = getOwnIdentity(); + if (identity.pid === own.pid && identity.pidNamespace === own.pidNamespace && identity.hostname === own.hostname) { + return `pid ${own.pid} (this process)`; + } let description = `pid ${identity.pid}`; if (identity.pidNamespace && own.pidNamespace && identity.pidNamespace !== own.pidNamespace) { description += ` in another PID namespace (${identity.pidNamespace})`; @@ -367,10 +383,10 @@ function isPortInUseOnAddress(port: number, loopbackAddress: string): Promise { +async function isHeldOnAllInterfaces(port: number, loopbackAddress: string): Promise { if (process.platform !== 'linux') return false; try { - return await isPortInUseOnAddress(port, CONTROL_ADDRESS); + return await isPortInUseOnAddress(port, controlAddressFor(loopbackAddress)); } catch { return false; } @@ -394,7 +410,7 @@ async function findConflictingPort(loopbackAddress: string): Promise): Promise { const isOwn = (index: number) => claims.has(claimKey(addressAt(index))); @@ -443,32 +459,42 @@ async function describePoolState(claimed: readonly number[], conflictPorts: Read const others = claimed.filter((index) => !isOwn(index)); const reports = await Promise.all(others.map((index) => queryHolder(addressAt(index)))); const addressesByHolder = new Map(); - const unidentified: string[] = []; + const addHolder = (holder: string, index: number) => + addressesByHolder.set(holder, [...(addressesByHolder.get(holder) ?? []), addressAt(index)]); + const unidentifiedIndices: number[] = []; + const occupied = new Map([...conflictPorts].filter(([index]) => !claimed.includes(index))); others.forEach((index, i) => { const report = reports[i]; - if (report === 'released') return; if (report === 'unidentified') { - unidentified.push(addressAt(index)); + unidentifiedIndices.push(index); return; } - const holder = describeHolder(report); - addressesByHolder.set(holder, [...(addressesByHolder.get(holder) ?? []), addressAt(index)]); + // A claim given up or still being checked is another allocation trying the address, not a holder of it. + if (report === 'released' || report.checking) { + const port = conflictPorts.get(index); + if (port !== undefined) occupied.set(index, port); + else if (report !== 'released') addHolder(`${describeHolder(report)} is checking`, index); + return; + } + addHolder(`${describeHolder(report)} holds`, index); }); const lines = [ `Pool ${addressAt(0)}-${addressAt(HARPER_LOOPBACK_POOL_COUNT - 1)} (${HARPER_LOOPBACK_POOL_COUNT} addresses), each held by a listener on its port ${CLAIM_PORT}:`, ]; if (ownAddresses.length > 0) lines.push(` pid ${process.pid} (this process) holds ${ownAddresses.join(', ')}`); - for (const [holder, addresses] of addressesByHolder) lines.push(` ${holder} holds ${addresses.join(', ')}`); - if (unidentified.length > 0) { - lines.push(` held by a process that did not identify itself: ${unidentified.join(', ')}`); - if (await isHeldOnAllInterfaces(CLAIM_PORT)) { + for (const [holder, addresses] of addressesByHolder) lines.push(` ${holder} ${addresses.join(', ')}`); + if (unidentifiedIndices.length > 0) { + lines.push(` held by a process that did not identify itself: ${unidentifiedIndices.map(addressAt).join(', ')}`); + if (await isHeldOnAllInterfaces(CLAIM_PORT, addressAt(unidentifiedIndices[0]))) { lines.push( ` port ${CLAIM_PORT} is bound on all interfaces by another program, which holds every address; set HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT to a free port for every run` ); } } - const occupied = [...conflictPorts].map(([index, port]) => `${addressAt(index)} (port ${port})`); - if (occupied.length > 0) lines.push(` unclaimed but bound by another Harper node: ${occupied.join(', ')}`); + if (occupied.size > 0) { + const addresses = [...occupied].sort(([a], [b]) => a - b).map(([index, port]) => `${addressAt(index)} (port ${port})`); + lines.push(` unclaimed but bound by another Harper node: ${addresses.join(', ')}`); + } lines.push( `Each holder is alive: its listener closes when it exits. \`lsof -nP -iTCP@
:${CLAIM_PORT} -sTCP:LISTEN\` names a holder by its pid on this host, and \`lsof -nP -iTCP@
-sTCP:LISTEN\` what binds an address's Harper ports (\`netstat -ano | findstr
:\` on Windows). Stop what leaked.` ); @@ -476,23 +502,13 @@ async function describePoolState(claimed: readonly number[], conflictPorts: Read } /** - * Retrieves the next available loopback address from the pool. Safe across concurrent processes, - * whatever their PID namespace or TMPDIR, as long as they share a network namespace. - * - * **How it works:** - * 1. Claims the first address whose claim port (`HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT`, default - * 9924) no other process listens on, by listening there itself until the address is released or - * this process exits - * 2. Skips an address whose Harper ports (operations, HTTP, HTTPS, MQTT, MQTTS) are already bound - * on it, such as by a Harper node that outlived its runner, and tries the next one - * 3. Checks that no other process accepts connections on the address's operations and HTTP ports - * 4. Returns the loopback address (e.g., "127.0.0.2") - * - * If every address is held, waits and retries until one becomes available, for up to - * `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default 10 minutes). A wait of 30 seconds or - * more logs one warning naming the holders. + * Allocates a loopback address from the pool, safely across concurrent processes that share a network namespace, + * whatever their PID namespace or TMPDIR. The address stays held until it is released or this process exits. An + * address whose Harper ports are still bound, such as by a node that outlived its runner, is skipped. If every + * address is held, waits for one for up to `HARPER_INTEGRATION_TEST_LOOPBACK_POOL_TIMEOUT_MS` (default 10 minutes), + * logging one warning naming the holders after 30 seconds. * - * @returns A promise that resolves with an allocated loopback address + * @returns A promise that resolves with an allocated loopback address (e.g. "127.0.0.2") * @throws {LoopbackAddressValidationError} If the address cannot be bound to, or cannot be checked for * another process's listener * @throws {ForeignListenerError} If another process accepts connections on the address's Harper ports and @@ -502,10 +518,10 @@ async function describePoolState(claimed: readonly number[], conflictPorts: Read export async function getNextAvailableLoopbackAddress(): Promise { const waitStartedAt = performance.now(); const warnedConflicts = new Set(); + const conflictPorts = new Map(); let noticed = false; while (true) { const claimed: number[] = []; - const conflictPorts = new Map(); for (let index = 0; index < HARPER_LOOPBACK_POOL_COUNT; index++) { const loopbackAddress = addressAt(index); const claim = await claimAddress(loopbackAddress); @@ -521,6 +537,7 @@ export async function getNextAvailableLoopbackAddress(): Promise { throw error; } if (accepted) { + claim.accepted = true; claims.set(claimKey(loopbackAddress), claim); return loopbackAddress; } diff --git a/test/loopbackAddressPool.test.ts b/test/loopbackAddressPool.test.ts index f1a4c61..f3b20f6 100644 --- a/test/loopbackAddressPool.test.ts +++ b/test/loopbackAddressPool.test.ts @@ -55,7 +55,6 @@ async function claimIsFree(): Promise { return isPortFree(HOST, claimPort); } -/** Resolves a reason to skip where this host cannot listen on all interfaces. */ async function listenOnAllInterfaces(host: string, port: number): Promise { try { return await listen(host, port); diff --git a/test/loopbackAddressPoolExhaustion.test.ts b/test/loopbackAddressPoolExhaustion.test.ts index 709c4fe..cba1ccc 100644 --- a/test/loopbackAddressPoolExhaustion.test.ts +++ b/test/loopbackAddressPoolExhaustion.test.ts @@ -1,6 +1,6 @@ import { test, after, mock } from 'node:test'; import { match, ok, strictEqual } from 'node:assert'; -import { createServer, type AddressInfo, type Socket } from 'node:net'; +import { createServer, type AddressInfo, type Server, type Socket } from 'node:net'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -9,20 +9,24 @@ import { pidNamespaceUnavailable, startHolder, type Holder } from './loopbackHol const HOST = '127.0.0.1'; -function freePort(): Promise { +function listen(port: number): Promise { return new Promise((resolve, reject) => { const server = createServer(); server.once('error', reject); - server.listen(0, HOST, () => { - const { port } = server.address() as AddressInfo; - server.close(() => resolve(port)); - }); + server.listen(port, HOST, () => resolve(server)); }); } +/** Distinct free ports: all are held at once, so the OS cannot hand out the same one twice. */ +async function freePorts(count: number): Promise { + const servers = await Promise.all(Array.from({ length: count }, () => listen(0))); + const ports = servers.map((server) => (server.address() as AddressInfo).port); + await Promise.all(servers.map((server) => new Promise((resolve) => server.close(resolve)))); + return ports; +} + const scratchDir = mkdtempSync(join(tmpdir(), 'loopback-pool-exhaustion-test-')); -const claimPort = await freePort(); -const canaryPort = await freePort(); +const [claimPort, canaryPort] = await freePorts(2); // The pool reads these once, when it is imported; the holder processes inherit them. process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '1'; @@ -42,11 +46,34 @@ after(() => rmSync(scratchDir, { recursive: true, force: true })); async function holdingHolder(options: Parameters[0] = {}): Promise { const ownTmp = mkdtempSync(join(scratchDir, 'holder-tmp-')); const holder = startHolder({ ...options, env: { TMPDIR: ownTmp, TMP: ownTmp, TEMP: ownTmp, ...options.env } }); - const outcome = await holder.outcome; - strictEqual(outcome.address, HOST, `the holder should hold ${HOST}, got ${JSON.stringify(outcome)}`); + try { + const outcome = await holder.outcome; + strictEqual(outcome.address, HOST, `the holder should hold ${HOST}, got ${JSON.stringify(outcome)}`); + } catch (error) { + await holder.kill(); + throw error; + } return holder; } +/** Resolves once `allocation` has logged that it is waiting, proving it found the pool full first. */ +async function untilWaiting(warn: ReturnType, allocation: Promise): Promise { + let settled = false; + allocation.then( + () => (settled = true), + () => (settled = true) + ); + while (!warn.mock.calls.some((call) => String(call.arguments[0]).includes('Still waiting'))) { + if (settled) throw new Error('the allocation settled without waiting'); + await sleep(10); + } +} + +/** Returns an allocation's address to the pool whether it has settled yet or not. */ +async function settleAllocation(allocation: Promise): Promise { + await allocation.then(releaseLoopbackAddress, () => {}); +} + async function exhaustion(): Promise { const error = await getNextAvailableLoopbackAddress().then( async (address) => { @@ -92,21 +119,31 @@ test("this process's own claim is listed as this process", async () => { test('an address released during the wait is handed out rather than rejected', async () => { const held = await getNextAvailableLoopbackAddress(); - const [address] = await Promise.all([getNextAvailableLoopbackAddress(), sleep(500).then(() => releaseLoopbackAddress(held))]); + const warn = mock.method(console, 'warn', () => {}); + const allocation = getNextAvailableLoopbackAddress(); try { - strictEqual(address, HOST); + await untilWaiting(warn, allocation); + await releaseLoopbackAddress(held); + strictEqual(await allocation, HOST); } finally { - await releaseLoopbackAddress(address); + warn.mock.restore(); + await releaseLoopbackAddress(held); + await settleAllocation(allocation); } }); test('a holder that exits during the wait frees its address at once', async () => { const holder = await holdingHolder(); - const [address] = await Promise.all([getNextAvailableLoopbackAddress(), sleep(500).then(() => holder.stop())]); + const warn = mock.method(console, 'warn', () => {}); + const allocation = getNextAvailableLoopbackAddress(); try { - strictEqual(address, HOST); + await untilWaiting(warn, allocation); + await holder.stop(); + strictEqual(await allocation, HOST); } finally { - await releaseLoopbackAddress(address); + warn.mock.restore(); + await holder.stop(); + await settleAllocation(allocation); } }); @@ -201,6 +238,10 @@ const strangers: [string, (socket: Socket) => void][] = [ ['answers with something else', (socket) => socket.write('HTTP/1.1 400 Bad Request\r\n\r\n')], ['answers with an unknown protocol', (socket) => socket.write('{"protocol":"other","pid":1}\n')], ['sends a reply too long to be one', (socket) => socket.write('x'.repeat(64 * 1024))], + [ + 'sends an identity line too long to be one', + (socket) => socket.write(`${JSON.stringify({ protocol: 'harper-integration-test-loopback-claim/1', pid: 4242, padding: 'x'.repeat(4096) })}\n`), + ], ['never answers', () => {}], ]; for (const [behavior, onConnection] of strangers) { @@ -219,3 +260,18 @@ for (const [behavior, onConnection] of strangers) { } }); } + +test('a claim still being checked by another allocation is not listed as holding its address', async () => { + const checking = await listenOnClaimPort((socket) => + socket.write(`${JSON.stringify({ protocol: 'harper-integration-test-loopback-claim/1', pid: 4242, checking: true })}\n`) + ); + const warn = mock.method(console, 'warn', () => {}); + try { + const error = await exhaustion(); + match(error.message, /pid 4242 is checking 127\.0\.0\.1/); + ok(!/pid 4242 holds/.test(error.message), error.message); + } finally { + warn.mock.restore(); + await checking.close(); + } +}); diff --git a/test/loopbackAddressPoolProbeFailure.test.ts b/test/loopbackAddressPoolProbeFailure.test.ts index 336b738..805c2ab 100644 --- a/test/loopbackAddressPoolProbeFailure.test.ts +++ b/test/loopbackAddressPoolProbeFailure.test.ts @@ -6,15 +6,21 @@ import { acceptsConnections, isPortFree } from '../src/portUtils.ts'; const HOST = '127.0.0.1'; const ALLOW_ENV = 'HARPER_INTEGRATION_TEST_ALLOW_FOREIGN_LISTENERS'; -async function freePort(): Promise { - const server = createServer(); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, HOST, () => resolve()); - }); - const { port } = server.address() as AddressInfo; - await new Promise((resolve) => server.close(() => resolve())); - return port; +/** Distinct free ports: all are held at once, so the OS cannot hand out the same one twice. */ +async function freePorts(count: number): Promise { + const servers = Array.from({ length: count }, () => createServer()); + await Promise.all( + servers.map( + (server) => + new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, HOST, () => resolve()); + }) + ) + ); + const ports = servers.map((server) => (server.address() as AddressInfo).port); + await Promise.all(servers.map((server) => new Promise((resolve) => server.close(() => resolve())))); + return ports; } /** @@ -36,11 +42,12 @@ describe('an address that cannot be checked', { skip: await skipReason() }, () = let pool: typeof import('../src/loopbackAddressPool.ts'); before(async () => { - claimPort = await freePort(); + let operationsPort: number; + [claimPort, operationsPort] = await freePorts(2); process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_START = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_POOL_COUNT = '1'; process.env.HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT = String(claimPort); - process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(await freePort()); + process.env.HARPER_INTEGRATION_TEST_CONFLICT_PROBE_PORT = String(operationsPort); process.env.HARPER_INTEGRATION_TEST_HTTP_CONFLICT_PROBE_PORT = '0'; process.env.HARPER_INTEGRATION_TEST_OTHER_CONFLICT_PROBE_PORTS = ''; delete process.env[ALLOW_ENV]; diff --git a/test/loopbackHolder.ts b/test/loopbackHolder.ts index 312523e..21831e1 100644 --- a/test/loopbackHolder.ts +++ b/test/loopbackHolder.ts @@ -18,11 +18,9 @@ export interface Holder { ready: Promise; /** The allocation's outcome; rejects if the child exits without one. */ outcome: Promise; - /** Starts the allocation of a holder started with `waitForGo`. */ go(): void; /** Ends stdin, which makes the holder exit, and waits for it. */ stop(): Promise; - /** SIGKILLs the holder and waits for it. */ kill(): Promise; } From b5377ed2ae4a3369dc531ebc8ae8e57b5683535e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 29 Oct 2026 09:39:55 -0600 Subject: [PATCH 6/6] fix(loopback): probe a random control address when diagnosing the claim port Waiters hold no claim when they diagnose a held pool, so two that diagnose at once could bind the same control address and one would wrongly report the claim port as bound on all interfaces. The test helper also marks both of a holder's early-exit rejections handled. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: integration-testing-44-pool-liveness-root-fix --- src/loopbackAddressPool.ts | 13 +++++++------ test/loopbackHolder.ts | 9 +++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/loopbackAddressPool.ts b/src/loopbackAddressPool.ts index d343253..a5d607f 100644 --- a/src/loopbackAddressPool.ts +++ b/src/loopbackAddressPool.ts @@ -1,4 +1,5 @@ import { setTimeout as sleep } from 'node:timers/promises'; +import { randomInt } from 'node:crypto'; import { readlinkSync } from 'node:fs'; import { hostname } from 'node:os'; import { connect, createServer, type Server, type Socket } from 'node:net'; @@ -225,7 +226,6 @@ function answerIdentity(claim: Claim, socket: Socket): void { socket.write(`${JSON.stringify({ protocol: IDENTITY_PROTOCOL, ...getOwnIdentity(), checking: !claim.accepted })}\n`); } -/** Resolves the claim, or `null` when another listener already holds the address's claim port. */ function claimAddress(address: string): Promise { return new Promise((resolve, reject) => { const server = createServer((socket) => answerIdentity(claim, socket)); @@ -276,7 +276,7 @@ function parseIdentity(line: string): HolderIdentity | null { }; } -/** Asks a claim's holder who it is. Whatever answers is untrusted, and only ever described. */ +/** Whatever answers is untrusted, and only ever described. */ function queryHolder(address: string): Promise { return new Promise((resolve) => { const socket = connect({ host: address, port: CLAIM_PORT }); @@ -383,10 +383,10 @@ function isPortInUseOnAddress(port: number, loopbackAddress: string): Promise { +async function isHeldOnAllInterfaces(port: number, controlAddress: string): Promise { if (process.platform !== 'linux') return false; try { - return await isPortInUseOnAddress(port, controlAddressFor(loopbackAddress)); + return await isPortInUseOnAddress(port, controlAddress); } catch { return false; } @@ -410,7 +410,7 @@ async function findConflictingPort(loopbackAddress: string): Promise 0) { lines.push(` held by a process that did not identify itself: ${unidentifiedIndices.map(addressAt).join(', ')}`); - if (await isHeldOnAllInterfaces(CLAIM_PORT, addressAt(unidentifiedIndices[0]))) { + // Waiters hold no claim here, so each probes a random control address rather than one they might share. + if (await isHeldOnAllInterfaces(CLAIM_PORT, `127.254.${randomInt(256)}.${randomInt(1, 255)}`)) { lines.push( ` port ${CLAIM_PORT} is bound on all interfaces by another program, which holds every address; set HARPER_INTEGRATION_TEST_LOOPBACK_CLAIM_PORT to a free port for every run` ); diff --git a/test/loopbackHolder.ts b/test/loopbackHolder.ts index 21831e1..bf0f1f6 100644 --- a/test/loopbackHolder.ts +++ b/test/loopbackHolder.ts @@ -49,10 +49,15 @@ export function startHolder({ env = {}, waitForGo = false, pidNamespace = false const exitedEarly = exited.then(([code]) => { throw new Error(`the holder exited (code ${code}) before reporting`); }); + const readyOrExit = Promise.race([ready, exitedEarly]); + const outcomeOrExit = Promise.race([outcome, exitedEarly]); + // A caller that stops at the first rejection would otherwise leave the other one unhandled. + readyOrExit.catch(() => {}); + outcomeOrExit.catch(() => {}); return { child, - ready: Promise.race([ready, exitedEarly]), - outcome: Promise.race([outcome, exitedEarly]), + ready: readyOrExit, + outcome: outcomeOrExit, go: () => child.stdin!.write('go\n'), async stop() { child.stdin!.end();