diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 63ac394f..514abe22 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -51,6 +51,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..aa4e71ad --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,297 @@ +/** + * Process isolation for walltime runs: every benchmark runs in fresh copies + * of the benchmark process (forks), so the allocator state, JIT state, and + * handles that other benchmarks or setup code leave behind can't change its + * measurement. Rounds within one process are correlated (JIT tier-up, heap + * sizing), so the rounds of several forks are pooled into one result. + */ +import { spawn, spawnSync } from "child_process"; +import { readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { deserialize, serialize } from "v8"; +import { constants, gunzipSync, gzipSync } from "zlib"; + +/** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Forks per walltime benchmark. Overrides the integration's `forks` option. */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** Forks per walltime benchmark when neither the env nor an option sets it. */ +export const DEFAULT_WALLTIME_FORKS = 3; +/** Set on each fork: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** Set on each fork: the file where it writes its round samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; +/** + * Set on the forks after the first one by {@link runForks}: the number of + * iterations that the first fork measured per round. Every fork of a benchmark + * MUST batch the same way, because the samples of all forks are pooled. + */ +export const FORK_ITERATIONS_PER_ROUND_ENV = + "CODSPEED_FORK_ITERATIONS_PER_ROUND"; +/** + * JSON array `[executable, ...args]` of the command that runs the benchmarks + * again. Integrations whose benchmarks run in a worker process (vitest) set it + * from the main process, because re-executing the worker alone can't work. + */ +export const ISOLATION_COMMAND_ENV = "CODSPEED_ISOLATION_COMMAND"; + +/** Walltime settings that every integration exposes next to its own options. */ +export interface WalltimeOptions { + /** + * Fresh processes that each walltime benchmark runs in; their rounds are + * reported together. {@link WALLTIME_FORKS_ENV} overrides it; defaults to + * {@link DEFAULT_WALLTIME_FORKS}. `CODSPEED_PROCESS_ISOLATION=false` runs + * everything in the benchmark process instead. + */ + forks?: number; +} + +export type IsolationMode = + /** All benchmarks are measured in this process. */ + | { kind: "SharedProcess" } + /** Every benchmark is measured in `forks` fresh processes, one after the other. */ + | { kind: "Forks"; forks: number } + /** + * A fork: measures only `uri` and writes its samples to `samplesFile`. The + * first fork picks `iterationsPerRound` itself; the ones after it inherit it. + */ + | { + kind: "Fork"; + uri: string; + samplesFile: string; + iterationsPerRound?: number; + }; + +/** + * The isolation of a walltime run, from the environment. `forks` is the + * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + */ +export function walltimeIsolation(forks?: number): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; + if (samplesFile === undefined) { + throw new Error( + `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, + ); + } + const iterationsPerRound = process.env[FORK_ITERATIONS_PER_ROUND_ENV]; + return { + kind: "Fork", + uri, + samplesFile, + iterationsPerRound: + iterationsPerRound === undefined || iterationsPerRound === "" + ? undefined + : validPositiveInt(iterationsPerRound, FORK_ITERATIONS_PER_ROUND_ENV), + }; + } + const isolation = process.env[PROCESS_ISOLATION_ENV]; + if (isolation === "0" || isolation === "false") { + return { kind: "SharedProcess" }; + } + if (isolation !== undefined && isolation !== "1" && isolation !== "true") { + throw new Error( + `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``, + ); + } + return { kind: "Forks", forks: resolveWalltimeForks(forks) }; +} + +/** + * The number of forks per walltime benchmark: {@link WALLTIME_FORKS_ENV}, + * else `option`, else {@link DEFAULT_WALLTIME_FORKS}. + */ +export function resolveWalltimeForks(option?: number): number { + const env = process.env[WALLTIME_FORKS_ENV]; + if (env !== undefined && env !== "") { + return validPositiveInt(env, WALLTIME_FORKS_ENV); + } + if (option !== undefined) { + return validPositiveInt(option, "forks"); + } + return DEFAULT_WALLTIME_FORKS; +} + +function validPositiveInt(raw: string | number, source: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { + throw new Error( + `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, + ); + } + return value; +} + +/** + * Records the command of this process as the one that forks run. Call it + * from the main process of integrations that measure in worker processes. It + * overwrites a command inherited from an outer process, which is not ours. + */ +export function recordIsolationCommand(): void { + process.env[ISOLATION_COMMAND_ENV] = JSON.stringify(currentCommand()); +} + +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +function isolationCommand(): string[] { + const recorded = process.env[ISOLATION_COMMAND_ENV]; + if (recorded === undefined) { + return currentCommand(); + } + return JSON.parse(recorded) as string[]; +} + +/** The pooled samples of the forks of one benchmark. */ +export interface ForkSamples { + /** Round samples (milliseconds), one per measured round. */ + samples: number[]; + /** Benchmark iterations one round measured; every fork MUST agree. */ + iterationsPerRound: number; +} + +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Throws when + * a fork fails, without running the next one. + * + * The first fork picks how many iterations one round batches; the ones after + * it inherit that count, so the pooled samples mean the same thing. + * + * Each fork re-runs the benchmark command, not fork(): Node can't fork, and a + * fork would only copy the calling thread. The fork's event loop stays free + * while it runs: a Vitest worker must keep answering its main process, whose + * calls time out otherwise. + */ +export async function runForks( + uri: string, + forks: number, +): Promise { + let result: ForkSamples | undefined; + for (let fork = 0; fork < forks; fork++) { + const next = await runFork(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); + } + return result ?? { samples: [], iterationsPerRound: 1 }; +} + +/** {@link runForks} for synchronous runs, which block anyway. */ +export function runForksSync(uri: string, forks: number): ForkSamples { + let result: ForkSamples | undefined; + for (let fork = 0; fork < forks; fork++) { + const next = runForkSync(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); + } + return result ?? { samples: [], iterationsPerRound: 1 }; +} + +function poolForkSamples( + uri: string, + pooled: ForkSamples | undefined, + next: ForkSamples, +): ForkSamples { + if (!pooled) return next; + if (pooled.iterationsPerRound !== next.iterationsPerRound) { + throw new Error( + `[CodSpeed] the forks of ${uri} measured a different number of iterations per round (${pooled.iterationsPerRound} and ${next.iterationsPerRound})`, + ); + } + // Not push(...): spreading a time-based run's samples overflows the stack. + for (const sample of next.samples) pooled.samples.push(sample); + return pooled; +} + +/** Forks run one after the other, so one file per parent process suffices. */ +const SAMPLES_FILE = join(tmpdir(), `codspeed-samples-${process.pid}.v8.gz`); + +/** Runs one fork of `uri` and returns the samples it wrote. */ +async function runFork( + uri: string, + iterationsPerRound?: number, +): Promise { + rmSync(SAMPLES_FILE, { force: true }); // a leftover file must not pass as this fork's + const [executable, ...args] = isolationCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: forkEnv(uri, iterationsPerRound), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (status, signal) => resolve([status, signal])); + const [status, signal] = await promise; + return forkSamples(uri, status, signal); +} + +function runForkSync(uri: string, iterationsPerRound?: number): ForkSamples { + rmSync(SAMPLES_FILE, { force: true }); + const [executable, ...args] = isolationCommand(); + const result = spawnSync(executable, args, { + stdio: "inherit", + env: forkEnv(uri, iterationsPerRound), + }); + if (result.error !== undefined) { + throw startError(uri, result.error); + } + return forkSamples(uri, result.status, result.signal); +} + +function forkEnv(uri: string, iterationsPerRound?: number): NodeJS.ProcessEnv { + return { + ...process.env, + [ISOLATED_BENCHMARK_ENV]: uri, + [FORK_SAMPLES_FILE_ENV]: SAMPLES_FILE, + [FORK_ITERATIONS_PER_ROUND_ENV]: iterationsPerRound?.toString(), + }; +} + +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** The samples a finished fork wrote, deleting the file; throws if it failed. */ +function forkSamples( + uri: string, + status: number | null, + signal: NodeJS.Signals | null, +): ForkSamples { + try { + if (status !== 0) { + throw new Error( + `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, + ); + } + try { + return deserialize(gunzipSync(readFileSync(SAMPLES_FILE))) as ForkSamples; + } catch (error) { + throw new Error( + `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, + ); + } + } finally { + rmSync(SAMPLES_FILE, { force: true }); + } +} + +/** + * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data, + * along with the number of benchmark iterations one round measured. + */ +export function writeForkSamples( + samplesFile: string, + samples: number[], + iterationsPerRound = 1, +) { + writeFileSync( + samplesFile, + gzipSync(serialize({ samples, iterationsPerRound }), { + level: constants.Z_BEST_SPEED, + }), + ); +} diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index fafa9e9e..baf97181 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -18,3 +18,49 @@ export const msToNs = (ms: number) => ms * 1e6; * @returns the seconds */ export const msToS = (ms: number) => ms / 1e3; + +/** tinybench-style statistics of round samples, in milliseconds. */ +export interface RoundsResult { + totalTime: number; + latency: { + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; + }; +} + +/** + * The result of nonempty round samples in milliseconds, e.g. the samples of + * several forks of one benchmark. Throws if no samples were recorded. + */ +export function samplesToResult(samples: number[]): RoundsResult { + const n = samples.length; + if (n === 0) { + throw new Error("[CodSpeed] Cannot compute statistics from empty samples"); + } + let totalTime = 0; + let min = Infinity; + let max = -Infinity; + for (const ms of samples) { + totalTime += ms; + if (ms < min) min = ms; + if (ms > max) max = ms; + } + const mean = totalTime / n; + const variance = + n > 1 + ? samples.reduce((sum, ms) => sum + (ms - mean) ** 2, 0) / (n - 1) + : 0; + return { + totalTime, + latency: { + min, + max, + mean, + sd: Math.sqrt(variance), + samples, + }, + }; +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..f6bf8500 --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,53 @@ +// Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. +const { + runForks, + runForksSync, + walltimeIsolation, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +const mode = walltimeIsolation(); +/** The count a first fork picks; the forks after it inherit it from the parent. */ +const CALIBRATED = Number(process.env.ISOLATION_FIXTURE_ITERATIONS ?? 1); + +function measure() { + console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + const inherited = mode.iterationsPerRound; + // Tells the test whether this fork picked the count or reused the first one. + console.log( + `${inherited === undefined ? "calibrated" : "reused"} ${inherited ?? CALIBRATED}`, + ); + const iterationsPerRound = + inherited !== undefined && + process.env.ISOLATION_FIXTURE_ACTION === "mismatch" + ? inherited + 1 // batches differently than the first fork + : (inherited ?? CALIBRATED); + writeForkSamples( + mode.samplesFile, + [Number(process.pid), 1], + iterationsPerRound, + ); +} + +async function main() { + if (mode.kind === "Fork") { + measure(); + return; + } + const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); + const result = + process.env.ISOLATION_FIXTURE_SYNC !== undefined + ? runForksSync("a", forks) + : await runForks("a", forks); + console.log(`samples ${result.samples.length}`); + console.log(`iterations ${result.iterationsPerRound}`); +} + +// A rejection ends the process with exit code 1. +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..fc5866c1 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,270 @@ +import { spawnSync } from "child_process"; +import { join } from "path"; +import { + FORK_ITERATIONS_PER_ROUND_ENV, + FORK_SAMPLES_FILE_ENV, + ISOLATED_BENCHMARK_ENV, + PROCESS_ISOLATION_ENV, + resolveWalltimeForks, + WALLTIME_FORKS_ENV, + walltimeIsolation, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (.+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +/** A fork tells whether it picked the iterations per round or reused them. */ +const CALIBRATED_LINE = /^calibrated (\d+)$/; +const REUSED_LINE = /^reused (\d+)$/; +const ITERATIONS_LINE = /^iterations (\d+)$/; +/** Passed to Node to check that forks keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; +/** Each fork of the fixture reports two samples. */ +const SAMPLES_PER_FORK = 2; + +const ENV_KEYS = [ + PROCESS_ISOLATION_ENV, + ISOLATED_BENCHMARK_ENV, + FORK_SAMPLES_FILE_ENV, + FORK_ITERATIONS_PER_ROUND_ENV, + WALLTIME_FORKS_ENV, +]; +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(env: Record): void { + for (const key of ENV_KEYS) { + delete process.env[key]; + } + for (const [key, value] of Object.entries(env)) { + if (value !== undefined) { + process.env[key] = value; + } + } +} + +afterEach(() => setEnv(savedEnv)); + +describe("resolveWalltimeForks", () => { + it("defaults to 3 forks", () => { + setEnv({}); + expect(resolveWalltimeForks()).toBe(3); + }); + + it("takes the option over the default", () => { + setEnv({}); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it("takes the env over the option", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "2" }); + expect(resolveWalltimeForks(5)).toBe(2); + }); + + it("ignores an empty env", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "" }); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects the env %p", (value) => { + setEnv({ [WALLTIME_FORKS_ENV]: value }); + expect(() => resolveWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); + + it.each([0, -1, 1.5, NaN])("rejects the option %p", (value) => { + setEnv({}); + expect(() => resolveWalltimeForks(value)).toThrow("forks"); + }); +}); + +describe("walltimeIsolation", () => { + it.each([ + [undefined, "Forks"], + ["1", "Forks"], + ["true", "Forks"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + setEnv({ [PROCESS_ISOLATION_ENV]: value }); + expect(walltimeIsolation().kind).toBe(kind); + }); + + it("runs everything in this process when isolation is off, whatever the fork count", () => { + setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); + expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + }); + + it("restricts a fork to its benchmark, whatever the isolation setting", () => { + setEnv({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + }); + }); + + it("tells a fork the iterations per round of the first fork", () => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: "4", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + iterationsPerRound: 4, + }); + }); + + it.each(["0", "-1", "1.5", "abc"])( + "rejects the iterations per round %p", + (value) => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: value, + }); + expect(() => walltimeIsolation()).toThrow(FORK_ITERATIONS_PER_ROUND_ENV); + }, + ); +}); + +interface FixtureOptions { + forks: number; + /** The iterations per round the first fork reports. Defaults to 1. */ + iterations?: number; + action?: "exit" | "no-samples" | "mismatch"; + sync?: boolean; +} + +function runFixture({ forks, iterations, action, sync }: FixtureOptions) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + ISOLATION_FIXTURE_FORKS: String(forks), + ISOLATION_FIXTURE_ITERATIONS: iterations?.toString(), + }; + for (const key of ENV_KEYS) { + delete env[key]; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const ran = lines + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + const samples = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + ran, + samples: samples ? Number(samples[1]) : undefined, + calibrated: counts(CALIBRATED_LINE, lines), + reused: counts(REUSED_LINE, lines), + iterations: counts(ITERATIONS_LINE, lines)[0], + }; +} + +function counts(pattern: RegExp, lines: string[]): number[] { + return lines + .map((line) => pattern.exec(line)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => Number(match[1])); +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("runForks ($name run)", ({ sync }) => { + it("runs every fork in a fresh process with the flags of its parent and pools their samples", () => { + const forks = 3; + const { status, ran, samples } = runFixture({ forks, sync }); + + expect(status).toBe(0); + expect(ran).toHaveLength(forks); + expect(new Set(ran.map((match) => match[1])).size).toBe(forks); + for (const match of ran) { + expect(JSON.parse(match[2])).toEqual([NODE_FLAG]); + } + expect(samples).toBe(forks * SAMPLES_PER_FORK); + }); + + it("reports one iteration per round when nothing batches", () => { + const { status, calibrated, reused, iterations } = runFixture({ + forks: 2, + sync, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([1]); + expect(reused).toEqual([1]); + expect(iterations).toBe(1); + }); + + it("runs the forks after the first one with its iterations per round", () => { + const forks = 3; + const { status, calibrated, reused, iterations } = runFixture({ + forks, + sync, + iterations: 4, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([4]); + expect(reused).toEqual(Array(forks - 1).fill(4)); + expect(iterations).toBe(4); + }); + + it("fails when a fork measures a different number of iterations per round", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + iterations: 2, + action: "mismatch", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("a different number of iterations per round"); + }); + + it("fails on a failing fork and runs no further fork", () => { + const { status, stderr, ran, samples } = runFixture({ + forks: 3, + sync, + action: "exit", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain(`failed with exit code ${CHILD_EXIT_CODE}`); + expect(ran).toHaveLength(1); + expect(samples).toBeUndefined(); + }); + + it("fails when a fork exits without reporting its samples", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + action: "no-samples", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not report its samples"); + }); +}); diff --git a/packages/core/tests/walltimeUtils.integ.test.ts b/packages/core/tests/walltimeUtils.integ.test.ts new file mode 100644 index 00000000..132b7559 --- /dev/null +++ b/packages/core/tests/walltimeUtils.integ.test.ts @@ -0,0 +1,31 @@ +import { samplesToResult } from "../src/walltime/utils"; + +describe("samplesToResult", () => { + it("computes pooled statistics", () => { + const samples = [2, 4, 4, 4, 5, 5, 7, 9]; + const { totalTime, latency } = samplesToResult(samples); + expect(totalTime).toBe(40); + expect(latency.min).toBe(2); + expect(latency.max).toBe(9); + expect(latency.mean).toBe(5); + // sample standard deviation: sqrt(32 / 7) + expect(latency.sd).toBeCloseTo(Math.sqrt(32 / 7), 10); + }); + + it("has zero deviation for a single sample", () => { + const { totalTime, latency } = samplesToResult([3]); + expect(totalTime).toBe(3); + expect(latency).toMatchObject({ min: 3, max: 3, mean: 3, sd: 0 }); + }); + + it("handles very large sample counts without overflowing the stack", () => { + const samples = new Array(500_000).fill(1); + expect(samplesToResult(samples).totalTime).toBe(500_000); + }); + + it("rejects empty samples", () => { + expect(() => samplesToResult([])).toThrow( + "[CodSpeed] Cannot compute statistics from empty samples", + ); + }); +}); diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..54c8d975 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,7 +14,9 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, { + kind: "SharedProcess", + }); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/batching.ts b/packages/tinybench-plugin/src/batching.ts new file mode 100644 index 00000000..b58804a2 --- /dev/null +++ b/packages/tinybench-plugin/src/batching.ts @@ -0,0 +1,107 @@ +import { msToNs } from "@codspeed/core"; +import { Bench, Fn, FnOptions, Task } from "tinybench"; +import { getBenchOptions } from "./benchOptions"; + +const MIN_ROUND_TIME_MS = 1; +export const MIN_ROUND_TIME_NS = msToNs(MIN_ROUND_TIME_MS); + +/** Round samples are batch averages, not individual-call latency distributions. */ +export function batchTask( + bench: Bench, + fn: Fn, + options: FnOptions | undefined, + inheritedIterations?: number, +) { + const state = { iterationsPerRound: inheritedIterations ?? 1 }; + if ( + options?.beforeEach || + options?.afterEach || + bench.concurrency === "task" + ) { + return { iterationsPerRound: 1, fn, options }; + } + + const { now } = getBenchOptions(bench); + let active = false; + let warmingUp = false; + let calibrated = inheritedIterations !== undefined; + let elapsed = 0; + let calls = 0; + + function finishCalibration() { + if (elapsed > 0) { + state.iterationsPerRound = Math.max( + 1, + Math.ceil((MIN_ROUND_TIME_MS * calls) / elapsed), + ); + } + calibrated = true; + } + function runBatch() { + const result = fn(); + if (result instanceof Promise) { + return (async () => { + await result; + for (let i = 1; i < state.iterationsPerRound; i++) await fn(); + })(); + } + for (let i = 1; i < state.iterationsPerRound; i++) fn(); + return result; + } + + function measure(): void | Promise { + const iterations = state.iterationsPerRound; + const start = now(); + const result = runBatch(); + function record() { + elapsed += now() - start; + calls += iterations; + if (!calibrated && elapsed >= MIN_ROUND_TIME_MS) finishCalibration(); + } + if (result instanceof Promise) return result.then(record); + record(); + } + + function prepare(mode?: "warmup" | "run"): void | Promise { + active = true; + warmingUp = mode === "warmup"; + if (warmingUp || inheritedIterations !== undefined) return; + if (calls > 0) { + finishCalibration(); + return; + } + // With warmup disabled, calibrate after setup/beforeAll, outside timed rounds. + // Wall-clock bounds also terminate calibration with a coarse/custom timer. + const deadline = performance.now() + MIN_ROUND_TIME_MS; + function calibrate(): void | Promise { + do { + const result = measure(); + if (result instanceof Promise) { + return result.then(() => { + if (performance.now() < deadline) return calibrate(); + finishCalibration(); + }); + } + } while (performance.now() < deadline); + finishCalibration(); + } + return calibrate(); + } + + return Object.assign(state, { + fn: () => { + // Tinybench may call the function at registration to detect promises. + if (!active) return fn(); + if (warmingUp) return measure(); + return runBatch(); + }, + options: { + ...options, + beforeAll(this: Task, mode?: "warmup" | "run") { + const result = options?.beforeAll?.call(this, mode); + if (result instanceof Promise) return result.then(() => prepare(mode)); + return prepare(mode); + }, + }, + }); +} diff --git a/packages/tinybench-plugin/src/benchOptions.ts b/packages/tinybench-plugin/src/benchOptions.ts index a1ffef42..a95b6ae0 100644 --- a/packages/tinybench-plugin/src/benchOptions.ts +++ b/packages/tinybench-plugin/src/benchOptions.ts @@ -4,6 +4,7 @@ import { Bench, Hook } from "tinybench"; // setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated // (at least with no-op defaults), so they are non-optional here. export interface ResolvedBenchOptions { + now: () => number; setup: Hook; teardown: Hook; warmup?: boolean; diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 298f97b9..fbba5cda 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,17 +7,29 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + walltimeIsolation, wrapWithRootFrameSync, + type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; +import { batchTask } from "./batching"; import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); -export function withCodSpeed(bench: Bench): Bench { +/** + * Walltime rounds batch sequential invocations toward a 1 ms target, estimated + * from warmup (or a short calibration when disabled). Forks share one batch size. + * Per-invocation hooks and task concurrency stay unbatched. CodSpeed reports + * per-invocation batch averages; total time is the sum of measured round durations. + */ +export function withCodSpeed( + bench: Bench, + options: WalltimeOptions = {}, +): Bench { const codspeedRunnerMode = getCodspeedRunnerMode(); if (codspeedRunnerMode === "disabled") { return bench; @@ -38,21 +50,30 @@ export function withCodSpeed(bench: Bench): Bench { } uri += `::${name}`; uriMap.set(name, uri); - taskDataMap.set(name, { fn, fnOpts: opts }); - // In walltime mode the task is driven by tinybench's own measured loop, so - // the root frame must be baked into the function tinybench stores rather - // than injected later (the function is an inaccessible private field on the - // task from tinybench v6 onwards). The sync wrapper is transparent to the - // function's return value, so it preserves tinybench's sync/async handling. - const registeredFn = - instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; - return rawAdd.bind(bench)(name, registeredFn, opts); + if (instrumentMode !== "walltime") { + taskDataMap.set(name, { fn, fnOpts: opts }); + return rawAdd.call(bench, name, fn, opts); + } + const isolation = walltimeIsolation(options.forks); + let inheritedIterations: number | undefined; + if (isolation.kind === "Fork") { + inheritedIterations = isolation.iterationsPerRound; + } + const batching = batchTask(bench, fn, opts, inheritedIterations); + taskDataMap.set(name, { fn, fnOpts: opts, batching }); + // Tinybench v6 stores the function in a private field, so wrap at registration. + return rawAdd.call( + bench, + name, + wrapWithRootFrameSync(batching.fn), + batching.options, + ); }; if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile); + setupCodspeedWalltimeBench(bench, rootCallingFile, options); } return bench; diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..1e10b4bf 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,21 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Walltime runners pass the isolation of this process; analysis shares it. */ + protected readonly isolationMode: IsolationMode; + + constructor( + bench: Bench, + rootCallingFile: string, + isolationMode: IsolationMode, + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolationMode = isolationMode; + } + + protected get isChild(): boolean { + return this.isolationMode.kind === "Fork"; } private setupBenchRun(): void { @@ -20,6 +31,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +60,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,10 +77,15 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { await this.runTaskAsync(task, uri); } @@ -68,14 +93,27 @@ export abstract class BaseBenchRunner { }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { this.runTaskSync(task, uri); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter( + ({ uri }) => + this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, + ); + } } diff --git a/packages/tinybench-plugin/src/taskData.ts b/packages/tinybench-plugin/src/taskData.ts index a5ecb2ab..58eedd82 100644 --- a/packages/tinybench-plugin/src/taskData.ts +++ b/packages/tinybench-plugin/src/taskData.ts @@ -7,6 +7,7 @@ import { Bench, Fn, FnOptions } from "tinybench"; export interface CapturedTaskData { fn: Fn; fnOpts?: FnOptions; + batching?: { iterationsPerRound: number }; } const taskDataMap = new WeakMap>(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 90e6b700..784152f8 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -6,19 +6,29 @@ import { mongoMeasurement, msToNs, msToS, + runForks, + runForksSync, + samplesToResult, + walltimeIsolation, + writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type RoundsResult, + type WalltimeOptions, } from "@codspeed/core"; -import { Bench, Task, TaskResult } from "tinybench"; +import { Bench, Task } from "tinybench"; +import { MIN_ROUND_TIME_NS } from "./batching"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, + options: WalltimeOptions, ): void { - const runner = new WalltimeBenchRunner(bench, rootCallingFile); + const isolationMode = walltimeIsolation(options.forks); + const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } @@ -91,6 +101,19 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const { samples, iterationsPerRound } = await runForks( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); + } + // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -103,8 +126,20 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - protected runTaskSync(task: Task, _uri: string): void { + protected runTaskSync(task: Task, uri: string): void { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const { samples, iterationsPerRound } = runForksSync( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); + } + if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -123,20 +158,40 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private registerCodspeedBenchmarkFromTask(task: Task): void { - const uri = this.getTaskUri(task); - if (!task.result) { - console.warn(` ⚠ No result data available for ${uri}`); + console.warn( + ` ⚠ No result data available for ${this.getTaskUri(task)}`, + ); return; } + const iterationsPerRound = + this.getTaskData(task).batching?.iterationsPerRound ?? 1; + // A fork hands its samples to the parent, which reports the benchmark. + if (this.isolationMode.kind === "Fork") { + writeForkSamples( + this.isolationMode.samplesFile, + [...task.result.latency.samples], + iterationsPerRound, + ); + return; + } + this.registerCodspeedBenchmark(task, task.result, iterationsPerRound); + } + private registerCodspeedBenchmark( + task: Task, + result: RoundsResult, + iterationsPerRound: number, + ): void { + const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( - task.result, + result, warmupIterations, + iterationsPerRound, ); this.codspeedBenchmarks.push({ @@ -145,7 +200,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { config: { max_rounds: opts.iterations ?? null, max_time_ns: opts.time ? msToNs(opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this + min_round_time_ns: iterationsPerRound > 1 ? MIN_ROUND_TIME_NS : null, warmup_time_ns: opts.warmup && opts.warmupTime ? msToNs(opts.warmupTime) : null, }, @@ -156,6 +211,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private finalizeWalltimeRun(isAsync: boolean): Task[] { + // A fork only hands its samples to the parent, which reports the results. + if (this.isChild) { + return this.bench.tasks; + } + // Write results to JSON file using core function if (this.codspeedBenchmarks.length > 0) { writeWalltimeResults(this.codspeedBenchmarks, isAsync); @@ -171,29 +231,32 @@ class WalltimeBenchRunner extends BaseBenchRunner { const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( - result: TaskResult, + result: RoundsResult, warmupIterations: number, + iterationsPerRound: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); + const sortedTimesNs = samples + .map((sample) => msToNs(sample) / iterationsPerRound) + .sort((a, b) => a - b); + const meanNs = msToNs(mean) / iterationsPerRound; + const stdevNs = msToNs(sd) / iterationsPerRound; const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); return { - min_ns: msToNs(min), - max_ns: msToNs(max), + min_ns: msToNs(min) / iterationsPerRound, + max_ns: msToNs(max) / iterationsPerRound, mean_ns: meanNs, stdev_ns: stdevNs, q1_ns, median_ns, q3_ns, total_time: msToS(result.totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration + iter_per_round: iterationsPerRound, rounds: sortedTimesNs.length, iqr_outlier_rounds, stdev_outlier_rounds, diff --git a/packages/tinybench-plugin/tests/batching.integ.test.ts b/packages/tinybench-plugin/tests/batching.integ.test.ts new file mode 100644 index 00000000..e1f4f307 --- /dev/null +++ b/packages/tinybench-plugin/tests/batching.integ.test.ts @@ -0,0 +1,218 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/batching.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const FIXTURE_SYNC_ENV = "BATCHING_FIXTURE_SYNC"; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const FORKS = 2; +/** Rounds the fixture's benchmarks measure per process once a round costs 1ms. */ +const ROUNDS_PER_FORK = 4; +/** The fixture's iteration cost: 0.01ms on the fake clock. */ +const BATCHED_ITERATION_NS = 10_000; +/** The lifecycle benchmark halves its iteration cost in the suite setup hook. */ +const LIFECYCLE_ITERATION_NS = 5_000; +/** The 1ms round target in units of the fixture's iteration cost. */ +const BATCHED_ITER_PER_ROUND = 100; +const LIFECYCLE_ITER_PER_ROUND = 200; +/** Allows for the fake clock's rounding, both ways, around the target count. */ +const ITERATIONS_TOLERANCE = 3; + +interface BenchmarkStats { + min_ns: number; + max_ns: number; + mean_ns: number; + rounds: number; + total_time: number; + iter_per_round: number; + warmup_iters: number; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `CODSPEED_WALLTIME_FORKS`. */ + forks?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forks, + sync, +}: FixtureOptions = {}): Promise> { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-batching-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forks !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forks); + } + if (sync) { + env[FIXTURE_SYNC_ENV] = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + expect(status, stdout).toBe(0); + + const results = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")) as { + benchmarks: { name: string; stats: BenchmarkStats }[]; + }; + for (const { name, stats } of data.benchmarks) { + results.set(name, stats); + } + } + return results; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +function getStats( + results: Map, + name: string, +): BenchmarkStats { + const stats = results.get(name); + if (!stats) { + throw new Error(`no results for ${name}`); + } + return stats; +} + +/** Every sample is one iteration, not one batch of iterations. */ +function expectPerInvocationStats( + stats: BenchmarkStats, + perInvocationNs: number, +): void { + expect(Math.round(stats.mean_ns)).toBe(perInvocationNs); + expect(Math.round(stats.min_ns)).toBe(perInvocationNs); + expect(Math.round(stats.max_ns)).toBe(perInvocationNs); + // The total stays the raw batched time: rounds x iterations x per-iteration. + expect(stats.total_time).toBeCloseTo( + (stats.rounds * stats.iter_per_round * stats.mean_ns) / 1e9, + 9, + ); +} + +function expectBatchedStats( + stats: BenchmarkStats, + { + rounds, + iterationsPerRound = BATCHED_ITER_PER_ROUND, + perInvocationNs = BATCHED_ITERATION_NS, + }: { + rounds: number; + iterationsPerRound?: number; + perInvocationNs?: number; + }, +): void { + expect(stats.iter_per_round).toBeGreaterThanOrEqual( + iterationsPerRound - ITERATIONS_TOLERANCE, + ); + expect(stats.iter_per_round).toBeLessThanOrEqual( + iterationsPerRound + ITERATIONS_TOLERANCE, + ); + expect(stats.rounds).toBe(rounds); + expectPerInvocationStats(stats, perInvocationNs); +} + +function expectHookedStats( + stats: BenchmarkStats, + { forks }: { forks: number }, +): void { + expect(stats.iter_per_round).toBe(1); + // Each 0.01ms call is timed on its own, so tinybench's 1ms budget is spent. + expect(stats.rounds).toBeGreaterThanOrEqual(98 * forks); + expect(stats.rounds).toBeLessThanOrEqual(102 * forks); + expectPerInvocationStats(stats, BATCHED_ITERATION_NS); +} + +describe("walltime batching", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches fast benchmarks and normalizes their stats in the shared process ($name run)", + async ({ sync }) => { + const stats = await runFixture({ shared: true, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: ROUNDS_PER_FORK, + }); + // The suite setup hook halves the iteration cost before calibration. + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + expect(getStats(stats, "lifecycle").warmup_iters).toBe(0); + expectHookedStats(getStats(stats, "hooked"), { forks: 1 }); + + if (!sync) { + // An overlapping async round would have failed the fixture run. + expectBatchedStats(getStats(stats, "async"), { + rounds: ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); + + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches and normalizes the pooled samples of several forks ($name run)", + async ({ sync }) => { + const stats = await runFixture({ forks: FORKS, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: FORKS * ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + // Batched and unbatched benchmarks pool very different sample counts. + expectHookedStats(getStats(stats, "hooked"), { forks: FORKS }); + + if (!sync) { + expectBatchedStats(getStats(stats, "async"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/tinybench-plugin/tests/fixtures/batching.mjs b/packages/tinybench-plugin/tests/fixtures/batching.mjs new file mode 100644 index 00000000..4e149b2a --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/batching.mjs @@ -0,0 +1,106 @@ +// Run by `batching.integ.test.ts`. A deterministic clock makes batching +// observable: every benchmark iteration costs exactly one tick, wherever +// tinybench or the plugin reads the clock. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +/** Milliseconds one benchmark iteration costs on the fake clock. */ +const ITERATION_MS = 0.01; +/** tinybench stops after this many rounds once a round costs 1ms. */ +const ROUNDS = 4; +const sync = process.env.BATCHING_FIXTURE_SYNC !== undefined; + +// Warmup enabled: the plugin learns the per-iteration cost from the warmup +// rounds and then batches ~1ms worth of iterations per measured round. +let tickA = 0; +const costA = ITERATION_MS; +const benchA = withCodSpeed( + new Bench({ iterations: ROUNDS, time: 1, warmupTime: 0, now: () => tickA }), +); + +benchA.add("batched", () => { + tickA += costA; +}); + +// A batched async benchmark must await every iteration of a round: overlapping +// iterations are a bug, and they fail the run instead of skewing the stats. +let inFlight = 0; +if (!sync) { + benchA.add("async", async () => { + if (inFlight++ > 0) { + throw new Error("async iterations overlapped"); + } + tickA += costA; + await Promise.resolve(); + inFlight--; + }); +} + +// beforeEach/afterEach are only correct when tinybench times each call alone. +let beforeEachCalls = 0; +let afterEachCalls = 0; +let hookedCalls = 0; +benchA.add( + "hooked", + () => { + if (beforeEachCalls > 0 && beforeEachCalls !== hookedCalls + 1) { + throw new Error("beforeEach did not run for this invocation"); + } + hookedCalls++; + tickA += costA; + }, + { + beforeAll: () => { + beforeEachCalls = 0; + afterEachCalls = 0; + hookedCalls = 0; + }, + beforeEach: () => { + beforeEachCalls++; + }, + afterEach: () => { + afterEachCalls++; + }, + afterAll: () => { + if (beforeEachCalls !== hookedCalls || afterEachCalls !== hookedCalls) { + throw new Error("afterEach did not run for every invocation"); + } + }, + }, +); + +// Warmup disabled: the plugin calibrates inside the run lifecycle, after the +// suite setup hook has run (which is what decides the iteration cost here). +let tickB = 0; +let costB = ITERATION_MS; +let lifecycleRuns = 0; +const benchB = withCodSpeed( + new Bench({ + iterations: ROUNDS, + time: 1, + warmup: false, + now: () => tickB, + setup: (_task, mode) => { + if (mode === "run") { + lifecycleRuns++; + costB = ITERATION_MS / 2; + } + }, + }), +); +benchB.add("lifecycle", () => { + tickB += costB; +}); + +if (sync) { + benchA.runSync(); + benchB.runSync(); +} else { + await benchA.run(); + await benchB.run(); +} + +// A fork that owns another benchmark's URI does not run this suite at all. +if (lifecycleRuns > 1) { + throw new Error(`ran the lifecycle ${lifecycleRuns} times`); +} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..6f59d933 --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,25 @@ +// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a benchmark. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +const options = process.env.ISOLATION_FIXTURE_FORKS_OPTION; +const bench = withCodSpeed( + new Bench({ iterations: 4, time: 1, warmupTime: 0 }), + options === undefined ? {} : { forks: Number(options) }, +); + +for (const name of ["first", "second"]) { + // tinybench calls the function once when it is added, in every process. + let calls = 0; + bench.add(name, () => { + if (++calls === 2) { + console.log(`ran ${process.pid} ${name}`); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..def38845 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,175 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const BENCHMARKS = ["first", "second"]; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +interface FixtureRun { + status: number | null; + /** PIDs that ran each benchmark. */ + pidsByName: Map>; + /** PID of the process the test started. */ + parentPid: number; + /** Rounds of each benchmark in the results file. */ + roundsByName: Map; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `forks` option of `withCodSpeed`. */ + forksOption?: number; + /** `CODSPEED_WALLTIME_FORKS`. */ + forksEnv?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forksOption, + forksEnv, + sync, +}: FixtureOptions = {}): Promise { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forksOption !== undefined) { + env.ISOLATION_FIXTURE_FORKS_OPTION = String(forksOption); + } + if (forksEnv !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forksEnv); + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of stdout.split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, parentPid: child.pid!, roundsByName }; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +/** PIDs that ran a benchmark besides the one we started. */ +function forkPids(run: FixtureRun, name: string): Set { + const pids = new Set(run.pidsByName.get(name)); + pids.delete(run.parentPid); + return pids; +} + +describe("walltime forks", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in 3 forks by default and pools their rounds ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + // Forks of different benchmarks are different processes too. + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(DEFAULT_FORKS * BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the withCodSpeed option", + async () => { + const run = await runFixture({ forksOption: 2 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(2); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the environment over the option", + async () => { + const run = await runFixture({ forksOption: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in the parent process when isolation is disabled", + async () => { + const run = await runFixture({ shared: true, forksEnv: 4 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.pidsByName.get(name)).toEqual(new Set([run.parentPid])); + } + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..0169373e --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,159 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +const RUN_TIMEOUT_MS = 180_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + isolation?: string; + forksEnv?: number; + pluginForks?: number; + firstForks?: number; + failing?: boolean; +} + +async function runFixture(options: FixtureOptions = {}) { + const logFile = join(tmpDir, "ran.log"); + writeFileSync(logFile, ""); + const profileFolder = join(tmpDir, "profile"); + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + ISOLATION_LOG_FILE: logFile, + [PROCESS_ISOLATION_ENV]: options.isolation, + [WALLTIME_FORKS_ENV]: options.forksEnv?.toString(), + ISOLATION_FIXTURE_PLUGIN_FORKS: options.pluginForks?.toString(), + ISOLATION_FIXTURE_FIRST_FORKS: options.firstForks?.toString(), + ISOLATION_FIXTURE_FAIL: options.failing ? "1" : undefined, + }; + + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + timeout: RUN_TIMEOUT_MS, + stdio: "ignore", + }); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of status === 0 ? readdirSync(resultsDir) : []) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, roundsByName }; +} + +describe("walltime forks", () => { + it( + "runs every benchmark in 3 forks by default and pools their rounds", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of ["first", "second"]) { + expect(run.pidsByName.get(name)?.size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the benchmark's option over the plugin's", + async () => { + const run = await runFixture({ firstForks: 2, pluginForks: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(2); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the environment over the benchmark's option", + async () => { + const run = await runFixture({ firstForks: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(1); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in one process when isolation is disabled", + async () => { + const run = await runFixture({ isolation: "false" }); + + expect(run.status).toBe(0); + const pids = new Set( + [...run.pidsByName.values()].flatMap((set) => [...set]), + ); + expect(pids.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "fails the run when a benchmark throws in its fork", + async () => { + const run = await runFixture({ failing: true, forksEnv: 1 }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..4c5a8178 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -5,11 +5,15 @@ import { mongoMeasurement, SetupInstrumentsRequestBody, SetupInstrumentsResponse, + type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; -import { resolveVitestBackend } from "./vitestBackend"; +import { + resolveVitestBackend, + scaleBenchTimeoutToForks, +} from "./vitestBackend"; // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; @@ -21,7 +25,8 @@ function resolveFile(name: string): string { return join(__dirname, `${name}.${fileExtension}`); } -export default function codspeedPlugin(): Plugin { +/** `options.forks`: see {@link WalltimeOptions}; isolation requires Vitest 5. */ +export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { @@ -37,19 +42,24 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { + const backend = resolveVitestBackend(); + backend.configureIsolation(userConfig, options); return { test: { pool: "forks", globalSetup: [resolveFile("globalSetup")], - ...resolveVitestBackend().getBenchmarkTestConfig( + ...backend.getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, ), }, }; }, + configureVitest({ project }) { + scaleBenchTimeoutToForks(project.config, options.forks); + }, }; } diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..77adabdb 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -51,6 +51,18 @@ export interface TinybenchOptions { iterations?: number; } +/** + * The `forks` option of the plugin, set in the main process for the workers, + * which inherit its env. Not {@link WALLTIME_FORKS_ENV}, which overrides the + * per-benchmark option instead of yielding to it. + */ +export const PLUGIN_FORKS_ENV = "CODSPEED_VITEST_PLUGIN_FORKS"; + +export function getPluginForks(): number | undefined { + const value = process.env[PLUGIN_FORKS_ENV]; + return value === undefined ? undefined : Number(value); +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..7cdf5cde 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -4,11 +4,17 @@ import { InstrumentHooks, optimizeFunction, optimizeFunctionSync, + resolveWalltimeForks, + runForks, + samplesToResult, setupCore, teardownCore, + walltimeIsolation, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, type Benchmark, + type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; import path from "path"; @@ -20,6 +26,7 @@ import type { BenchRunOptions, } from "vitest"; import { + getPluginForks, installInstrumentHooks, tinybenchTaskToBenchmark, writeAndLogWalltimeResults, @@ -221,6 +228,15 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + const mode = walltimeIsolation(); + if (mode.kind === "Forks") { + return runForkParent(registrations, options, getUri); + } + // A fork measures only its benchmark; Vitest still gets every result. + const measured = registrations.filter( + ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), + ); + const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -229,7 +245,7 @@ async function runWalltime( retainSamples: true, }); - for (const { name, fn, fnOpts } of registrations) { + for (const { name, fn, fnOpts } of measured) { // tinybench v6 keeps `fn` private, so the root frame is baked in here bench.add( name, @@ -242,9 +258,56 @@ async function runWalltime( const tasks = await bench.run(); throwOnErroredTasks(tasks); - collectWalltimeResults(bench, getUri); + if (mode.kind === "Fork") { + for (const task of tasks) { + writeForkSamples(mode.samplesFile, [ + ...(task.result?.latency.samples ?? []), + ]); + } + } else { + collectWalltimeResults(bench, getUri); + } - return tasks.map(tinybenchTaskToBenchResult); + const results = new Map( + tasks.map((task) => [task.name, tinybenchTaskToBenchResult(task)]), + ); + return registrations.map( + ({ name }) => results.get(name) ?? analysisResult(name), + ); +} + +/** + * Measures every benchmark in fresh processes and reports their pooled + * rounds. Vitest gets zeroed results, like in analysis mode. + */ +async function runForkParent( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, +): Promise { + const { Bench } = await importHostTinybench(); + // Never run: it resolves tinybench's defaults for the reported config. + const bench = new Bench(options); + const benchmarks: Benchmark[] = []; + for (const { name, fnOpts } of registrations) { + const uri = getUri(name); + const forks = resolveWalltimeForks( + (fnOpts as WalltimeOptions | undefined)?.forks ?? + (options as WalltimeOptions | undefined)?.forks ?? + getPluginForks(), + ); + const { samples } = await runForks(uri, forks); + const benchmark = tinybenchTaskToBenchmark( + { name, result: samplesToResult(samples) }, + uri, + resolvedOptions(bench), + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + writeAndLogWalltimeResults(benchmarks); + return registrations.map(({ name }) => analysisResult(name)); } /** Fail the test like Vitest's own provider does, instead of returning empty results. */ @@ -260,11 +323,9 @@ function throwOnErroredTasks(tasks: TinybenchTask[]): void { } } -function collectWalltimeResults( - bench: TinybenchWithTasks, - getUri: (name: string) => string, -): void { - const options: TinybenchOptions = { +/** The options the bench runs with, as reported to CodSpeed. */ +function resolvedOptions(bench: TinybenchWithTasks): TinybenchOptions { + return { time: bench.time, iterations: bench.iterations, // A bench with warmup turned off keeps its warmup timings, so they'd @@ -272,6 +333,13 @@ function collectWalltimeResults( warmupTime: bench.warmup ? bench.warmupTime : 0, warmupIterations: bench.warmup ? bench.warmupIterations : 0, }; +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options = resolvedOptions(bench); const benchmarks: Benchmark[] = []; for (const task of bench.tasks) { if (task.result?.state !== "completed") continue; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 46c171d7..fd8a6aee 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,11 +1,22 @@ -import { getInstrumentMode } from "@codspeed/core"; +import { + getGitDir, + getInstrumentMode, + recordIsolationCommand, + walltimeIsolation, + type WalltimeOptions, +} from "@codspeed/core"; import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; -import { version as vitestVersion } from "vitest/node"; +import { type ResolvedConfig, version as vitestVersion } from "vitest/node"; +import { PLUGIN_FORKS_ENV } from "./instrument"; /** The parts of the Vitest integration that depend on the installed major. */ export interface VitestBackend { isActive(mode: string): boolean; + configureIsolation( + userConfig: ViteUserConfig, + options: WalltimeOptions, + ): void; getBenchmarkTestConfig( v8Flags: string[], @@ -34,6 +45,29 @@ class V5Backend implements VitestBackend { return getInstrumentMode() !== "disabled"; } + configureIsolation( + userConfig: ViteUserConfig, + options: WalltimeOptions, + ): void { + if (getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(options.forks); + switch (isolationMode.kind) { + case "SharedProcess": + return; + case "Forks": + // Workers cannot re-launch Vitest from their own argv. + recordIsolationCommand(); + if (options.forks !== undefined) { + process.env[PLUGIN_FORKS_ENV] = String(options.forks); + } + return; + case "Fork": + restrictToBenchmarkFile(userConfig, isolationMode.uri); + return; + } + } + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, @@ -53,6 +87,38 @@ class V5Backend implements VitestBackend { } } +function restrictToBenchmarkFile( + userConfig: ViteUserConfig, + uri: string, +): void { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; +} + +export function scaleBenchTimeoutToForks( + config: ResolvedConfig, + forksOption: number | undefined, +): void { + if (!config.benchmark.enabled || getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(forksOption); + if (isolationMode.kind !== "Forks") return; + + // All forks run within the registering test. Per-benchmark forks aren't + // known here; scale Vitest's resolved timeout by the integration's budget. + config.testTimeout *= isolationMode.forks; +} + /** * Vitest 3/4: `vitest bench` runs in the `"benchmark"` Vite mode, and each * instrument mode has its own `test.runner` (`legacy/analysis`, `legacy/walltime`). @@ -64,6 +130,9 @@ class LegacyBackend implements VitestBackend { return mode === "benchmark"; } + // Vitest 3/4 do not support process isolation. + configureIsolation(): void {} + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..be799958 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,38 @@ +import { appendFileSync } from "fs"; +import { describe, test, type BenchRegistration } from "vitest"; + +// Vitest swallows what workers print, so the PIDs that run each benchmark go +// to a file. `beforeAll` only runs when tinybench measures the benchmark. +function logPid(name: string) { + return () => + appendFileSync( + process.env.ISOLATION_LOG_FILE!, + `ran ${process.pid} ${name}\n`, + ); +} + +const firstForks = process.env.ISOLATION_FIXTURE_FIRST_FORKS; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks: BenchRegistration[] = [ + bench( + "first", + { + beforeAll: logPid("first"), + ...(firstForks && { forks: Number(firstForks) }), + }, + () => {}, + ), + bench("second", { beforeAll: logPid("second") }, () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks, { iterations: 4, time: 1 }); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..387bb2bf --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +const pluginForks = process.env.ISOLATION_FIXTURE_PLUGIN_FORKS; + +export default defineConfig({ + plugins: [codspeedPlugin(pluginForks ? { forks: Number(pluginForks) } : {})], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + }, +}); diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..e874589b 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -10,5 +10,8 @@ export default defineConfig({ test: { exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], mockReset: true, + benchmark: { + exclude: ["**/node_modules/**", "**/tests/**/*", "**/.rollup.cache/**/*"], + }, }, });