diff --git a/.github/workflows/fuzz-image.yml b/.github/workflows/fuzz-image.yml new file mode 100644 index 0000000..303eb42 --- /dev/null +++ b/.github/workflows/fuzz-image.yml @@ -0,0 +1,175 @@ +name: Fuzz image + +# Builds Dockerfile.fuzz -- the fast-check round-trip properties in tests/fuzz -- +# and publishes it to GHCR for the ChainSafe fuzz platform. +# +# When an image is built: +# - manual run on main (Actions tab) -> build, test, publish `main` +# - manual run on any other branch -> build, test, publish `branch-` +# - pull request touching code the targets exercise -> build and test only +# Publishing is deliberately manual: the fuzz platform follows the `main` tag +# and restarts the campaign on every publish, so run this once enough good +# changes have landed, not on every commit. Pull requests still build and test +# so a broken Dockerfile.fuzz is caught before the next publish. +# +# Two jobs: `build and test` runs for every event with read-only permissions; +# `publish` runs only on a manual run, holds the only write token, and pushes +# the exact image the build job tested. +# +# Tags: `main` is what the fuzz platform runs, and only a run on main moves it. +# A run on another branch publishes `branch-` instead, for trying a branch +# on the platform before it merges. `sha-` names each build's source +# commit, also recorded as the org.opencontainers.image.revision label. +# Old versions beyond the newest 20 are deleted. + +on: + pull_request: + paths: + - "src/**" + - "tests/**" + - "build.zig" + - "build.zig.zon" + - "package.json" + - "pnpm-lock.yaml" + - "Dockerfile.fuzz" + - "Dockerfile.fuzz.dockerignore" + - ".github/workflows/fuzz-image.yml" + workflow_dispatch: + +concurrency: + group: fuzz-image-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: ghcr.io/chainsafe/zapi-fuzz + +jobs: + build: + name: build and test + runs-on: ubuntu-latest + # Read-only: this job runs on pull requests and never pushes. + permissions: + contents: read + steps: + - name: Checkout Repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: Dockerfile.fuzz + platforms: linux/amd64 + build-args: REVISION=${{ github.sha }} + tags: zapi-fuzz:ci + load: true + # Off so the image is a single manifest: one package version per + # publish, which the version cleanup below relies on. + provenance: false + sbom: false + cache-from: type=gha,scope=fuzz-image + cache-to: type=gha,scope=fuzz-image,mode=max + + # The same restrictions the fuzz platform applies: a non-root uid, read-only + # root filesystem, no capabilities, no network, an init process. + - name: A campaign starts and reports status under the platform's restrictions + run: | + mkdir -p /tmp/fz/state /tmp/fz/artifacts /tmp/fz/failures /tmp/fz/metrics /tmp/fz/local + sudo chown -R 60001:60001 /tmp/fz + docker run -d --name smoke --user 60001:60001 --read-only --cap-drop all \ + --security-opt no-new-privileges --network none --init --pids-limit 4096 \ + -e FUZZ_PROJECT=zapi -e FUZZ_TARGET=round-trip \ + -e FUZZ_STATE=/fuzz/state -e FUZZ_ARTIFACTS=/fuzz/artifacts \ + -e FUZZ_FAILURES=/fuzz/failures -e FUZZ_METRICS=/fuzz/metrics \ + -e TMPDIR=/local \ + -v /tmp/fz/state:/fuzz/state -v /tmp/fz/artifacts:/fuzz/artifacts \ + -v /tmp/fz/failures:/fuzz/failures -v /tmp/fz/metrics:/fuzz/metrics \ + -v /tmp/fz/local:/local \ + zapi-fuzz:ci + sleep 30 + docker logs smoke + test "$(docker inspect -f '{{.State.Running}}' smoke)" = true + ! docker logs smoke 2>&1 | grep -q '^FATAL' + cat /tmp/fz/metrics/status + awk '$1 == "execs_total" && $2 > 0 { ok = 1 } END { exit !ok }' /tmp/fz/metrics/status + # Every property passes on a clean build; a failure here is a real bug. + awk '$1 == "crashes_total" && $2 == 0 { ok = 1 } END { exit !ok }' /tmp/fz/metrics/status + docker stop -t 60 smoke + test "$(docker inspect -f '{{.State.ExitCode}}' smoke)" = 0 + + # Hand the tested image itself to the publish job, so what is published is + # exactly what passed above -- no second build. + - name: Save the tested image + if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/heads/') + run: docker save zapi-fuzz:ci -o /tmp/zapi-fuzz.tar + + - name: Upload the tested image + if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/heads/') + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: zapi-fuzz-image + path: /tmp/zapi-fuzz.tar + retention-days: 1 + + publish: + name: publish + needs: build + # Manual runs on a branch only; see the header comment. + if: github.event_name == 'workflow_dispatch' && startsWith(github.ref, 'refs/heads/') + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download the tested image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: zapi-fuzz-image + path: /tmp + + - name: Log in to GHCR + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push + id: push + run: | + # main moves `main`; any other branch gets its own tag, sanitized to + # Docker's tag charset and length. + if [ "$GITHUB_REF_NAME" = main ]; then + tag=main + else + tag="branch-$(printf '%s' "$GITHUB_REF_NAME" | tr 'A-Z' 'a-z' | tr -c 'a-z0-9._-' '-' | cut -c1-100)" + fi + docker load -i /tmp/zapi-fuzz.tar + docker tag zapi-fuzz:ci "${IMAGE}:sha-${GITHUB_SHA::7}" + docker tag zapi-fuzz:ci "${IMAGE}:${tag}" + docker push "${IMAGE}:sha-${GITHUB_SHA::7}" + docker push "${IMAGE}:${tag}" + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + echo "digest=$(docker image inspect -f '{{index .RepoDigests 0}}' "${IMAGE}:${tag}" | sed 's/.*@//')" >> "$GITHUB_OUTPUT" + + - name: Summary + run: | + { + echo "### Fuzz image published" + echo + echo "\`${IMAGE}:${{ steps.push.outputs.tag }}\` now points at \`${{ steps.push.outputs.digest }}\`," + echo "built from \`${GITHUB_SHA}\`. Fuzz platform targets following this tag" + echo "restart on the new image; their counters carry over." + } >> "$GITHUB_STEP_SUMMARY" + + # Without provenance or SBOM attestations, each publish is exactly one + # package version, so this keeps the newest 20 builds. + - name: Delete old image versions + uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0 + with: + package-name: zapi-fuzz + package-type: container + min-versions-to-keep: 20 diff --git a/Dockerfile.fuzz b/Dockerfile.fuzz new file mode 100644 index 0000000..b34de5e --- /dev/null +++ b/Dockerfile.fuzz @@ -0,0 +1,63 @@ +# Fuzz image for the ChainSafe fuzz platform: the fast-check round-trip properties +# in tests/fuzz/, driven by tests/fuzz/platform/run-fuzz.ts. The platform provides +# the FUZZ_* environment and the /fuzz volume. See tests/fuzz/README.md. +# +# docker build -f Dockerfile.fuzz -t zapi-fuzz . + +# --- build: Zig compiles the test_fuzz_numeric.node addon +FROM node:24-bookworm-slim AS build + +ARG ZIG_VERSION=0.16.0 +# From https://ziglang.org/download/index.json -- update together with ZIG_VERSION. +ARG ZIG_SHA256_X86_64=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 +ARG ZIG_SHA256_AARCH64=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 +ARG TARGETARCH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN case "${TARGETARCH}" in \ + amd64) arch=x86_64; sum="${ZIG_SHA256_X86_64}" ;; \ + arm64) arch=aarch64; sum="${ZIG_SHA256_AARCH64}" ;; \ + *) echo "unsupported TARGETARCH '${TARGETARCH}'" >&2; exit 1 ;; \ + esac \ + && curl -fsSL -o /tmp/zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-${arch}-linux-${ZIG_VERSION}.tar.xz" \ + && echo "${sum} /tmp/zig.tar.xz" | sha256sum -c - \ + && mkdir /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 \ + && rm /tmp/zig.tar.xz +ENV PATH=/opt/zig:$PATH + +WORKDIR /src +RUN corepack enable +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile +COPY . . + +# ReleaseSafe keeps Zig's safety checks (bounds, overflow), so those bugs panic +# instead of passing silently. -Dcpu=baseline: otherwise the addon targets the +# build runner's CPU and dies with SIGILL on a host without its extensions. +RUN --mount=type=cache,target=/root/.cache/zig \ + cd tests && zig build --fork=.. --prefix ../zig-out -Doptimize=ReleaseSafe -Dcpu=baseline + +# --- runtime: node, the addon, the tests and their node_modules only +FROM node:24-bookworm-slim + +WORKDIR /opt/zapi +COPY --from=build /src/node_modules ./node_modules +COPY --from=build /src/zig-out/lib/test_fuzz_numeric.node ./zig-out/lib/ +COPY --from=build /src/package.json ./ +COPY --from=build /src/tests/fuzz ./tests/fuzz +COPY --from=build /src/tests/utils ./tests/utils +# The platform runs the container as an arbitrary non-root uid. +RUN chmod -R a+rX /opt/zapi + +ARG REVISION=unknown +ENV FUZZ_REVISION=${REVISION} +LABEL org.opencontainers.image.source="https://github.com/ChainSafe/zapi" \ + org.opencontainers.image.title="zapi-fuzz" \ + org.opencontainers.image.description="fast-check round-trip fuzz targets for zapi" \ + org.opencontainers.image.revision="${REVISION}" + +ENTRYPOINT ["node", "/opt/zapi/tests/fuzz/platform/run-fuzz.ts"] diff --git a/Dockerfile.fuzz.dockerignore b/Dockerfile.fuzz.dockerignore new file mode 100644 index 0000000..7f5b231 --- /dev/null +++ b/Dockerfile.fuzz.dockerignore @@ -0,0 +1,7 @@ +# Build context for Dockerfile.fuzz (BuildKit reads .dockerignore). +.git +**/.zig-cache +**/zig-out +**/zig-pkg +**/node_modules +lib diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md index 67cac6b..b2dbd45 100644 --- a/tests/fuzz/README.md +++ b/tests/fuzz/README.md @@ -49,6 +49,28 @@ Once fixed, **persist the counterexample** so it runs forever: For numbers, use a JSON number directly. For `Uint8Array`, use a JSON array of byte values, e.g. `[0, 255, 1]`. NaN and ±Infinity aren't representable in JSON and are not currently supported as persisted seeds — those values are already in `edgeNumbers` and exercised on every run, so persisting them as regression cases adds nothing. If a future fuzz target needs persisted special-number seeds, extend `loadSeeds` and the relevant `revive*` helper at that time. 3. Commit with the fix. +## Continuous fuzzing (ChainSafe fuzz platform) + +`Dockerfile.fuzz` packages these properties for the [fuzz platform](https://github.com/ChainSafe/lodestar-ansible-development/tree/main/playbooks/fuzz). `.github/workflows/fuzz-image.yml` builds and smoke-tests it on PRs. Publishing is manual: run the workflow from the Actions tab on `main` to move `ghcr.io/chainsafe/zapi-fuzz:main` (on another branch it publishes `branch-`). The platform switches the campaign to each new image. + +The entrypoint, `tests/fuzz/platform/run-fuzz.ts`, re-runs every `*.fuzz.test.ts` in a loop (`FUZZ_RUNS` cases per property, default 100 000, fresh random seed each time) until `SIGTERM`: + +- a failing property is saved once per test + shrunk counterexample to `$FUZZ_FAILURES/failure-.json` (with seed and path for replay) and logged as `NEW_FAILURE`; fuzzing continues; +- `$FUZZ_METRICS/status` is rewritten after every iteration (`execs_total` counts property cases; no `coverage`/`corpus_entries`, since fast-check is not coverage-guided); +- counters persist in `$FUZZ_STATE/stats.json`; +- a suite that fails to load, or a run with no tests, exits non-zero (an infrastructure failure). + +Run it locally as the platform does: + +```bash +docker build -f Dockerfile.fuzz -t zapi-fuzz . +mkdir -p /tmp/fz && chmod 777 /tmp/fz +docker run --rm --init --read-only --network none --cap-drop all --user 60001:60001 \ + -e TMPDIR=/fuzz/local -v /tmp/fz:/fuzz zapi-fuzz +``` + +To reproduce a saved failure, take its `seed` and `path` into the `it.only` shown above. + ## Adding a new fuzz target 1. Add the round-trip export to `mod.zig`. diff --git a/tests/fuzz/platform/run-fuzz.ts b/tests/fuzz/platform/run-fuzz.ts new file mode 100644 index 0000000..175ebd4 --- /dev/null +++ b/tests/fuzz/platform/run-fuzz.ts @@ -0,0 +1,177 @@ +// Entrypoint of the zapi fuzz image (Dockerfile.fuzz) on the ChainSafe fuzz platform. +// +// Re-runs the fast-check properties (tests/fuzz/*.fuzz.test.ts) in a loop, each +// iteration with a fresh random seed, and fulfils the platform's container contract: +// - runs until SIGTERM; any other exit is a failure +// - a property failure is a finding, not a crash of this process: it is saved, +// deduplicated by test name + shrunk counterexample, to $FUZZ_FAILURES and +// announced with a NEW_FAILURE line +// - $FUZZ_METRICS/status is rewritten after every iteration (seconds apart) +// - cumulative counters live in $FUZZ_STATE, so they survive restarts +// - an infrastructure failure (addon fails to load, no tests ran) exits non-zero +// +// fast-check is not coverage-guided, so there is no corpus: `coverage` and +// `corpus_entries` are not reported, and `execs_total` counts property cases. +import {spawn} from "node:child_process"; +import {createHash} from "node:crypto"; +import * as fs from "node:fs"; +import * as path from "node:path"; + +const env = (name: string, fallback: string): string => process.env[name] || fallback; + +const ROOT = path.resolve(import.meta.dirname, "../../.."); +const STATE = env("FUZZ_STATE", "/fuzz/state"); +const FAILURES = env("FUZZ_FAILURES", "/fuzz/failures"); +const METRICS = env("FUZZ_METRICS", "/fuzz/metrics"); +const TMP = env("TMPDIR", "/tmp"); +// Cases per property per iteration. Keep an iteration well under a minute: the +// status file is the platform's liveness signal and is written between iterations. +const RUNS = Number(env("FUZZ_RUNS", "100000")); +const STATS_FILE = path.join(STATE, "stats.json"); +const REPORT_FILE = path.join(TMP, "vitest-report.json"); + +type Stats = {execsTotal: number; iterations: number; lastFindTimestamp: number}; +type AssertionResult = {fullName: string; status: string; failureMessages: string[]}; +type Report = { + numTotalTests: number; + numFailedTests: number; + testResults: {name: string; message?: string; assertionResults: AssertionResult[]}[]; +}; + +for (const dir of [STATE, FAILURES, METRICS, TMP]) fs.mkdirSync(dir, {recursive: true}); + +function writeAtomic(file: string, content: string): void { + fs.writeFileSync(`${file}.tmp`, content); + fs.renameSync(`${file}.tmp`, file); +} + +function loadStats(): Stats { + try { + return JSON.parse(fs.readFileSync(STATS_FILE, "utf8")) as Stats; + } catch { + return {execsTotal: 0, iterations: 0, lastFindTimestamp: 0}; + } +} + +const stats = loadStats(); +let execsPerSecond = 0; + +function writeStatus(): void { + const crashes = fs.readdirSync(FAILURES).filter((f) => f.startsWith("failure-")).length; + const lines = [ + `execs_total ${stats.execsTotal}`, + `execs_per_second ${execsPerSecond.toFixed(1)}`, + `crashes_total ${crashes}`, + "hangs_total 0", + ]; + if (stats.lastFindTimestamp > 0) lines.push(`last_find_timestamp_seconds ${stats.lastFindTimestamp}`); + writeAtomic(path.join(METRICS, "status"), `${lines.join("\n")}\n`); + writeAtomic(STATS_FILE, JSON.stringify(stats)); +} + +function saveFailure(test: AssertionResult): void { + const message = test.failureMessages.join("\n"); + // The seed differs every iteration; the shrunk counterexample does not. + const counterexample = /Counterexample: (.*)/.exec(message)?.[1] ?? message.split("\n")[0]; + const id = createHash("sha256").update(`${test.fullName}\0${counterexample}`).digest("hex").slice(0, 16); + const file = path.join(FAILURES, `failure-${id}.json`); + if (fs.existsSync(file)) return; + const seed = /seed: (-?\d+), path: "([^"]*)"/.exec(message); + const record = { + counterexample, + message, + path: seed?.[2] ?? null, + revision: env("FUZZ_REVISION", "unknown"), + seed: seed ? Number(seed[1]) : null, + test: test.fullName, + timestamp: new Date().toISOString(), + }; + writeAtomic(file, `${JSON.stringify(record, null, 2)}\n`); + stats.lastFindTimestamp = Math.floor(Date.now() / 1000); + console.log(`NEW_FAILURE file=failure-${id}.json test="${test.fullName}" counterexample=${counterexample}`); +} + +function fatal(reason: string, output: string): never { + console.log(`FATAL ${reason}; last output:\n${output.split("\n").slice(-30).join("\n")}`); + writeStatus(); + process.exit(1); +} + +let stopping = false; +let child: ReturnType | undefined; +for (const signal of ["SIGTERM", "SIGINT"] as const) { + process.on(signal, () => { + console.log(`${signal}: stopping`); + stopping = true; + child?.kill("SIGTERM"); + }); +} + +function runIteration(): Promise<{code: number | null; output: string}> { + fs.rmSync(REPORT_FILE, {force: true}); + return new Promise((resolve) => { + child = spawn( + process.execPath, + [ + path.join(ROOT, "node_modules/vitest/vitest.mjs"), + "run", + "tests/fuzz/", + "--exclude=**/oracle.test.ts", + "--reporter=json", + `--outputFile=${REPORT_FILE}`, + ], + {cwd: ROOT, env: {...process.env, FUZZ_RUNS: String(RUNS)}, stdio: ["ignore", "pipe", "pipe"]} + ); + let output = ""; + const collect = (chunk: Buffer): void => { + output = (output + chunk.toString()).slice(-64 * 1024); + }; + child.stdout?.on("data", collect); + child.stderr?.on("data", collect); + child.on("close", (code) => resolve({code, output})); + }); +} + +console.log( + `START project=${env("FUZZ_PROJECT", "zapi")} target=${env("FUZZ_TARGET", "round-trip")} runs=${RUNS} resumed_execs=${stats.execsTotal}` +); +writeStatus(); + +while (!stopping) { + const started = Date.now(); + const {code, output} = await runIteration(); + if (stopping) break; + + let report: Report; + try { + report = JSON.parse(fs.readFileSync(REPORT_FILE, "utf8")) as Report; + } catch { + fatal(`vitest exited ${code} without a report`, output); + } + // A suite that fails to load (e.g. the addon is missing) has a message but no tests. + const suiteErrors = report.testResults + .filter((f) => f.message) + .map((f) => `${f.name}: ${f.message}`) + .join("\n"); + if (suiteErrors) fatal("a test suite failed to load", suiteErrors); + if (report.numTotalTests === 0) fatal("vitest ran no tests", output); + if (code !== 0 && report.numFailedTests === 0) fatal(`vitest exited ${code} with no failed test`, output); + + for (const file of report.testResults) { + for (const test of file.assertionResults) if (test.status === "failed") saveFailure(test); + } + + const execs = report.numTotalTests * RUNS; + stats.execsTotal += execs; + stats.iterations += 1; + execsPerSecond = execs / ((Date.now() - started) / 1000); + writeStatus(); + if (stats.iterations % 20 === 0) { + console.log( + `STATS iterations=${stats.iterations} execs_total=${stats.execsTotal} execs_per_second=${execsPerSecond.toFixed(0)}` + ); + } +} + +writeStatus(); +process.exit(0);