diff --git a/.github/ISSUE_TEMPLATE/model_request.md b/.github/ISSUE_TEMPLATE/model_request.md index b7c8a51a..9c0e74d8 100644 --- a/.github/ISSUE_TEMPLATE/model_request.md +++ b/.github/ISSUE_TEMPLATE/model_request.md @@ -16,5 +16,5 @@ Where the weights live and under what license. Benchmark numbers, or what it does that the supported archs do not. -CONTRIBUTING.md has the six-site walkthrough if you want to send the port -yourself. +docs/COMPILED.md shows how to serve a model compiled by ggml-compiler without +a port; docs/ARCHITECTURE.md covers a hand-written one. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5743416a..ed391763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,15 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke `scripts/convert_cosmos3_policy_to_gguf.py` converts the checkpoint. Agrees with lerobot's policy in bf16 to 2e-3 to 2e-2 on the action chunk; the client preset `cosmos3_policy` and `eval/run_libero.sh -m cosmos3_policy` serve it. CPU and CUDA only. +- **Models compiled by ggml-compiler.** A GGUF written by `ggml-compiler + compile` that carries a `vla` serving manifest loads as the `compiled` arch + and serves through `vla-server`, `vla-cli`, `vla-bench` and `libvla` with no + model-specific code: the manifest names each program input's role (camera, + instruction, state, noise) and preprocessing, and the actions output with its + statistics. ggml-compiler's runtime is built in-tree against vla.cpp's own + ggml (`VLA_COMPILED`, default ON), so compiled models run on every backend + vla.cpp builds. First version: fixed shapes, one entry point, token ids from + the client. [docs/COMPILED.md](docs/COMPILED.md). - **Hexagon NPU on the Qualcomm Dragonwing IQ-10 (Linux).** `scripts/build_linux_hexagon.sh` cross-builds vla.cpp with `GGML_HEXAGON=ON` in Qualcomm's toolchain container, `vla-server` included. On the IQ-10 (v81, diff --git a/CMakeLists.txt b/CMakeLists.txt index 9310d400..b45e7def 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,25 @@ function(vla_exclude_fetched_targets dir) endfunction() vla_exclude_fetched_targets(${llama_SOURCE_DIR}) +# Models compiled by ggml-compiler run from their GGUF through its runtime, +# built here against this tree's ggml so one process holds one ggml (its +# backends included). The runtime is pinned to a ggml-compiler commit and +# checks that our llama.cpp tag is the one it was tested with. +option(VLA_COMPILED "Serve GGUFs written by ggml-compiler (fetches its runtime)" ON) +if(VLA_COMPILED) + set(VLA_GGML_COMPILER_TAG "f72f90189dff555fca70b3a43c1a6d5aa89dac16" CACHE STRING "ggml-compiler git ref providing the runtime") + set(GGML_COMPILER_EXTERNAL_GGML ON CACHE BOOL "" FORCE) + set(GGML_COMPILER_GGML_SOURCE_DIR ${llama_SOURCE_DIR} CACHE PATH "" FORCE) + set(GGML_COMPILER_EXTERNAL_GGML_TAG ${VLA_LLAMA_TAG} CACHE STRING "" FORCE) + set(GGML_COMPILER_BUILD_TOOLS OFF CACHE BOOL "" FORCE) + FetchContent_Declare(ggml_compiler + GIT_REPOSITORY https://github.com/khanhnd61-vr/ggml-compiler + GIT_TAG ${VLA_GGML_COMPILER_TAG} + SOURCE_SUBDIR runtime + ) + FetchContent_MakeAvailable(ggml_compiler) +endif() + # SentencePiece, and through it a protobuf, is only needed to tokenize --text with # a tokenizer stored in the GGUF. Optional so a build without it has no protobuf. option(VLA_SPM "Link SentencePiece for in-GGUF tokenizers (needs system protobuf)" ON) @@ -269,6 +288,11 @@ target_include_directories(vla_core ) # The VLA archs call no llama_* API; only vlm_core needs llama. target_link_libraries(vla_core PUBLIC ggml) +if(VLA_COMPILED) + target_sources(vla_core PRIVATE src/models/compiled.cpp) + target_link_libraries(vla_core PUBLIC ggml_compiler_rt) + target_compile_definitions(vla_core PUBLIC VLA_COMPILED) +endif() # The FoldQuant CPU reference must match the CUDA and SYCL kernels bit for bit, # so no FMA contraction on either side (aarch64 GCC contracts by default; under # icpx, after the project-wide -fp-model=precise, which would allow it). @@ -630,6 +654,9 @@ install(FILES LICENSE.md ${llama_SOURCE_DIR}/licenses/LICENSE-jsonhpp install(FILES ${llama_SOURCE_DIR}/LICENSE DESTINATION ${CMAKE_INSTALL_DOCDIR} RENAME LICENSE.llama.cpp COMPONENT libvla) install(TARGETS vla_core vla-cli vla-bench) +if(VLA_COMPILED) + install(TARGETS ggml_compiler_rt) +endif() install(FILES scripts/tokenize_prompt.py DESTINATION ${CMAKE_INSTALL_DATADIR}/vla) if(VLA_BUILD_SERVER) install(TARGETS vlm_core vla-server vlm-server) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d46e00cc..1d620420 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,35 +42,76 @@ the binaries fetch them: ./build/vla-cli -hf vrfai/smolvla-libero-gguf --image assets/front.jpg --tokens 1,100,2 ``` -## Adding an architecture - -Seven sites, all mechanical. Every arch loads one GGUF with its vision tower -bundled; `mmproj_path` is accepted and ignored. - -1. `src/arch.h` - add to `enum class Arch`. -2. `src/arch.h` - declare `_create(mmproj_path, ckpt_path, config_path, opts)`. -3. `src/model.cpp` - add `.architecture` to the `try_str` list in - `detect_arch_gguf`. -4. `src/model.cpp` - map the string to the enum in the same function. -5. `src/model.cpp` - add a `case` to the `model_load` switch. -6. `src/model.cpp` - if the arch has no flow-matching solver, add it to the - `opts.num_steps` refusal list in `model_load`; if it has one, read the step - count with `resolve_num_steps`. -7. `CMakeLists.txt` - add `src/models/.cpp` to `vla_core`. - -Then write `src/models/.cpp`. Before adding a helper, check `src/`: -`gguf_reader.h` (tensor and KV reads), `loader.h` (weight upload and fusion), -`modules/preprocess.h` (view checks, CHW preprocessing), `modules/dual_tower.h` -(DINOv2 + SigLIP), `modules/qwen3vl_vit.h` (Qwen3-VL tower), `layers/` (attention, -FFN, norms, RoPE, time embeddings, causal mask), `scratch_ctx.h` (compute context -and graph reuse), `backend.h` (accelerator selection). - -Your loader must fail rather than return a half-built model: check every tensor -lookup, and check `real_*_dim <= max_*_dim` (`config_is_sane` in `src/model.cpp` -does this for all archs). - -A converter goes in `scripts/convert__to_gguf.py`, and its tensor-name -remap should be covered by `tests/py/test_converters.py`. +## Benchmarking new hardware + +New models are not hand-written into this tree (see +[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#adding-an-architecture)); new +devices are. A benchmark report is one file, +`docs/benchmark/.md`, written from +[docs/benchmark/TEMPLATE.md](docs/benchmark/TEMPLATE.md). The template fixes +the section and column order so that reports compare cell for cell; this is +the procedure that fills it. + +1. **Build once, at one commit.** Configure a Release build of `main` with the + backend's flags and run the whole round from it. If the code changes, start + the round over. The report records the short SHA, the llama.cpp tag + (`scripts/llama_tag.sh`), the cmake flags including `GGML_NATIVE`, the + driver or toolkit version and the compiler (`scripts/print_versions.sh` + prints a block), the OS and kernel, and the power state: limits, governor + or profile, AC or battery, `nvpmodel` on Jetson. + +2. **Fetch the checkpoints.** Every report runs the `vrfai/*` GGUFs in the + template's "Model configuration" table, at the views, input size, token + count and steps given there. `-hf vrfai/` fetches them; check each + file's sha256 against the one Hugging Face lists, so every device runs + identical weights. + +3. **Measure the defaults.** One process per model, run three times: + + ```bash + ./build/vla-bench --ckpt smolvla-libero.gguf --images 2 --size 512 --warmup 3 --reps 20 + ``` + + Report the process with the lowest mean, copying `min`, `mean`, `p50`, + `p90` and `vision` as printed (`--markdown` prints the row). Memory is the + peak over the three processes, taken with the probe the template names + for the backend: `nvidia-smi --query-compute-apps` sampled every 0.5 s on + discrete CUDA, DRM `fdinfo` on Intel GPUs and NPUs, peak RSS where the + device shares the process's memory (Jetson, Apple silicon, CPU). Per-model + extras: `--extra-token 151697 --extra-count 32` for VLA-JEPA, + `VLA_GR00T_EMBODIMENT` for GR00T (`new_embodiment`, `libero_panda`, + `libero_sim` for N1.5, N1.6, N1.7), `VLA_OCTO_UNNORM_DATASET=libero_object` + for Octo, `--size 640 --height 480` for ACT. + +4. **Screen the runtime flags.** For each model, run 2 warmups and 5 reps + per flag set from the template's list (`--flash-attn`, `--mm-prec default`, + `--weight-dtype bf16|f16`, their combinations, `--act-dtype bf16` on CUDA + only, plus any backend-specific candidate). A set replaces the defaults + only if its mean is more than 3% lower in the screen and again under the + full protocol of step 3. The screen means go in the collapsed table; the + winner goes in "Fastest configuration". + +5. **Account for every model.** A model with no latency row gets a bullet + under "Not run": `N/A` when the arch has no path on the backend, "does + not fit" with the quoted allocation failure, "not attempted" with both + sizes when the weights plainly exceed the device's memory, or the error + it printed. Anything a reader needs to interpret the numbers (a power + limit that puts `min` well below `p50`, a required environment variable, + a row whose output is known to be wrong) goes under "Notes". + +6. **Write it up.** Copy the template, fill every ``, delete the + HTML comments, and end with a "Reproducing" block that gives the cmake + line and one `vla-bench` row. Never name a host, a user or an IP address. + Add the device to the summary table in + [docs/benchmark/README.md](docs/benchmark/README.md): the `min ms` at the + defaults per model, `N/A`, `-` or `n/r` with the meanings the README + gives, and a sentence in its intro if the round ran at a commit the other + reports did not. + +A new backend, as opposed to a new device on a backend the tree already +supports, also needs a numeric check: run `vla_predict_check` on fixed inputs +against the CPU build and report the differences, as +[docs/backend/rocm.md](docs/backend/rocm.md) does. ## Ports of upstream quirks diff --git a/README.md b/README.md index b38eaec7..3e6fbf1d 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ variables are in [docs/USAGE.md](docs/USAGE.md). Models (rows) against platforms (columns). Per-device latency and memory are in [docs/benchmark/](docs/benchmark). -Use [ggml-compiler](https://github.com/khanhnd61-vr/ggml-compiler) and follow [the instruction](docs/ARCHITECTURE.md#adding-an-architecture) to add a new model. +A model not listed here runs without a vla.cpp change: compile it with [ggml-compiler](https://github.com/khanhnd61-vr/ggml-compiler) and serve the GGUF ([docs/COMPILED.md](docs/COMPILED.md)). | Model | CPU (x86-64 / ARM) | [CUDA (NVIDIA)](docs/backend/cuda.md) | [ROCm (AMD)](docs/backend/rocm.md) | [SYCL (Intel)](docs/backend/sycl.md) | [Metal (Apple)](docs/backend/metal.md) | [OpenVINO (Intel)](docs/backend/ov.md) | [Hexagon (Qualcomm)](docs/backend/hexagon.md) | @@ -151,7 +151,7 @@ Installing, wiring and running the client are in the fork's | [docs/ADOPTION.md](docs/ADOPTION.md) | C ABI, Python bindings, release packaging, and what is left | | [docs/UPSTREAMING.md](docs/UPSTREAMING.md) | The ggml-openvino fixes to send upstream to llama.cpp | | [CHANGELOG.md](CHANGELOG.md) | Per-release changes, with the fastest configuration and success rate per model | -| [CONTRIBUTING.md](CONTRIBUTING.md) | Proving a change is numerically neutral, adding an architecture | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Proving a change is numerically neutral, benchmarking new hardware | | [bindings/python/](bindings/python/README.md) | Python bindings | --- diff --git a/docs/ADOPTION.md b/docs/ADOPTION.md index d1a10a44..39b0e695 100644 --- a/docs/ADOPTION.md +++ b/docs/ADOPTION.md @@ -22,8 +22,9 @@ Done: `vla-cli`, `vla-server` and `vla-bench`, cached under `$VLA_CACHE`. A repo with several GGUFs lists them instead of guessing. 6. **Reproducible benchmarks.** `vla-bench` emits the README table rows. -7. **Contributor path.** `CONTRIBUTING.md` has the six-site walkthrough for - adding an architecture, plus issue and PR templates. +7. **Contributor path.** `CONTRIBUTING.md` has the benchmark protocol for a + new device, plus issue and PR templates; new models go through + ggml-compiler (`docs/COMPILED.md`). 8. **Instruction in, action out.** `vla-cli --text` builds each arch's real prompt. Octo carries its tokenizer in the GGUF, and `scripts/add_tokenizer_to_gguf.py` adds one to pi0, pi0.5 and OpenVLA-OFT, so diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 00c62802..da95a58b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -13,7 +13,9 @@ map; the source is the detail. - `src/model.cpp` - loads a checkpoint, detects the architecture from its GGUF keys (or safetensors namespace), and dispatches to the matching factory. - `src/models/*.cpp` - one translation unit per architecture. Each owns its ggml - contexts, vision tower, weights, and compute graph. + contexts, vision tower, weights, and compute graph. `compiled.cpp` is the one + that owns no graph: it runs a GGUF written by ggml-compiler through its + runtime, from the `vla` manifest in `compiled_manifest.h` ([COMPILED.md](COMPILED.md)). - `src/gguf_reader.h` - the shared GGUF reader (metadata, tensor bytes, on-demand embedding rows). `src/loader.h` uploads weights at the resident dtype. - `src/layers/` - ggml building blocks: linear, attention, FFN, norms, RoPE, @@ -92,40 +94,34 @@ reproduce ggml's output bit for bit ([backend/cuda.md](backend/cuda.md#the-bit-e ## Adding an architecture -[ggml-compiler](https://github.com/khanhnd61-vr/ggml-compiler) makes the graph -implementation easier: compile the model's PyTorch inference path into ggml C++ -source, with one graph builder per stage and a GGUF containing weights and the -compiled computation. Its +New models are not hand-written into this tree. Compile the policy with +[ggml-compiler](https://github.com/khanhnd61-vr/ggml-compiler) into a GGUF that +carries its weights, its op graphs and a `vla` serving manifest, and the +`compiled` arch (`src/models/compiled.cpp`) serves it through ggml-compiler's +runtime, built here against this tree's ggml. `vla-server`, `vla-cli`, +`vla-bench` and `libvla` run it unchanged; every backend vla.cpp builds is +available to it. [COMPILED.md](COMPILED.md) is the guide; ggml-compiler's [export guide](https://github.com/khanhnd61-vr/ggml-compiler/blob/main/docs/EXPORT.md) and [examples](https://github.com/khanhnd61-vr/ggml-compiler/tree/main/examples) cover ACT, SmolVLA, pi0/pi0.5, GR00T N1.7 and Diffusion Policy. 1. Wrap the reference model in an `export.py:program` function returning a - `ggml_compiler.Program`. Define tensor inputs and outputs, fix deployment - shapes, and split iterative policies into an encoder/prefix stage and a - reusable denoising step. Record preprocessing, normalization, noise inputs, - and any prompt or embodiment specialization. -2. Generate the ggml C++ source and GGUF: + `ggml_compiler.Program`: tensor inputs and outputs, fixed deployment + shapes, iterative policies split into a prefix stage and a reusable + denoising step, and the manifest (`ggml_compiler.vla`) naming each input's + role and preprocessing. +2. Compile and verify: ```bash - ggml-compiler compile export.py:program --out out/new_model \ - --emit gguf,cpp --name new_model + ggml-compiler compile export.py:program --out out/new_model --emit gguf ggml-compiler verify out/new_model/model.gguf --backend cpu - ggml-compiler verify out/new_model/model.gguf --backend cuda ``` - Also compare the staged PyTorch inference with the original policy. The - compiler's `verify` checks against the staged reference saved at compile - time; build the emitted C++ checker to compare its graph builders with the - compiler runtime. -3. Integrate the validated computation into a vla.cpp architecture. Adapt the - generated graph builders to vla.cpp's weight loader, graph lifecycle, and - prediction interface, including the policy's preprocessing and action units. - The generated C++ uses ggml-compiler's runtime/build API; its GGUF is not - automatically loadable by `vla-cli`. Check ggml version compatibility and - validate each target backend before marking it supported. - -For registration, extend the `Arch` enum, declare a `*_create` factory in `arch.h`, implement it under -`src/models/`, wire detection and dispatch in `src/model.cpp`, and add a converter in -`scripts/`. Reuse `gguf_reader.h`, `loader.h`, `layers/` and `modules/` rather than -copying them. [CONTRIBUTING.md](../CONTRIBUTING.md) lists the exact sites. +3. `vla-server out/new_model/model.gguf`. + +A hand-written arch is still how a model gets vla.cpp's own kernels (fused +CUDA nodes, FoldQuant) and per-backend tuning. For one, extend the `Arch` enum, +declare a `*_create` factory in `arch.h`, implement it under `src/models/`, wire +detection and dispatch in `src/model.cpp`, and add a converter in `scripts/`. +Reuse `gguf_reader.h`, `loader.h`, `layers/` and `modules/` rather than copying +them; `src/models/act.cpp` is the smallest complete example. diff --git a/docs/COMPILED.md b/docs/COMPILED.md new file mode 100644 index 00000000..2c9569a6 --- /dev/null +++ b/docs/COMPILED.md @@ -0,0 +1,121 @@ +# Serving a model compiled by ggml-compiler + +vla.cpp no longer adds hand-written architectures for every new policy. A +model the engine does not know is compiled with +[ggml-compiler](https://github.com/khanhnd61-vr/ggml-compiler) into one GGUF +that holds its weights, its op graphs and a **serving manifest**, and +`vla-server`, `vla-cli`, `vla-bench` and `libvla` run that GGUF like any +built-in model. No vla.cpp change, no plugin binary, no model-specific code. + +- [How it works](#how-it-works) +- [Compiling a policy](#compiling-a-policy) +- [Running it](#running-it) +- [The manifest](#the-manifest) +- [Build](#build) +- [Limits of this first version](#limits-of-this-first-version) + +## How it works + +`model_load` detects `general.architecture = "ggml_compiler"` and dispatches to +the `compiled` arch (`src/models/compiled.cpp`). That arch: + +1. picks the backend the way every arch does (`src/backend.h`: the compiled-in + accelerator, else CPU) and loads the program onto it through ggml-compiler's + runtime, `libggml_compiler_rt`, which this tree builds against its own ggml + (one ggml per process, so every backend vla.cpp has is available to compiled + models); +2. reads the `vla` manifest from the GGUF's metadata and checks it against the + program: every input has a role, the actions output exists and has + `chunk x dim` values; +3. fills `Config` from the manifest, so `vla-server`'s request validation, the + protobuf, the C API and the Python bindings work unchanged; +4. on each `predict`, does only what the manifest describes: image resize and + normalization, token padding, state normalization, noise sampling, the run, + and action un-normalization. + +The runtime checks every node of every graph against the backend at load, so an +op the backend cannot run is a named load error, not a wrong action. + +## Compiling a policy + +In a ggml-compiler checkout, following its +[export guide](https://github.com/khanhnd61-vr/ggml-compiler/blob/main/docs/EXPORT.md) +and [serving guide](https://github.com/khanhnd61-vr/ggml-compiler/blob/main/docs/SERVING.md): + +```bash +uv run ggml-compiler compile examples/act/export.py:program --out out/act --emit gguf +uv run ggml-compiler verify out/act/model.gguf --backend cpu +``` + +The export script attaches the manifest with `Program(metadata={"vla": ...})`, +built by `ggml_compiler.vla`; `compile` refuses a manifest that does not cover +the program's inputs. Reduced-precision weights come from a ggml-compiler dtype +policy (`--policy`), not from `--weight-dtype`, which the compiled arch rejects. + +## Running it + +```bash +./build/vla-cli --ckpt out/act/model.gguf --image front.jpg --image wrist.jpg --state 0,0,0,0,0,0 --pretty +./build/vla-server out/act/model.gguf +``` + +The load prints what the manifest declares: + +``` +vla: arch = compiled (ggml-compiler) +vla(compiled): ggml-compiler 0.1.0 on CPU; 2 camera(s), state, chunk 100 x 6 action dims +vla(compiled): view 0 'observation.images.front' 640x480 +vla(compiled): view 1 'observation.images.wrist' 640x480 +``` + +Cameras go in manifest order. A compiled program carries no tokenizer: pass +`--tokens` to `vla-cli`, and token ids in `PredictRequest.lang_tokens` as every +client already does. Images arrive at the compiled size unless the manifest +says `fit: resize`. + +## The manifest + +Version 1, stored as JSON under `vla` in the GGUF key `ggml_compiler.meta`. +The full reference is ggml-compiler's +[SERVING.md](https://github.com/khanhnd61-vr/ggml-compiler/blob/main/docs/SERVING.md#manifest-reference); +in short: + +| Role | Program tensor | vla.cpp does | +|---|---|---| +| `images[i]` | `[1, 3, h, w]` float | request view i, scaled to [0, 1], `(x - mean) / std` per channel, CHW. `fit: exact` refuses another size, `resize` scales bilinearly. An optional `mask` input is 1 when the view was sent. | +| `lang` | `tokens` `[1, max_len]` int, optional `mask` | token ids padded with `pad_id`; `Config::n_lang = max_len` | +| `state` | `[1, dim]` float | `real_dim` values normalized (`mean_std`: LeRobot's `(x - mean) / (std + eps)`; `min_max`; or `none`), the rest zero | +| `noise` | any float input | the request's noise, or N(0, 1) | +| `actions` | `[1, chunk, dim]` float | un-normalized over `real_dim` columns; `Config::n_suffix = chunk`, `max_action_dim = dim` | + +Inputs are converted to the dtype the program declares (f32, f16 or i32), so a +0/1 mask may be either float or int in the program. + +## Build + +`VLA_COMPILED` (default ON) fetches ggml-compiler's `runtime/` at the ref in +`VLA_GGML_COMPILER_TAG` and builds it against this tree's ggml with +`GGML_COMPILED_EXTERNAL_GGML`; the runtime refuses a llama.cpp tag other than +the one it was tested with, which is the tree's `VLA_LLAMA_TAG`. `-DVLA_COMPILED=OFF` +drops the dependency; a compiled GGUF then fails to load with a message saying +so. To build against a local ggml-compiler checkout: + +```bash +cmake -S . -B build -DFETCHCONTENT_SOURCE_DIR_GGML_COMPILER=/path/to/ggml-compiler +``` + +Unit test: `tests/test_compiled_manifest.cpp` (ctest `compiled_manifest`). + +## Limits of this first version + +- Programs with symbolic dims are refused; compile with fixed shapes and pad. +- Only the default entry point is served; a multi-entry program (a KV-cached + VLM) runs its first entry. +- No tokenizer and no prompt template: clients send token ids. +- `precomputed_img_emb` and `encode_images` are not supported. +- `--weight-dtype`, `--act-dtype` and `--num-steps` do not apply; those are + compile-time choices. +- Image fitting is exact or bilinear resize; letterboxing (LeRobot's + `resize_with_pad`) is not implemented yet. +- vla.cpp's fused CUDA and FoldQuant paths reach ggml through its extension + hooks and are not used by compiled graphs. diff --git a/docs/USAGE.md b/docs/USAGE.md index 5767baeb..8397d1a9 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -88,6 +88,14 @@ Clients: the LIBERO and SimplerEnv runners in [EVAL.md](EVAL.md), and the real-robot client in the README's [Rollout on a real robot](../README.md#rollout-on-a-real-robot). +## Models compiled by ggml-compiler + +A GGUF written by `ggml-compiler compile` loads like any other on every binary +above, provided the build has `VLA_COMPILED=ON` (the default) and the program +carries a `vla` serving manifest. `--text` is not available for it (no +tokenizer in the GGUF); pass `--tokens`. `--weight-dtype`, `--act-dtype` and +`--num-steps` do not apply. See [COMPILED.md](COMPILED.md). + ## Runtime flags The same on `vla-server`, `vla-cli` and `vla-bench` (`--help` for the full diff --git a/src/arch.h b/src/arch.h index 80d04d64..a51563da 100644 --- a/src/arch.h +++ b/src/arch.h @@ -74,6 +74,7 @@ enum class Arch { TURBOVLA, // TurboVLA (DINOv3 + BERT + VL Fusion + ACT decoder). ACT, // LeRobot ACT (ResNet + transformer encoder-decoder, no language). COSMOS3_POLICY,// NVIDIA Cosmos3 Policy world-action model (Nemotron MoT + Wan2.2 VAE, UniPC). + COMPILED, // Any policy compiled by ggml-compiler, run from its GGUF's program and `vla` manifest. }; /** @@ -261,6 +262,16 @@ std::unique_ptr cosmos3_policy_create(const std::string& mmproj_p const std::string& config_path, const Options& opts); +/** + * @brief Load a GGUF written by `ggml-compiler compile`. Needs a build with + * VLA_COMPILED=ON; the program's `vla` manifest supplies the config. + * @copydetails smolvla_create + */ +std::unique_ptr compiled_create(const std::string& mmproj_path, + const std::string& ckpt_path, + const std::string& config_path, + const Options& opts); + /** * @brief Inspect a GGUF and identify the architecture tag. * diff --git a/src/model.cpp b/src/model.cpp index 5e69a617..3f45dd5b 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -87,6 +87,10 @@ bool detect_arch_gguf(const std::string& path, Arch* out) { *out = Arch::SMOLVLA; ok = true; } + else if (arch_str == "ggml_compiler") { + *out = Arch::COMPILED; + ok = true; + } else if (arch_str == "pi0") { *out = Arch::PI0; ok = true; @@ -254,7 +258,8 @@ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, : arch == Arch::VLA_ADAPTER ? "vla_adapter" : arch == Arch::OPENVLA_OFT ? "openvla_oft" : arch == Arch::TURBOVLA ? "turbovla" - : arch == Arch::ACT ? "act" : nullptr; + : arch == Arch::ACT ? "act" + : arch == Arch::COMPILED ? "compiled" : nullptr; if (name) { std::fprintf(stderr, "vla(%s): num_steps is not supported\n", name); return nullptr; @@ -331,6 +336,16 @@ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, std::printf("vla: arch = cosmos3_policy\n"); impl = cosmos3_policy_create(mmproj_path, ckpt_path, config_path, opts); break; + case Arch::COMPILED: + std::printf("vla: arch = compiled (ggml-compiler)\n"); +#ifdef VLA_COMPILED + impl = compiled_create(mmproj_path, ckpt_path, config_path, opts); +#else + std::fprintf(stderr, "vla: %s was written by ggml-compiler; this build has VLA_COMPILED=OFF\n", + ckpt_path.c_str()); + return nullptr; +#endif + break; } if (!impl) return nullptr; diff --git a/src/models/compiled.cpp b/src/models/compiled.cpp new file mode 100644 index 00000000..ce45a6b4 --- /dev/null +++ b/src/models/compiled.cpp @@ -0,0 +1,431 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A model compiled by ggml-compiler: the GGUF carries the weights, the op +// graphs and a `vla` serving manifest (compiled_manifest.h). The runtime +// (libggml_compiler_rt, built against this tree's ggml) runs the program on +// the backend backend.h picks; this file is only the glue the manifest +// describes: image resize and normalization, token padding, state +// normalization, noise, and action un-normalization. + +#include "arch.h" +#include "backend.h" +#include "models/compiled_manifest.h" +#include "modules/preprocess.h" + +#include "ggml_compiler_rt.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vla { + +namespace { + +const char * kTag = "vla(compiled)"; + +int64_t numel(const ggml_compiler_tensor_info & info) { + int64_t n = 1; + for (int i = 0; i < info.ndim; ++i) + n *= info.shape[i]; + return n; +} + +std::string shape_str(const ggml_compiler_tensor_info & info) { + std::string s = "["; + for (int i = 0; i < info.ndim; ++i) + s += (i ? ", " : "") + std::to_string((long long) info.shape[i]); + return s + "]"; +} + +// PyTorch F.interpolate(mode="bilinear", align_corners=False, antialias=False) +// of an interleaved RGB view into [oh, ow, 3] floats in [0, 1]. +void resize_bilinear_hwc(const ImageView & v, int64_t oh, int64_t ow, std::vector & out) { + out.resize((size_t) oh*ow*3); + const int64_t ih = v.h, iw = v.w; + const float sy = (float) ih / (float) oh, sx = (float) iw / (float) ow; + auto px = [&](int64_t y, int64_t x, int c) -> float { + const size_t i = ((size_t) y*iw + x)*3 + c; + return v.format == PixelFormat::U8 ? ((const uint8_t *) v.data)[i]/255.0f : ((const float *) v.data)[i]; + }; + for (int64_t y = 0; y < oh; ++y) { + const float fy = std::max(0.f, ((float) y + 0.5f)*sy - 0.5f); + const int64_t y0 = std::min((int64_t) fy, ih - 1), y1 = std::min(y0 + 1, ih - 1); + const float ly = fy - (float) y0; + for (int64_t x = 0; x < ow; ++x) { + const float fx = std::max(0.f, ((float) x + 0.5f)*sx - 0.5f); + const int64_t x0 = std::min((int64_t) fx, iw - 1), x1 = std::min(x0 + 1, iw - 1); + const float lx = fx - (float) x0; + float * dst = &out[((size_t) y*ow + x)*3]; + for (int c = 0; c < 3; ++c) { + const float top = px(y0, x0, c)*(1.f - lx) + px(y0, x1, c)*lx; + const float bot = px(y1, x0, c)*(1.f - lx) + px(y1, x1, c)*lx; + dst[c] = top*(1.f - ly) + bot*ly; + } + } + } +} + +class CompiledArch final : public ModelArchBase { +public: + CompiledArch() : ModelArchBase(Arch::COMPILED) {} + ~CompiledArch() override { + if (model) + ggml_compiler_free(model); // frees the backend it was loaded on too + } + + ggml_compiler_model * model = nullptr; + compiled::Manifest mf; + std::unordered_map inputs; // the default entry's + ggml_compiler_tensor_info out_info{}; + std::mt19937 rng{std::random_device{}()}; + + // Scratch, kept across calls. + std::vector f32; + std::vector hwc; + std::vector f16; + std::vector i32; + + // Upload n floats (or ints) as input `name`, converting to the dtype the + // program declares. The count must match the input exactly. + template + bool set(const std::string & name, const T * data, int64_t n); + + std::vector predict(const Inputs & in) override; +}; + +template +bool CompiledArch::set(const std::string & name, const T * data, int64_t n) { + const auto it = inputs.find(name); + if (it == inputs.end()) { + std::fprintf(stderr, "%s: no program input '%s'\n", kTag, name.c_str()); + return false; + } + const ggml_compiler_tensor_info & info = it->second; + if (numel(info) != n) { + std::fprintf(stderr, "%s: input '%s' is %s (%lld values), got %lld\n", kTag, name.c_str(), + shape_str(info).c_str(), (long long) numel(info), (long long) n); + return false; + } + const void * src = nullptr; + size_t nbytes = 0; + switch (info.dtype) { + case GGML_COMPILER_DTYPE_F32: + if constexpr (std::is_same_v) { + src = data; + } else { + f32.resize((size_t) n); + for (int64_t i = 0; i < n; ++i) f32[(size_t) i] = (float) data[i]; + src = f32.data(); + } + nbytes = (size_t) n*sizeof(float); + break; + case GGML_COMPILER_DTYPE_F16: + f16.resize((size_t) n); + for (int64_t i = 0; i < n; ++i) f16[(size_t) i] = ggml_fp32_to_fp16((float) data[i]); + src = f16.data(); + nbytes = (size_t) n*sizeof(uint16_t); + break; + case GGML_COMPILER_DTYPE_I32: + if constexpr (std::is_same_v) { + src = data; + } else { + i32.resize((size_t) n); + for (int64_t i = 0; i < n; ++i) i32[(size_t) i] = (int32_t) std::lrint(data[i]); + src = i32.data(); + } + nbytes = (size_t) n*sizeof(int32_t); + break; + } + if (!src || nbytes != info.nbytes) { + std::fprintf(stderr, "%s: input '%s' has an unsupported dtype\n", kTag, name.c_str()); + return false; + } + if (ggml_compiler_set_input(model, name.c_str(), src, nbytes) != 0) { + std::fprintf(stderr, "%s: set_input('%s'): %s\n", kTag, name.c_str(), ggml_compiler_last_error()); + return false; + } + return true; +} + +std::vector CompiledArch::predict(const Inputs & in) { + using clock = std::chrono::steady_clock; + const auto t0 = clock::now(); + stats = Stats{}; + + if (in.precomputed_img_emb) { + std::fprintf(stderr, "%s: precomputed_img_emb is not supported; pass raw images\n", kTag); + return {}; + } + + // Cameras, in manifest order. A missing camera is allowed only when the + // program has a mask for it. + const int n_cams = (int) mf.images.size(); + if (in.n_images > n_cams) { + std::fprintf(stderr, "%s: the program takes %d camera view(s), got %d\n", kTag, n_cams, in.n_images); + return {}; + } + for (int v = 0; v < n_cams; ++v) { + const compiled::ImageSpec & spec = mf.images[(size_t) v]; + const bool present = v < in.n_images && in.images && in.images[v].data; + f32.assign((size_t) 3*spec.h*spec.w, 0.f); + if (present) { + const ImageView & iv = in.images[v]; + if (iv.w == spec.w && iv.h == spec.h) { + image_to_chw(iv, spec.mean, spec.std_, f32.data()); + } else if (spec.resize) { + resize_bilinear_hwc(iv, spec.h, spec.w, hwc); + const ImageView r{hwc.data(), (int) spec.w, (int) spec.h, PixelFormat::F32_RGB_01}; + image_to_chw(r, spec.mean, spec.std_, f32.data()); + } else { + std::fprintf(stderr, "%s: view %d is %dx%d; '%s' was compiled for %lldx%lld (fit: exact)\n", kTag, v, + iv.w, iv.h, spec.input.c_str(), (long long) spec.w, (long long) spec.h); + return {}; + } + } else if (spec.mask.empty()) { + std::fprintf(stderr, "%s: the program takes %d camera view(s) and '%s' has no mask, got %d\n", kTag, + n_cams, spec.input.c_str(), in.n_images); + return {}; + } + // set converts through f16/i32, never through f32, so the image + // can stay in f32. + if (!set(spec.input, f32.data(), (int64_t) 3*spec.h*spec.w)) + return {}; + if (!spec.mask.empty()) { + const float on = present ? 1.f : 0.f; + if (!set(spec.mask, &on, 1)) + return {}; + } + } + + // Instruction. + if (mf.lang) { + const compiled::LangSpec & l = *mf.lang; + if (!in.lang_tokens || in.n_lang <= 0) { + std::fprintf(stderr, "%s: the program takes an instruction (up to %lld tokens); none given\n", kTag, + (long long) l.max_len); + return {}; + } + if (in.n_lang > l.max_len) { + std::fprintf(stderr, "%s: %d language tokens exceed the compiled maximum %lld\n", kTag, in.n_lang, + (long long) l.max_len); + return {}; + } + std::vector tok((size_t) l.max_len, l.pad_id); + std::copy(in.lang_tokens, in.lang_tokens + in.n_lang, tok.begin()); + if (!set(l.tokens, tok.data(), l.max_len)) + return {}; + if (!l.mask.empty()) { + std::vector mask((size_t) l.max_len, 0.f); + std::fill(mask.begin(), mask.begin() + in.n_lang, 1.f); + if (!set(l.mask, mask.data(), l.max_len)) + return {}; + } + } else if (in.n_lang > 0) { + std::fprintf(stderr, "%s: this program takes no instruction; drop the language tokens\n", kTag); + return {}; + } + + // State. + if (mf.state) { + const compiled::StateSpec & s = *mf.state; + if (!in.state) { + std::fprintf(stderr, "%s: the program takes a %lld-dim state\n", kTag, (long long) s.real_dim); + return {}; + } + std::vector st((size_t) s.dim, 0.f); + for (int64_t i = 0; i < s.real_dim; ++i) + st[(size_t) i] = s.norm.normalize(in.state[i], (size_t) i); + if (!set(s.input, st.data(), s.dim)) + return {}; + } + + // Noise: the caller's, or N(0, 1). + if (!mf.noise.empty()) { + const int64_t n = numel(inputs.at(mf.noise)); + std::vector z((size_t) n); + if (in.noise) { + std::copy(in.noise, in.noise + n, z.begin()); + } else { + std::normal_distribution normal(0.f, 1.f); + for (auto & x : z) x = normal(rng); + } + if (!set(mf.noise, z.data(), n)) + return {}; + } + + const auto tc = clock::now(); + if (ggml_compiler_run(model) != 0) { + std::fprintf(stderr, "%s: run failed: %s\n", kTag, ggml_compiler_last_error()); + return {}; + } + + const compiled::ActionSpec & a = mf.actions; + const int64_t n_out = a.chunk*a.dim; + std::vector out((size_t) n_out); + if (out_info.dtype == GGML_COMPILER_DTYPE_F32) { + if (ggml_compiler_get_output(model, a.output.c_str(), out.data(), out_info.nbytes) != 0) { + std::fprintf(stderr, "%s: get_output('%s'): %s\n", kTag, a.output.c_str(), ggml_compiler_last_error()); + return {}; + } + } else { + f16.resize((size_t) n_out); + if (ggml_compiler_get_output(model, a.output.c_str(), f16.data(), out_info.nbytes) != 0) { + std::fprintf(stderr, "%s: get_output('%s'): %s\n", kTag, a.output.c_str(), ggml_compiler_last_error()); + return {}; + } + for (int64_t i = 0; i < n_out; ++i) + out[(size_t) i] = ggml_fp16_to_fp32(f16[(size_t) i]); + } + for (int64_t t = 0; t < a.chunk; ++t) + for (int64_t j = 0; j < a.real_dim; ++j) { + float & y = out[(size_t) (t*a.dim + j)]; + y = a.norm.denormalize(y, (size_t) j); + } + + ggml_compiler_stats st{}; + ggml_compiler_last_stats(model, &st); + stats.ms_inference = (float) st.ms_total; + (void) tc; + stats.ms_total = std::chrono::duration(clock::now() - t0).count(); + return out; +} + +} // namespace + +std::unique_ptr compiled_create(const std::string & mmproj_path, const std::string & ckpt_path, + const std::string & config_path, const Options & opts) { + (void) config_path; + if (!mmproj_path.empty()) + std::printf("%s: ignoring --mmproj; the program bundles its vision tower\n", kTag); + if (opts.weight_dtype || opts.act_dtype) { + std::fprintf(stderr, "%s: weight and activation dtypes are fixed at compile time; use a ggml-compiler " + "dtype policy instead of --weight-dtype/--act-dtype\n", kTag); + return nullptr; + } + + auto m = std::make_unique(); + + const Backend b = backend_init(kTag, default_cpu_threads()); + if (!b.handle) + return nullptr; + // The runtime owns the backend from here on, failure included. + m->model = ggml_compiler_load_on(ckpt_path.c_str(), b.handle, nullptr, 0); + if (!m->model) { + std::fprintf(stderr, "%s: %s\n", kTag, ggml_compiler_last_error()); + return nullptr; + } + + if (ggml_compiler_n_symbols(m->model) > 0) { + std::fprintf(stderr, "%s: the program has symbolic dims; serving them is not supported yet, compile with " + "fixed shapes\n", kTag); + return nullptr; + } + const int n_entries = ggml_compiler_n_entries(m->model); + if (n_entries > 1) + std::printf("%s: %d entry points; serving the default, '%s'\n", kTag, n_entries, + ggml_compiler_entry_name(m->model, 0)); + + const char * meta = ggml_compiler_metadata(m->model, "ggml_compiler.meta"); + std::string err; + if (!compiled::parse_manifest(meta ? meta : "{}", m->mf, err)) { + std::fprintf(stderr, "%s: %s: %s\n", kTag, ckpt_path.c_str(), err.c_str()); + return nullptr; + } + + // The manifest must cover the default entry's inputs exactly, and name a + // real output of the right size. + for (int i = 0; i < ggml_compiler_entry_n_inputs(m->model, 0); ++i) { + ggml_compiler_tensor_info info{}; + ggml_compiler_entry_input_info(m->model, 0, i, &info); + m->inputs[info.name] = info; + } + std::vector named = m->mf.input_names(); + for (const auto & n : named) { + if (!m->inputs.count(n)) { + std::fprintf(stderr, "%s: the manifest names input '%s', which the program does not have\n", kTag, + n.c_str()); + return nullptr; + } + } + for (const auto & [n, info] : m->inputs) { + if (std::find(named.begin(), named.end(), n) == named.end()) { + std::fprintf(stderr, "%s: program input '%s' %s has no role in the manifest\n", kTag, n.c_str(), + shape_str(info).c_str()); + return nullptr; + } + } + bool have_out = false; + for (int i = 0; i < ggml_compiler_entry_n_outputs(m->model, 0); ++i) { + ggml_compiler_tensor_info info{}; + ggml_compiler_entry_output_info(m->model, 0, i, &info); + if (m->mf.actions.output == info.name) { + m->out_info = info; + have_out = true; + } + } + if (!have_out) { + std::fprintf(stderr, "%s: the manifest's actions output '%s' is not a program output\n", kTag, + m->mf.actions.output.c_str()); + return nullptr; + } + if (numel(m->out_info) != m->mf.actions.chunk*m->mf.actions.dim) { + std::fprintf(stderr, "%s: output '%s' is %s, not chunk %lld x dim %lld\n", kTag, m->mf.actions.output.c_str(), + shape_str(m->out_info).c_str(), (long long) m->mf.actions.chunk, (long long) m->mf.actions.dim); + return nullptr; + } + if (m->out_info.dtype != GGML_COMPILER_DTYPE_F32 && m->out_info.dtype != GGML_COMPILER_DTYPE_F16) { + std::fprintf(stderr, "%s: output '%s' is not a float tensor\n", kTag, m->mf.actions.output.c_str()); + return nullptr; + } + for (const auto & im : m->mf.images) { + const ggml_compiler_tensor_info & info = m->inputs.at(im.input); + if (numel(info) != 3*im.h*im.w) { + std::fprintf(stderr, "%s: input '%s' is %s, not an image of 3 x %lld x %lld\n", kTag, im.input.c_str(), + shape_str(info).c_str(), (long long) im.h, (long long) im.w); + return nullptr; + } + } + + Config & c = m->cfg; + c.n_lang = m->mf.lang ? m->mf.lang->max_len : 0; + c.n_state = m->mf.state ? 1 : 0; + c.n_suffix = m->mf.actions.chunk; + c.max_state_dim = m->mf.state ? m->mf.state->dim : 0; + c.real_state_dim = m->mf.state ? m->mf.state->real_dim : 0; + c.max_action_dim = m->mf.actions.dim; + c.real_action_dim = m->mf.actions.real_dim; + c.denormalized = true; + + std::printf("%s: %s on %s; %zu camera(s), %s%s%schunk %lld x %lld action dims\n", kTag, + ggml_compiler_version(), ggml_compiler_backend_name(m->model), m->mf.images.size(), + m->mf.lang ? "instruction, " : "", m->mf.state ? "state, " : "", m->mf.noise.empty() ? "" : "noise, ", + (long long) m->mf.actions.chunk, (long long) m->mf.actions.real_dim); + for (size_t v = 0; v < m->mf.images.size(); ++v) { + const auto & im = m->mf.images[v]; + std::printf("%s: view %zu '%s' %lldx%lld%s\n", kTag, v, im.input.c_str(), (long long) im.w, (long long) im.h, + im.resize ? " (other sizes resized)" : ""); + } + return m; +} + +} // namespace vla diff --git a/src/models/compiled_manifest.h b/src/models/compiled_manifest.h new file mode 100644 index 00000000..6ff06980 --- /dev/null +++ b/src/models/compiled_manifest.h @@ -0,0 +1,296 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The `vla` serving manifest a ggml-compiler GGUF carries in its metadata +// (ggml_compiler.meta -> "vla"): which program input is which camera, the +// instruction, the state and the noise, which output is the action chunk, and +// the preprocessing of each. Version 1 is specified in ggml-compiler's +// docs/SERVING.md. Header-only and runtime-free so it can be unit-tested. + +#pragma once + +#include "nlohmann/json.hpp" + +#include +#include +#include +#include +#include +#include + +namespace vla { +namespace compiled { + +constexpr int kManifestVersion = 1; + +// LeRobot's normalizers: (x - mean) / (std + eps), or 2 (x - min) / (max - min + eps) - 1. +struct Norm { + enum Kind { NONE, MEAN_STD, MIN_MAX }; + Kind kind = NONE; + std::vector a, b; // mean/std or min/max, real_dim values + float eps = 1e-8f; + + float normalize(float x, size_t i) const { + switch (kind) { + case MEAN_STD: return (x - a[i]) / (b[i] + eps); + case MIN_MAX: return 2.f*(x - a[i]) / (b[i] - a[i] + eps) - 1.f; + default: return x; + } + } + float denormalize(float y, size_t i) const { + switch (kind) { + case MEAN_STD: return y*(b[i] + eps) + a[i]; + case MIN_MAX: return (y + 1.f)*0.5f*(b[i] - a[i] + eps) + a[i]; + default: return y; + } + } +}; + +struct ImageSpec { + std::string input; // [1, 3, h, w] float input + std::string mask; // optional [1] input: 1 when the camera was sent + int64_t h = 0, w = 0; + bool resize = false; // "fit": "resize" scales another size bilinearly; "exact" refuses it + float mean[3] = {0.f, 0.f, 0.f}; + float std_[3] = {1.f, 1.f, 1.f}; +}; + +struct LangSpec { + std::string tokens; // [1, max_len] int input + std::string mask; // optional [1, max_len] 0/1 input + int64_t max_len = 0; + int32_t pad_id = 0; +}; + +struct StateSpec { + std::string input; // [1, dim] float input + int64_t dim = 0, real_dim = 0; + Norm norm; +}; + +struct ActionSpec { + std::string output; // [1, chunk, dim] float output + int64_t chunk = 0, dim = 0, real_dim = 0; + Norm norm; +}; + +struct Manifest { + int version = 0; + std::vector images; + std::optional lang; + std::optional state; + std::string noise; // input name; empty when the program takes none + ActionSpec actions; + + // Every program input the manifest fills, in host order. + std::vector input_names() const { + std::vector n; + for (const auto & im : images) { + n.push_back(im.input); + if (!im.mask.empty()) n.push_back(im.mask); + } + if (lang) { + n.push_back(lang->tokens); + if (!lang->mask.empty()) n.push_back(lang->mask); + } + if (state) n.push_back(state->input); + if (!noise.empty()) n.push_back(noise); + return n; + } +}; + +namespace detail { + +using json = nlohmann::json; + +inline const json & need(const json & j, const char * key, const char * where) { + const auto it = j.find(key); + if (it == j.end()) + throw std::runtime_error(std::string(where) + " lacks '" + key + "'"); + return *it; +} + +inline std::string str(const json & j, const char * key, const char * where) { + const json & v = need(j, key, where); + if (!v.is_string() || v.get().empty()) + throw std::runtime_error(std::string(where) + "." + key + " must be a non-empty string"); + return v.get(); +} + +inline std::string opt_str(const json & j, const char * key) { + const auto it = j.find(key); + return it != j.end() && it->is_string() ? it->get() : std::string(); +} + +inline int64_t positive(const json & j, const char * key, const char * where) { + const json & v = need(j, key, where); + if (!v.is_number_integer() || v.get() <= 0) + throw std::runtime_error(std::string(where) + "." + key + " must be a positive integer"); + return v.get(); +} + +inline std::vector floats(const json & j, const char * key, const char * where, size_t n) { + const json & v = need(j, key, where); + if (!v.is_array() || v.size() != n) + throw std::runtime_error(std::string(where) + "." + key + " must hold " + std::to_string(n) + " numbers"); + std::vector out; + out.reserve(n); + for (const auto & x : v) { + if (!x.is_number()) + throw std::runtime_error(std::string(where) + "." + key + " must hold numbers"); + out.push_back(x.get()); + } + return out; +} + +inline Norm norm(const json & j, const char * where, size_t n) { + Norm out; + const std::string kind = opt_str(j, "norm").empty() ? "none" : opt_str(j, "norm"); + if (kind == "none") + return out; + if (kind == "mean_std") { + out.kind = Norm::MEAN_STD; + out.a = floats(j, "mean", where, n); + out.b = floats(j, "std", where, n); + } else if (kind == "min_max") { + out.kind = Norm::MIN_MAX; + out.a = floats(j, "min", where, n); + out.b = floats(j, "max", where, n); + } else { + throw std::runtime_error(std::string(where) + ".norm '" + kind + "' is not mean_std, min_max or none"); + } + if (const auto it = j.find("eps"); it != j.end()) { + if (!it->is_number()) + throw std::runtime_error(std::string(where) + ".eps must be a number"); + out.eps = it->get(); + } + return out; +} + +inline void real_dims(const json & j, const char * where, int64_t & dim, int64_t & real_dim) { + dim = positive(j, "dim", where); + real_dim = j.contains("real_dim") ? positive(j, "real_dim", where) : dim; + if (real_dim > dim) + throw std::runtime_error(std::string(where) + ".real_dim exceeds dim"); +} + +} // namespace detail + +// Parses the whole ggml_compiler.meta JSON and validates its "vla" object. +// Returns false with err set when there is no usable manifest. +inline bool parse_manifest(const std::string & meta_json, Manifest & out, std::string & err) { + using namespace detail; + try { + const json meta = json::parse(meta_json); + const auto it = meta.find("vla"); + if (it == meta.end() || !it->is_object()) + throw std::runtime_error("metadata has no 'vla' object: the program was compiled without a serving manifest " + "(Program(metadata={'vla': ggml_compiler.vla.manifest(...)}))"); + const json & v = *it; + Manifest m; + const json & ver = need(v, "version", "vla"); + if (!ver.is_number_integer() || ver.get() != kManifestVersion) + throw std::runtime_error("vla.version " + ver.dump() + " is not " + std::to_string(kManifestVersion) + + ", the only version this build reads"); + m.version = kManifestVersion; + + const json & images = need(v, "images", "vla"); + if (!images.is_array()) + throw std::runtime_error("vla.images must be an array"); + for (size_t i = 0; i < images.size(); ++i) { + const std::string where = "vla.images[" + std::to_string(i) + "]"; + const json & im = images[i]; + if (!im.is_object()) + throw std::runtime_error(where + " must be an object"); + ImageSpec s; + s.input = str(im, "input", where.c_str()); + s.mask = opt_str(im, "mask"); + s.h = positive(im, "height", where.c_str()); + s.w = positive(im, "width", where.c_str()); + const std::string layout = opt_str(im, "layout").empty() ? "NCHW" : opt_str(im, "layout"); + if (layout != "NCHW") + throw std::runtime_error(where + ".layout '" + layout + "' is not NCHW"); + const std::string fit = opt_str(im, "fit").empty() ? "exact" : opt_str(im, "fit"); + if (fit == "resize") + s.resize = true; + else if (fit != "exact") + throw std::runtime_error(where + ".fit '" + fit + "' is not exact or resize"); + const std::vector mean = im.contains("mean") ? floats(im, "mean", where.c_str(), 3) + : std::vector{0.f, 0.f, 0.f}; + const std::vector std_ = im.contains("std") ? floats(im, "std", where.c_str(), 3) + : std::vector{1.f, 1.f, 1.f}; + for (int c = 0; c < 3; ++c) { + if (std_[(size_t) c] == 0.f) + throw std::runtime_error(where + ".std has a zero"); + s.mean[c] = mean[(size_t) c]; + s.std_[c] = std_[(size_t) c]; + } + m.images.push_back(std::move(s)); + } + + if (const auto l = v.find("lang"); l != v.end() && !l->is_null()) { + if (!l->is_object()) + throw std::runtime_error("vla.lang must be an object"); + LangSpec s; + s.tokens = str(*l, "tokens", "vla.lang"); + s.mask = opt_str(*l, "mask"); + s.max_len = positive(*l, "max_len", "vla.lang"); + if (const auto p = l->find("pad_id"); p != l->end()) { + if (!p->is_number_integer()) + throw std::runtime_error("vla.lang.pad_id must be an integer"); + s.pad_id = p->get(); + } + m.lang = std::move(s); + } + + if (const auto st = v.find("state"); st != v.end() && !st->is_null()) { + if (!st->is_object()) + throw std::runtime_error("vla.state must be an object"); + StateSpec s; + s.input = str(*st, "input", "vla.state"); + real_dims(*st, "vla.state", s.dim, s.real_dim); + s.norm = norm(*st, "vla.state", (size_t) s.real_dim); + m.state = std::move(s); + } + + if (const auto n = v.find("noise"); n != v.end() && !n->is_null()) { + if (!n->is_object()) + throw std::runtime_error("vla.noise must be an object"); + m.noise = str(*n, "input", "vla.noise"); + } + + const json & a = need(v, "actions", "vla"); + if (!a.is_object()) + throw std::runtime_error("vla.actions must be an object"); + m.actions.output = str(a, "output", "vla.actions"); + m.actions.chunk = positive(a, "chunk", "vla.actions"); + real_dims(a, "vla.actions", m.actions.dim, m.actions.real_dim); + m.actions.norm = norm(a, "vla.actions", (size_t) m.actions.real_dim); + + std::vector names = m.input_names(); + std::sort(names.begin(), names.end()); + for (size_t i = 1; i < names.size(); ++i) + if (names[i] == names[i-1]) + throw std::runtime_error("vla names input '" + names[i] + "' more than once"); + + out = std::move(m); + return true; + } catch (const std::exception & e) { + err = e.what(); + return false; + } +} + +} // namespace compiled +} // namespace vla diff --git a/src/serving/vla-cli.cpp b/src/serving/vla-cli.cpp index aa55f76e..e5503b18 100644 --- a/src/serving/vla-cli.cpp +++ b/src/serving/vla-cli.cpp @@ -134,6 +134,7 @@ const char * arch_slug(Arch a) { case Arch::TURBOVLA: return "turbovla"; case Arch::ACT: return "act"; case Arch::COSMOS3_POLICY: return "cosmos3_policy"; + case Arch::COMPILED: return "compiled"; } return ""; } @@ -337,13 +338,14 @@ int main(int argc, char ** argv) { std::fprintf(stderr, "vla-cli: cannot detect the arch of %s\n", ckpt.c_str()); return 1; } - // ACT reads no instruction; every other arch needs one. - const bool takes_lang = arch != Arch::ACT; + // ACT reads no instruction; every other built-in arch needs one. A compiled + // program says in its manifest, which predict() checks after the load. + const bool takes_lang = arch != Arch::ACT && arch != Arch::COMPILED; if (takes_lang && tokens_s.empty() && text_s.empty()) { usage(argv[0]); return 1; } - if (!takes_lang && (!tokens_s.empty() || !text_s.empty())) { + if (arch == Arch::ACT && (!tokens_s.empty() || !text_s.empty())) { std::fprintf(stderr, "vla-cli: %s takes no instruction; drop --text/--tokens\n", arch_slug(arch)); return 1; } @@ -352,6 +354,11 @@ int main(int argc, char ** argv) { ckpt.c_str(), arch_slug(arch)); return 1; } + if (arch == Arch::COMPILED && !text_s.empty()) { + std::fprintf(stderr, "vla-cli: a compiled program carries no tokenizer; tokenize the instruction " + "yourself and pass --tokens\n"); + return 1; + } // Validate the cheap args before loading the model. std::vector lang; std::vector attn; // Octo only; empty leaves Inputs::attention_mask null. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e39f75ad..76ae6180 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,6 +13,11 @@ add_executable(vla_predict_check predict_check.cpp) target_link_libraries(vla_predict_check PRIVATE vla_core) target_compile_options(vla_predict_check PRIVATE -Wall -Wextra) +add_executable(test_compiled_manifest test_compiled_manifest.cpp) +target_include_directories(test_compiled_manifest PRIVATE ${CMAKE_SOURCE_DIR}/src ${llama_SOURCE_DIR}/vendor) +target_compile_options(test_compiled_manifest PRIVATE -Wall -Wextra) +add_test(NAME compiled_manifest COMMAND test_compiled_manifest) + add_executable(test_vision_common test_vision_common.cpp) target_include_directories(test_vision_common PRIVATE ${CMAKE_SOURCE_DIR}/src) target_compile_options(test_vision_common PRIVATE -Wall -Wextra) diff --git a/tests/test_compiled_manifest.cpp b/tests/test_compiled_manifest.cpp new file mode 100644 index 00000000..ad367431 --- /dev/null +++ b/tests/test_compiled_manifest.cpp @@ -0,0 +1,164 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The `vla` serving manifest parser: a full manifest, the defaults, the +// normalizers' round trip, and the errors a bad manifest must name. + +#include "models/compiled_manifest.h" + +#include +#include +#include + +using vla::compiled::Manifest; +using vla::compiled::Norm; +using vla::compiled::parse_manifest; + +static int g_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_fail; \ + } \ + } while (0) + +static bool parses(const std::string & vla_json, Manifest & m, std::string & err) { + return parse_manifest("{\"other\": 1, \"vla\": " + vla_json + "}", m, err); +} + +static bool rejects(const std::string & vla_json, const char * needle) { + Manifest m; + std::string err; + if (parses(vla_json, m, err)) { + std::fprintf(stderr, "FAIL: accepted %s\n", vla_json.c_str()); + return false; + } + if (err.find(needle) == std::string::npos) { + std::fprintf(stderr, "FAIL: error '%s' lacks '%s'\n", err.c_str(), needle); + return false; + } + return true; +} + +static const char * kFull = R"({ + "version": 1, + "images": [ + {"input": "images.0", "height": 512, "width": 512, "layout": "NCHW", "fit": "exact", + "mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5], "mask": "img_masks.0"}, + {"input": "images.1", "height": 240, "width": 320, "fit": "resize"} + ], + "lang": {"tokens": "lang_tokens", "mask": "lang_masks", "max_len": 48, "pad_id": 7}, + "state": {"input": "state", "dim": 32, "real_dim": 2, "norm": "mean_std", "mean": [1, 2], "std": [3, 4], "eps": 0.5}, + "noise": {"input": "noise"}, + "actions": {"output": "x", "chunk": 50, "dim": 32, "real_dim": 2, "norm": "min_max", "min": [0, -1], "max": [1, 1]} +})"; + +static const char * kAct = R"({ + "version": 1, + "images": [{"input": "front", "height": 480, "width": 640}], + "state": {"input": "state", "dim": 6, "norm": "none"}, + "actions": {"output": "actions", "chunk": 100, "dim": 6} +})"; + +int main() { + { + Manifest m; + std::string err; + CHECK(parses(kFull, m, err)); + if (!err.empty()) std::fprintf(stderr, "%s\n", err.c_str()); + CHECK(m.version == 1); + CHECK(m.images.size() == 2); + CHECK(m.images[0].input == "images.0" && m.images[0].mask == "img_masks.0"); + CHECK(m.images[0].h == 512 && m.images[0].w == 512 && !m.images[0].resize); + CHECK(m.images[0].mean[1] == 0.5f && m.images[0].std_[2] == 0.5f); + CHECK(m.images[1].h == 240 && m.images[1].w == 320 && m.images[1].resize && m.images[1].mask.empty()); + CHECK(m.images[1].mean[0] == 0.f && m.images[1].std_[0] == 1.f); + CHECK(m.lang && m.lang->tokens == "lang_tokens" && m.lang->mask == "lang_masks"); + CHECK(m.lang && m.lang->max_len == 48 && m.lang->pad_id == 7); + CHECK(m.state && m.state->dim == 32 && m.state->real_dim == 2); + CHECK(m.state && m.state->norm.kind == Norm::MEAN_STD && m.state->norm.eps == 0.5f); + CHECK(m.state && std::fabs(m.state->norm.normalize(8.f, 1) - (8.f - 2.f)/(4.f + 0.5f)) < 1e-6f); + CHECK(m.noise == "noise"); + CHECK(m.actions.output == "x" && m.actions.chunk == 50 && m.actions.dim == 32 && m.actions.real_dim == 2); + CHECK(m.actions.norm.kind == Norm::MIN_MAX); + // min_max: y = 2 (x - min) / (max - min + eps) - 1 and back. + const float x = 0.25f; + const float y = m.actions.norm.normalize(x, 1); + CHECK(std::fabs(y - (2.f*(0.25f + 1.f)/(2.f + 1e-8f) - 1.f)) < 1e-6f); + CHECK(std::fabs(m.actions.norm.denormalize(y, 1) - x) < 1e-6f); + const std::vector names = m.input_names(); + const std::vector want = {"images.0", "img_masks.0", "images.1", "lang_tokens", + "lang_masks", "state", "noise"}; + CHECK(names == want); + } + { + Manifest m; + std::string err; + CHECK(parses(kAct, m, err)); + CHECK(!m.lang && m.noise.empty()); + CHECK(m.state && m.state->real_dim == 6 && m.state->norm.kind == Norm::NONE); + CHECK(m.state && m.state->norm.normalize(3.f, 0) == 3.f); + CHECK(m.actions.real_dim == 6 && m.actions.norm.kind == Norm::NONE); + CHECK(m.actions.norm.denormalize(2.f, 5) == 2.f); + } + { + // mean_std round trip with LeRobot's eps. + Norm n; + n.kind = Norm::MEAN_STD; + n.a = {10.f}; + n.b = {2.f}; + const float y = n.normalize(14.f, 0); + CHECK(std::fabs(y - 4.f/(2.f + 1e-8f)) < 1e-6f); + CHECK(std::fabs(n.denormalize(y, 0) - 14.f) < 1e-5f); + } + { + Manifest m; + std::string err; + CHECK(!parse_manifest("{\"eos\": [1]}", m, err) && err.find("no 'vla'") != std::string::npos); + CHECK(!parse_manifest("not json", m, err)); + } + CHECK(rejects(R"({"version": 2, "images": [], "actions": {"output": "a", "chunk": 1, "dim": 1}})", "version")); + CHECK(rejects(R"({"version": 1, "actions": {"output": "a", "chunk": 1, "dim": 1}})", "images")); + CHECK(rejects(R"({"version": 1, "images": []})", "actions")); + CHECK(rejects(R"({"version": 1, "images": [{"input": "i", "height": 0, "width": 4}], + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "height")); + CHECK(rejects(R"({"version": 1, "images": [{"input": "i", "height": 4, "width": 4, "fit": "pad"}], + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "fit")); + CHECK(rejects(R"({"version": 1, "images": [{"input": "i", "height": 4, "width": 4, "layout": "NHWC"}], + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "layout")); + CHECK(rejects(R"({"version": 1, "images": [{"input": "i", "height": 4, "width": 4, "mean": [0.5]}], + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "3 numbers")); + CHECK(rejects(R"({"version": 1, "images": [], "state": {"input": "s", "dim": 2, "real_dim": 3}, + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "real_dim")); + CHECK(rejects(R"({"version": 1, "images": [], "state": {"input": "s", "dim": 2, "norm": "mean_std", "mean": [0, 0]}, + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "std")); + CHECK(rejects(R"({"version": 1, "images": [], "state": {"input": "s", "dim": 2, "norm": "quantile"}, + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "norm")); + CHECK(rejects(R"({"version": 1, "images": [], "actions": {"output": "a", "chunk": 1, "dim": 2, "norm": "mean_std", + "mean": [0, 0], "std": [1]}})", "std")); + CHECK(rejects(R"({"version": 1, "images": [{"input": "x", "height": 4, "width": 4}], + "state": {"input": "x", "dim": 2}, "actions": {"output": "a", "chunk": 1, "dim": 1}})", + "more than once")); + CHECK(rejects(R"({"version": 1, "images": [], "lang": {"tokens": "t", "max_len": 0}, + "actions": {"output": "a", "chunk": 1, "dim": 1}})", "max_len")); + + if (g_fail) { + std::fprintf(stderr, "%d check(s) failed\n", g_fail); + return 1; + } + std::printf("compiled_manifest: ok\n"); + return 0; +}