From c6277ed30a47d2a94861f45d34de31c44f7b825b Mon Sep 17 00:00:00 2001 From: thanhdq1o2 Date: Mon, 9 Nov 2026 17:56:26 +0700 Subject: [PATCH 1/4] loader: WeightLoader::GemmScope for a per-module GEMM type GEMM weights declared while a GemmScope lives take its type instead of the loader's, and the loader's type comes back when it ends. An arch can then keep one module in a different resident type from the rest. --- src/loader.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/loader.h b/src/loader.h index cec75760..bb11edb8 100644 --- a/src/loader.h +++ b/src/loader.h @@ -102,6 +102,20 @@ class WeightLoader { ggml_type gemm_type() const { return gemm_; } + + // GEMM weights declared while this lives take type t instead of the + // loader's; the previous type comes back when it goes out of scope. + class GemmScope { + public: + GemmScope(WeightLoader & L, ggml_type t) : L_(L), prev_(L.gemm_) { L.gemm_ = t; } + ~GemmScope() { L_.gemm_ = prev_; } + GemmScope(const GemmScope &) = delete; + GemmScope & operator=(const GemmScope &) = delete; + private: + WeightLoader & L_; + ggml_type prev_; + }; + bool ok() const { return ok_; } From fd3d02b2e1f4f2fba35c1c007c74b84c979db709 Mon Sep 17 00:00:00 2001 From: thanhdq1o2 Date: Mon, 9 Nov 2026 17:56:26 +0700 Subject: [PATCH 2/4] hexagon: let an arch keep a weight's matmuls on the CPU fallback_keep_on_cpu() registers a weight whose products the accelerator computes too coarsely for an arch; accel_runs() then leaves every MUL_MAT that reads it to the CPU. Hexagon's matrix unit rounds the activations to F16 whatever the weight type, so a weight registered in F32 gets the CPU F32 product. backend.h adds hmx_f16_activations() and keep_matmul_on_cpu(), which does nothing in other builds. --- src/backend.h | 24 ++++++++++++++++++++++++ src/backend_fallback.cpp | 21 +++++++++++++++++++++ src/backend_fallback.h | 10 ++++++++++ 3 files changed, 55 insertions(+) diff --git a/src/backend.h b/src/backend.h index 60a51755..f2244332 100644 --- a/src/backend.h +++ b/src/backend.h @@ -191,6 +191,30 @@ inline ggml_tensor * fa_kv([[maybe_unused]] ggml_context * ctx, ggml_tensor * t) #endif } +/** + * @brief Whether the compiled-in backend's matrix unit multiplies with F16 + * activations whatever the weight type (Hexagon's HMX). + */ +constexpr bool hmx_f16_activations() { +#ifdef GGML_USE_HEXAGON + return true; +#else + return false; +#endif +} + +/** + * @brief Keep the matmuls with weight @p w on the CPU where the compiled-in + * backend would round their activations to F16 (Hexagon's matrix unit). + * + * See fallback_keep_on_cpu(). Elsewhere this does nothing. + */ +inline void keep_matmul_on_cpu([[maybe_unused]] const ggml_tensor * w) { +#ifdef GGML_USE_HEXAGON + fallback_keep_on_cpu(w); +#endif +} + /** * @brief `ggml_concat(a, b, 2)` of [d0, d1, n] tensors, as the compiled-in * backend runs it. diff --git a/src/backend_fallback.cpp b/src/backend_fallback.cpp index a4eaed17..bb1313a7 100644 --- a/src/backend_fallback.cpp +++ b/src/backend_fallback.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,17 @@ namespace vla { namespace { +// fallback_keep_on_cpu(): weights by address, with the name to tell a freed +// tensor's address reused by another apart. +std::mutex g_cpu_mu; +std::unordered_map g_cpu_weights; + +bool kept_on_cpu(const ggml_tensor * w) { + std::lock_guard lock(g_cpu_mu); + auto it = g_cpu_weights.find(w); + return it != g_cpu_weights.end() && it->second == w->name; +} + enum class Where : uint8_t { Accel, Cpu, Either }; struct FallbackCtx { @@ -302,6 +314,10 @@ bool concat_is_dma_row_copy(const ggml_tensor * t) { // op it claims and computes differently from the CPU reference is refused here. bool accel_runs(FallbackCtx * fc, const ggml_tensor * t, const char ** why) { *why = "rejects"; + if (t->op == GGML_OP_MUL_MAT && kept_on_cpu(t->src[0])) { + *why = "leaves to the CPU a registered weight's"; + return false; + } if (fc->hexagon && t->op == GGML_OP_UNARY && ggml_get_unary_op(t) == GGML_UNARY_OP_GELU) { // ggml-hexagon maps GELU and GELU_QUICK to one kernel, x*sigmoid(1.702x), // which is GELU_QUICK. ggml_gelu is the tanh form; on SmolVLA's SigLIP @@ -527,4 +543,9 @@ void hexagon_default_skel_path() { #endif } +void fallback_keep_on_cpu(const ggml_tensor * w) { + std::lock_guard lock(g_cpu_mu); + g_cpu_weights[w] = w->name; +} + } // namespace vla diff --git a/src/backend_fallback.h b/src/backend_fallback.h index e42c4a2a..2b1d162d 100644 --- a/src/backend_fallback.h +++ b/src/backend_fallback.h @@ -49,6 +49,16 @@ namespace vla { */ ggml_backend_t fallback_backend_new(ggml_backend_t accel, int n_threads); +/** + * @brief Keep every matmul whose weight is @p w on the CPU. + * + * For a weight whose products the accelerator computes too coarsely for the + * arch: Hexagon's matrix unit rounds the activations to F16. An arch registers + * such weights while it loads them, in F32 so that the CPU product is the F32 + * reference's. + */ +void fallback_keep_on_cpu(const ggml_tensor * w); + /** * @brief Point the Hexagon FastRPC loader at this executable's directory. * From f0b12ae356a2e75c2c0509694726100e30897f43 Mon Sep 17 00:00:00 2001 From: thanhdq1o2 Date: Mon, 9 Nov 2026 17:56:26 +0700 Subject: [PATCH 3/4] gr00tn1d6: SigLIP2's first blocks in F32 on Hexagon Hexagon's matrix unit multiplies with F16 activations, and in SigLIP2's first blocks that rounding grows through the tower: a served LIBERO observation lands 1.0e-2 from a CPU-only F32 build on the Dragonwing IQ-10, above the 2.9e-3 bar. F32 weights alone leave it at 5.4e-3. Unless --weight-dtype is given, the tower and its projector now keep F32 weights on Hexagon and the first three blocks' matmuls run on the CPU. Other backends are unchanged. GR00T N1.6 goes from 1.0e-2 to 8.9e-4 served, 9.4e-4 and 5.0e-4 on fixed inputs (one and two views), at 689.5 to 739.7 ms (mean, vla-bench). The other archs give the same actions. --- src/models/gr00tn1d6.cpp | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index 963ce5a8..c2d4dfb8 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -274,14 +274,31 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, WeightLoader L("gr00tn1d6", g, m->ctx_weights, m->matmul_type); - m->vit.declare(L, "vit", m->vit_layers, /*patch_embd_is_gemm=*/true); - - m->mm_ln_w = L.f32 ("mm.ln.weight"); - m->mm_ln_b = L.f32 ("mm.ln.bias"); - m->mm_fc1_w = L.gemm("mm.fc1.weight"); - m->mm_fc1_b = L.f32 ("mm.fc1.bias"); - m->mm_fc2_w = L.gemm("mm.fc2.weight"); - m->mm_fc2_b = L.f32 ("mm.fc2.bias"); + // Where the matrix unit multiplies with F16 activations (Hexagon), the + // rounding in SigLIP2's first blocks grows through the tower: a served + // LIBERO observation lands 1.0e-2 from the CPU F32 reference on the + // Dragonwing IQ-10. Unless --weight-dtype is given, the tower and its + // projector keep F32 weights there and the first three blocks' matmuls run + // on the CPU: 8.9e-4, 7% slower. + const bool precise_vit = vla::hmx_f16_activations() && !opts.weight_dtype.has_value(); + { + WeightLoader::GemmScope vit_type(L, precise_vit ? GGML_TYPE_F32 : L.gemm_type()); + m->vit.declare(L, "vit", m->vit_layers, /*patch_embd_is_gemm=*/true); + + m->mm_ln_w = L.f32 ("mm.ln.weight"); + m->mm_ln_b = L.f32 ("mm.ln.bias"); + m->mm_fc1_w = L.gemm("mm.fc1.weight"); + m->mm_fc1_b = L.f32 ("mm.fc1.bias"); + m->mm_fc2_w = L.gemm("mm.fc2.weight"); + m->mm_fc2_b = L.f32 ("mm.fc2.bias"); + } + if (precise_vit) + for (size_t i = 0; i < 3 && i < m->vit.enc.blk.size(); ++i) { + const EncBlockW & w = m->vit.enc.blk[i]; + for (ggml_tensor * t : {w.Wq, w.Wk, w.Wv, w.Wqkv, w.Wo, w.Wfc1, w.Wfc2}) + if (t) + vla::keep_matmul_on_cpu(t); + } m->lm.declare(L, "vlm", fq.present ? &fq.llm : nullptr); From 634e8a15be178f69ed69fb88715ad7b3bb48ee04 Mon Sep 17 00:00:00 2001 From: thanhdq1o2 Date: Tue, 10 Nov 2026 09:23:15 +0700 Subject: [PATCH 4/4] docs: GR00T N1.6 within the bar served, IQ-10 benchmark at f0b12ae hexagon.md: the HMX rounds activations to F16 whatever the weight type; GR00T N1.6's SigLIP2 tower keeps F32 weights and its first three blocks' matmuls on the CPU, 1.0e-2 to 8.9e-4 served. GR00T N1.5 spreads the same rounding over the model: no single module on the CPU brings it under the bar, only every matmul does. Fidelity and serving tables updated. qualcomm-iq10.md and the summary row at f0b12ae: GR00T N1.6 682.1 to 733.7 ms, the other archs within run-to-run noise. --- CHANGELOG.md | 6 ++- docs/backend/hexagon.md | 27 ++++++---- docs/benchmark/README.md | 4 +- docs/benchmark/qualcomm-iq10.md | 93 +++++++++++++++++---------------- 4 files changed, 71 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94526581..3a54e77e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,10 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke run on the HMX: OpenVLA-OFT's LM, Octo's block transformer, Evo-1's DiT cross-attention and GR00T's DiT (OpenVLA-OFT 35% faster, Evo-1 29%, Octo 29%, GR00T N1.5 11%); VLA-Adapter's LM only for V. Fidelity is measured - against a CPU-only build of the same commit. GR00T N1.5 and N1.6 are above - the bar on a served LIBERO observation, as the CPU is at F16. See + against a CPU-only build of the same commit. GR00T N1.6 keeps its SigLIP2 + tower in F32 weights and its first three blocks' matmuls on the CPU, which + brings it within the bar on a served LIBERO observation (8.9e-4, 7% slower); + GR00T N1.5 stays above it there, as the CPU is at F16. See `docs/backend/hexagon.md` for the build, the skel install and the numbers. - **ROCm/HIP backend for VLA inference.** `-DGGML_HIP=ON` selects ggml's HIP backend without compiling vla.cpp's CUDA-only kernels. On Linux `gfx1200` diff --git a/docs/backend/hexagon.md b/docs/backend/hexagon.md index d87a1ee2..88586103 100644 --- a/docs/backend/hexagon.md +++ b/docs/backend/hexagon.md @@ -149,6 +149,16 @@ VLA-JEPA's LM does the same: with flash attention a served LIBERO observation lands 6.2e-3 from the reference, without it 2.2e-3, at a cost of 683 to 810 ms with two views. +The HMX also multiplies with F16 activations whatever the weight type. In GR00T +N1.6's SigLIP2 tower that rounding grows from the first blocks: a served LIBERO +observation landed 1.0e-2 from the reference, and F32 weights alone left it at +5.4e-3. On Hexagon the tower and its projector keep F32 weights and the first +three blocks' matmuls run on the CPU (`fallback_keep_on_cpu()`): 8.9e-4, at 682 +to 734 ms. GR00T N1.5 spreads the same rounding over the whole model: with F32 +weights, running its vision tower, LM, VL self-attention or DiT on the CPU +leaves it at 1.1e-2 to 1.3e-2 each, and only every matmul on the CPU, with +flash attention off, brings it under the bar (1.8e-3). + The softmax takes rows of up to 32 elements or a multiple of 32. Where a sequence can grow without changing its result, vla.cpp pads it to `gemm_width()` on Hexagon rather than send every layer's softmax to the CPU: @@ -165,7 +175,7 @@ tower takes two camera views that way: its action chunk lands 0.16 from the CPU reference. The wrapper runs those matmuls on the CPU. On Hexagon `linear()` in `src/layers/linear.h` multiplies a contiguous batch as one 2D matrix of rows instead, the same product, so most of them never reach the rule: GR00T N1.6 -with two views lands 7.5e-4 from the reference, and the matmuls of TurboVLA's +with two views lands 5.0e-4 from the reference, and the matmuls of TurboVLA's DINOv3, which takes its two views as a batch, run on the NPU. A copy that transposes the innermost dim, such as the `CONT` of a permuted V @@ -382,7 +392,7 @@ differ from the benchmark's for VLA-JEPA, VLA-Adapter, OpenVLA-OFT and Octo: | π0.5 | 2 | 4.8e-4 | | Evo-1 | 1 | 6.2e-4 | | GR00T N1.5 | 1 | 1.9e-3 | -| GR00T N1.6 | 1 | 1.3e-3 | +| GR00T N1.6 | 1 | 9.4e-4 | | GR00T N1.7 | 1 | 2.0e-3 | | VLA-JEPA | 2 | 2.1e-3 | | VLA-Adapter | 2 | 3.9e-4 | @@ -391,7 +401,7 @@ differ from the benchmark's for VLA-JEPA, VLA-Adapter, OpenVLA-OFT and Octo: | TurboVLA | 2 | 3.7e-4 | | ACT | 1 | 6.1e-4 | -GR00T N1.6 with two views is 7.5e-4. At the benchmark's single view VLA-JEPA is +GR00T N1.6 with two views is 5.0e-4. At the benchmark's single view VLA-JEPA is 1.8e-3, VLA-Adapter 4.8e-4 and OpenVLA-OFT 1.8e-4. ACT takes its one 640x480 view, the benchmark's input. @@ -414,7 +424,7 @@ requests carrying one fixed 640x480 view and a fixed 14-dim state. | π0.5 | 3.7e-4 | | | Evo-1 | 1.1e-3 | | | GR00T N1.5 | 1.2e-2 | 2.8e-2 | -| GR00T N1.6 | 1.0e-2 | 9.0e-3 | +| GR00T N1.6 | 8.9e-4 | | | GR00T N1.7 | 2.0e-3 | | | π0 | 5.8e-4 | | | VLA-JEPA | 2.2e-3 | | @@ -424,11 +434,10 @@ requests carrying one fixed 640x480 view and a fixed 14-dim state. | TurboVLA | 8.5e-4 | | | ACT | 1.0e-3 | | -GR00T N1.5 and N1.6 are above the bar on this input, and so is the CPU itself -with F16 weights: N1.5 lands closer to the reference than the CPU at F16, N1.6 -slightly further (1.0e-2 against 9.0e-3). They are sensitive to 16-bit precision -rather than miscomputed. Running them with F32 weights on the NPU has not been -tried yet. +GR00T N1.5 is above the bar on this input, and so is the CPU itself with F16 +weights (2.8e-2). It is sensitive to 16-bit precision rather than miscomputed: +F32 weights on the NPU leave it at 1.2e-2, since the HMX rounds the activations +(see above). ## IQ-10, imagined diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index 50791593..e03ec3ed 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -12,7 +12,7 @@ AGX Orin row ran at `e6653e8`, on the bit-exact CUDA path ran at `d0e5d00`, PR #35 merged with main `f464653` (same llama.cpp pin), with fixed-input checks against the CPU build in [backend/rocm.md](../backend/rocm.md). The Qualcomm Dragonwing IQ-10 row ran at -`bd166b8` (same llama.cpp pin). Each report +`f0b12ae` (same llama.cpp pin). Each report lists the device, the build, each model's configuration, latency and peak memory at the defaults, and the fastest runtime flags for that device. @@ -36,7 +36,7 @@ means the model does not fit or fails there; each report says which and why. | [Intel Arc B390 iGPU (OpenVINO)](core-ultra-x7-358h.md) | N/A | n/r | 28.5 | 116.3 | 170.2 | 106.0 | 122.8 | 131.4 | N/A | 213.1 | 620.5 | 232.5 | 1262.5 | 566.0 | | [Intel Arc A380 (SYCL)](arc-a380.md) | 60.5 | n/r | 103.5 | 355.9 | 408.9 | 472.0 | 646.0 | 615.3 | N/A | 734.5 | 921.9 | 945.6 | - | 1047.3 | | [Snapdragon X Hexagon NPU](snapdragon-x-hexagon.md) | 239.7 | n/r | 2280.8 | 821.0 | 4072.1 | 2755.7 | 2940.0 | 1400.2 | N/A | 1239.3 | 5730.0 | 9321.7 | - | 7208.6 | -| [Qualcomm Dragonwing IQ-10 Hexagon NPU](qualcomm-iq10.md) | 81.7 | 147.8 | 184.6 | 489.7 | 664.0 | 530.2 | 680.9 | 739.6 | N/A | 583.3 | 1117.6 | 1114.0 | 1117.7 | 1653.5 | +| [Qualcomm Dragonwing IQ-10 Hexagon NPU](qualcomm-iq10.md) | 81.2 | 147.7 | 184.0 | 490.2 | 662.0 | 524.7 | 731.7 | 739.2 | N/A | 583.5 | 1127.8 | 1103.9 | 1121.4 | 1650.2 | | [Intel AI Boost NPU (OpenVINO)](core-ultra-x7-358h.md) | N/A | n/r | 23.5 | 90.7 | 126.7 | 134.0 | 161.1 | 144.9 | N/A | 161.3 | 438.3 | 430.2 | 405.8 | 256.3 | | [Intel Core i7-14700F (CPU)](core-i7-14700f.md) | 39.8 | n/r | 195.1 | 825.5 | 1140.6 | 1913.8 | 1627.9 | 1453.4 | N/A | 1689.2 | 6455.7 | 6068.1 | 9869.0 | 4159.6 | | [Intel Core i9-14900HX (CPU)](core-i9-14900hx.md) | 67.0 | n/r | 314.7 | 1227.2 | 1659.3 | 1935.4 | 1688.9 | 1505.5 | N/A | 1822.6 | 6182.1 | 5735.0 | 9126.4 | 4118.2 | diff --git a/docs/benchmark/qualcomm-iq10.md b/docs/benchmark/qualcomm-iq10.md index d583607b..f3635a9a 100644 --- a/docs/benchmark/qualcomm-iq10.md +++ b/docs/benchmark/qualcomm-iq10.md @@ -15,9 +15,9 @@ through vla.cpp's fallback wrapper. | Power | CPU governor `performance` | | OS | Ubuntu 24.04.4 LTS, kernel 6.6.110 | | Toolchain | `ghcr.io/snapdragon-toolchain/arm64-linux:v0.7`: clang 21.1.8, Hexagon SDK 6.6.0.0 (tools 19.0.07) | -| Commit | `bd166b8` | +| Commit | `f0b12ae` | | llama.cpp | `b11223` | -| Date | 2026-11-06 10:46 to 11:16 (UTC+07) | +| Date | 2026-11-09 17:58 to 18:28 (UTC+07) | | Build | `scripts/build_linux_hexagon.sh` (servers included); the HTP skel processed for the board and installed in `/vendor/dsp/cdsp0..3` | | Runtime flags | defaults; the fastest flags per model are in the second table | | Method | 3 warmups + 20 timed reps per process, 3 processes. The process with the lowest mean is reported, and memory is the peak over all three. | @@ -69,19 +69,19 @@ vision stage separately. | Model | Views | Input | min ms | mean ms | p50 ms | p90 ms | vision ms | Peak RSS MiB | System RAM MiB | |---|--:|--:|--:|--:|--:|--:|--:|--:|--:| -| Octo-Small | 2 | 256 + 128 | 81.7 | 84.6 | 82.3 | 89.6 | 21.9 | 253 | 728 | -| ACT | 1 | 640 x 480 | 147.8 | 151.2 | 150.7 | 153.6 | - | 193 | 471 | -| TurboVLA | 2 | 256 | 184.6 | 191.8 | 190.3 | 199.4 | - | 249 | 1118 | -| VLA-JEPA | 1 | 256 | 489.7 | 495.8 | 494.7 | 501.6 | 196.4 | 184 | 4023 | -| GR00T N1.5 | 1 | 224 | 530.2 | 532.0 | 531.3 | 534.3 | 138.5 | 162 | 3734 | -| SmolVLA | 2 | 512 | 583.3 | 584.0 | 583.9 | 584.7 | 229.8 | 146 | 1194 | -| VLA-Adapter | 1 | 224 | 664.0 | 668.6 | 666.6 | 673.7 | 257.8 | 872 | 3584 | -| GR00T N1.6 | 1 | 224 | 680.9 | 682.1 | 681.6 | 683.9 | 145.3 | 146 | 4889 | -| GR00T N1.7 | 1 | 256 | 739.6 | 746.4 | 745.1 | 753.8 | 196.4 | 190 | 5215 | -| π0.5 | 2 | 224 | 1114.0 | 1116.1 | 1115.5 | 1117.8 | 278.0 | 285 | 5802 | -| π0 | 2 | 224 | 1117.6 | 1120.5 | 1119.9 | 1123.8 | 277.8 | 284 | 5547 | -| OpenVLA-OFT | 1 | 224 | 1117.7 | 1120.2 | 1120.4 | 1121.8 | 262.3 | 844 | 15321 | -| Evo-1 | 1 | 448 | 1653.5 | 1661.4 | 1661.2 | 1665.6 | 815.6 | 319 | 1556 | +| Octo-Small | 2 | 256 + 128 | 81.2 | 85.6 | 83.6 | 91.1 | 21.9 | 252 | 743 | +| ACT | 1 | 640 x 480 | 147.7 | 153.1 | 152.0 | 157.7 | - | 194 | 470 | +| TurboVLA | 2 | 256 | 184.0 | 191.9 | 191.4 | 197.1 | - | 249 | 1047 | +| VLA-JEPA | 1 | 256 | 490.2 | 498.4 | 497.9 | 505.6 | 199.0 | 184 | 4012 | +| GR00T N1.5 | 1 | 224 | 524.7 | 525.9 | 525.7 | 527.8 | 138.6 | 162 | 3644 | +| SmolVLA | 2 | 512 | 583.5 | 584.4 | 584.2 | 585.2 | 229.9 | 145 | 1194 | +| VLA-Adapter | 1 | 224 | 662.0 | 668.3 | 666.0 | 677.9 | 257.9 | 872 | 3600 | +| GR00T N1.6 | 1 | 224 | 731.7 | 733.7 | 733.1 | 735.1 | 194.5 | 274 | 5691 | +| GR00T N1.7 | 1 | 256 | 739.2 | 742.7 | 741.9 | 746.2 | 197.7 | 191 | 5049 | +| π0.5 | 2 | 224 | 1103.9 | 1106.8 | 1106.1 | 1109.9 | 277.2 | 285 | 5790 | +| OpenVLA-OFT | 1 | 224 | 1121.4 | 1123.0 | 1121.9 | 1126.0 | 262.4 | 844 | 15323 | +| π0 | 2 | 224 | 1127.8 | 1131.2 | 1130.8 | 1134.6 | 276.9 | 285 | 5531 | +| Evo-1 | 1 | 448 | 1650.2 | 1658.4 | 1656.5 | 1666.6 | 821.1 | 319 | 1553 | ### Fastest configuration @@ -109,40 +109,37 @@ inputs), N1.7 6.2e-3 on fixed inputs (2.7e-3 served). | Model | Fastest flags | min ms | mean ms | p50 ms | p90 ms | vision ms | Peak RSS MiB | System RAM MiB | vs defaults | |---|---|--:|--:|--:|--:|--:|--:|--:|--:| -| Octo-Small | *(defaults)* | 81.7 | 84.6 | 82.3 | 89.6 | 21.9 | 253 | 728 | - | -| ACT | *(defaults)* | 147.8 | 151.2 | 150.7 | 153.6 | - | 193 | 471 | - | -| TurboVLA | *(defaults)* | 184.6 | 191.8 | 190.3 | 199.4 | - | 249 | 1118 | - | -| VLA-JEPA | `--flash-attn` | 427.8 | 434.2 | 432.6 | 441.7 | 196.1 | 184 | 3971 | -12.4% | -| GR00T N1.5 | *(defaults)* | 530.2 | 532.0 | 531.3 | 534.3 | 138.5 | 162 | 3734 | - | -| SmolVLA | *(defaults)* | 583.3 | 584.0 | 583.9 | 584.7 | 229.8 | 146 | 1194 | - | -| VLA-Adapter | *(defaults)* | 664.0 | 668.6 | 666.6 | 673.7 | 257.8 | 872 | 3584 | - | -| GR00T N1.6 | *(defaults)* | 680.9 | 682.1 | 681.6 | 683.9 | 145.3 | 146 | 4889 | - | -| GR00T N1.7 | `--flash-attn --mm-prec default` | 715.1 | 719.2 | 718.7 | 722.9 | 195.5 | 190 | 5074 | -3.6% | -| π0.5 | *(defaults)* | 1114.0 | 1116.1 | 1115.5 | 1117.8 | 278.0 | 285 | 5802 | - | -| π0 | *(defaults)* | 1117.6 | 1120.5 | 1119.9 | 1123.8 | 277.8 | 284 | 5547 | - | -| OpenVLA-OFT | *(defaults)* | 1117.7 | 1120.2 | 1120.4 | 1121.8 | 262.3 | 844 | 15321 | - | -| Evo-1 | *(defaults)* | 1653.5 | 1661.4 | 1661.2 | 1665.6 | 815.6 | 319 | 1556 | - | +| Octo-Small | *(defaults)* | 81.2 | 85.6 | 83.6 | 91.1 | 21.9 | 252 | 743 | - | +| ACT | *(defaults)* | 147.7 | 153.1 | 152.0 | 157.7 | - | 194 | 470 | - | +| TurboVLA | *(defaults)* | 184.0 | 191.9 | 191.4 | 197.1 | - | 249 | 1047 | - | +| VLA-JEPA | `--flash-attn --mm-prec default` | 428.6 | 435.0 | 434.1 | 442.6 | 197.6 | 184 | 3971 | -12.7% | +| GR00T N1.5 | *(defaults)* | 524.7 | 525.9 | 525.7 | 527.8 | 138.6 | 162 | 3644 | - | +| SmolVLA | *(defaults)* | 583.5 | 584.4 | 584.2 | 585.2 | 229.9 | 145 | 1194 | - | +| VLA-Adapter | *(defaults)* | 662.0 | 668.3 | 666.0 | 677.9 | 257.9 | 872 | 3600 | - | +| GR00T N1.7 | `--flash-attn --mm-prec default` | 714.1 | 719.2 | 718.5 | 725.5 | 196.6 | 191 | 5081 | -3.2% | +| GR00T N1.6 | *(defaults)* | 731.7 | 733.7 | 733.1 | 735.1 | 194.5 | 274 | 5691 | - | +| π0.5 | *(defaults)* | 1103.9 | 1106.8 | 1106.1 | 1109.9 | 277.2 | 285 | 5790 | - | +| OpenVLA-OFT | *(defaults)* | 1121.4 | 1123.0 | 1121.9 | 1126.0 | 262.4 | 844 | 15323 | - | +| π0 | *(defaults)* | 1127.8 | 1131.2 | 1130.8 | 1134.6 | 276.9 | 285 | 5531 | - | +| Evo-1 | *(defaults)* | 1650.2 | 1658.4 | 1656.5 | 1666.6 | 821.1 | 319 | 1553 | - |
Screen means (ms) per flag set | Model | defaults | `--flash-attn` | `--mm-prec default` | `--flash-attn --mm-prec default` | `--flash-attn 0` | |---|--:|--:|--:|--:|--:| -| Octo-Small | 90.8* | 83.0 | 83.2 | 85.3 | 88.1 | -| ACT | 154.4 | 158.4 | 154.7 | 153.2 | 179.9 | -| TurboVLA | 192.0 | 192.6 | 194.0 | 191.9 | 257.3 | -| VLA-JEPA | 498.2 | 431.3 | 494.0 | 439.8 | 602.4 | -| GR00T N1.5 | 534.0 | 526.4 | 527.0 | 526.4 | 811.6 | -| SmolVLA | 584.4 | 584.7 | 585.2 | 585.4 | 2145.2 | -| VLA-Adapter | 671.6 | 670.4 | 669.4 | 671.9 | 950.8 | -| GR00T N1.6 | 689.2 | 685.8 | 687.3 | 683.0 | 851.0 | -| GR00T N1.7 | 758.3 | 733.1 | 752.8 | 723.2 | 858.2 | -| π0.5 | 1125.2 | 1117.6 | 1134.2 | 1134.2 | 2039.2 | -| π0 | 1113.2 | 1126.1 | 1114.8 | 1126.1 | 2056.3 | -| OpenVLA-OFT | 1125.5 | 1127.8 | 1123.4 | 1123.5 | 1402.2 | -| Evo-1 | 1668.8 | 1669.2 | 1669.8 | 1657.4 | 3116.8 | - -\* Octo-Small's defaults screen was re-run: the first run (672.8 ms, with a -241 ms vision stage) overlapped a transient slowdown on the board. +| Octo-Small | 84.5 | 84.9 | 86.0 | 87.9 | 83.1 | +| ACT | 155.5 | 155.2 | 158.4 | 147.8 | 176.9 | +| TurboVLA | 191.1 | 193.3 | 188.9 | 191.5 | 252.8 | +| VLA-JEPA | 501.1 | 441.9 | 504.7 | 439.9 | 598.9 | +| GR00T N1.5 | 525.8 | 525.2 | 526.4 | 525.6 | 809.1 | +| SmolVLA | 584.3 | 584.8 | 585.6 | 584.5 | 2145.5 | +| VLA-Adapter | 677.1 | 667.2 | 673.1 | 674.4 | 950.4 | +| GR00T N1.6 | 739.0 | 736.8 | 737.4 | 741.2 | 897.7 | +| GR00T N1.7 | 749.3 | 727.4 | 744.9 | 723.3 | 851.0 | +| π0.5 | 1125.3 | 1114.0 | 1133.7 | 1115.2 | 2053.2 | +| OpenVLA-OFT | 1127.0 | 1124.8 | 1128.0 | 1121.5 | 1397.1 | +| π0 | 1113.4 | 1113.5 | 1113.9 | 1113.7 | 2070.2 | +| Evo-1 | 1657.1 | 1657.0 | 1656.2 | 1662.9 | 3106.6 |
@@ -162,9 +159,13 @@ its SigLIP2 tower's patch matmul and flash attention, batched over the views, still run on the CPU; see [backend/hexagon.md](../backend/hexagon.md#op-coverage-measured). +GR00T N1.6's row runs its SigLIP2 tower with F32 weights and its first three +blocks' matmuls on the CPU, which keeps it within the fidelity bar when served; +the F32 tower is also why its System RAM is higher than N1.5's. + These rows time the defaults; they do not vouch for accuracy. On a served LIBERO -observation, GR00T N1.5 and N1.6 land above the 2.9e-3 fidelity bar: N1.5 -closer to the reference than the CPU at F16, N1.6 slightly further; see +observation, GR00T N1.5 lands above the 2.9e-3 fidelity bar, closer to the +reference than the CPU at F16; see [backend/hexagon.md](../backend/hexagon.md#serving). ## Reproducing