diff --git a/CHANGELOG-paperless.md b/CHANGELOG-paperless.md index 6fa73f92..6303b016 100644 --- a/CHANGELOG-paperless.md +++ b/CHANGELOG-paperless.md @@ -4,6 +4,15 @@ Changes carried by the `paperless` branch of `stumpylog/sqlite-vec` beyond upstr Each upstream PR is carried as one squash commit. Fixes or follow-ups to it are separate commits listed beneath it. Entries read: upstream PR, fork commit, summary. +## v0.1.10-paperless.4 (unreleased) + +Cheaper float32 cosine KNN on x86. Both changes are fork-only: they have no upstream PR. + +- [`0a241f9`](https://github.com/stumpylog/sqlite-vec/commit/0a241f9) Centralize AVX CPU capability detection into a `vec_avx_caps` struct (`avx`, `avx2`, `fma`) filled once in `sqlite3_vec_init`. The three AVX dispatch sites (l2, cosine, hamming) read it instead of each caching `__builtin_cpu_supports("avx2")` in a function-local static. The flags are 0 until init has run, which selects the scalar path. The idea and struct are from `feature/simd-additional` (`277aed8`); the rest of that branch is not carried. +- [`43e3c80`](https://github.com/stumpylog/sqlite-vec/commit/43e3c80) Compute the cosine query norm once per KNN query, not once per row, with an AVX2+FMA kernel that accumulates only dot and |a|^2 (gated on `vec_avx_caps.avx2` and `.fma`), and drop rows that cannot beat the k-th distance before `min_idx()`. On a 768-dimension float vector the kernel takes 59 ns against 87 ns for the #364 kernel with the data in L1, and 68 ns against 108 ns at a 3 MB working set. A full scan of 60k vectors runs 110M instructions instead of 182M. Wall time moves about 0 to 10%, because a full scan is bound by memory and page reads, not by the kernel. Float32 cosine distances can differ from the scalar and #364 results in the last digits. NEON is unchanged. + +Considered and not carried: the rest of `feature/simd-additional`. paperless-ngx only uses float32 cosine, and its `cosine_float_avx2` (`2c64ea3`) measured about 14% slower than the #364 kernel with the data in L1. + ## v0.1.10-paperless.3 NEON on Linux arm64, plus memory-safety and leak fixes. diff --git a/sqlite-vec.c b/sqlite-vec.c index 0890c6d5..b603e59f 100644 --- a/sqlite-vec.c +++ b/sqlite-vec.c @@ -166,6 +166,16 @@ enum VectorElementType { #define PORTABLE_ALIGN32 __attribute__((aligned(32))) #define PORTABLE_ALIGN64 __attribute__((aligned(64))) +// Runtime CPU capability flags, filled in sqlite3_vec_init from CPUID, which +// is deterministic, so writing them on every connection open is harmless. +// Dispatch sites read them with no per-call check. They stay 0 until init has +// run, which selects the scalar path. +static struct { + int avx; + int avx2; + int fma; +} vec_avx_caps = {0, 0, 0}; + __attribute__((target("avx,avx2"))) static f32 l2_sqr_float_avx(const void *pVect1v, const void *pVect2v, const void *qty_ptr) { @@ -251,6 +261,34 @@ static f32 cosine_float_avx(const void *pVect1v, const void *pVect2v, return 1 - (dot / (sqrt(aMag) * sqrt(bMag))); } #undef VEC_AVX_MULADD + +// Cosine distance with the query's squared magnitude (bMag) computed once by +// the caller, and fused multiply-add. Only dot and |a|^2 are accumulated, which +// is two thirds of the loads and arithmetic of cosine_float_avx. Callers must +// check vec_avx_caps.avx2 and vec_avx_caps.fma first. +__attribute__((target("avx,avx2,fma"))) +static f32 cosine_float_avx_fma_qn(const f32 *a, const f32 *b, size_t qty, + f32 bMag) { + const f32 *aEnd = a + (qty & ~(size_t)15); + __m256 dot0 = _mm256_setzero_ps(), dot1 = _mm256_setzero_ps(); + __m256 aa0 = _mm256_setzero_ps(), aa1 = _mm256_setzero_ps(); + while (a < aEnd) { + __m256 a0 = _mm256_loadu_ps(a), a1 = _mm256_loadu_ps(a + 8); + dot0 = _mm256_fmadd_ps(a0, _mm256_loadu_ps(b), dot0); + dot1 = _mm256_fmadd_ps(a1, _mm256_loadu_ps(b + 8), dot1); + aa0 = _mm256_fmadd_ps(a0, a0, aa0); + aa1 = _mm256_fmadd_ps(a1, a1, aa1); + a += 16; + b += 16; + } + f32 dot = hsum_float_avx(_mm256_add_ps(dot0, dot1)); + f32 aMag = hsum_float_avx(_mm256_add_ps(aa0, aa1)); + for (size_t i = qty & ~(size_t)15; i < qty; i++, a++, b++) { + dot += *a * *b; + aMag += *a * *a; + } + return 1 - (dot / (sqrt(aMag) * sqrt(bMag))); +} #endif #ifdef SQLITE_VEC_ENABLE_NEON @@ -549,9 +587,7 @@ static f32 distance_l2_sqr_float(const void *a, const void *b, const void *d) { } #endif #ifdef SQLITE_VEC_ENABLE_AVX - static int has_avx2 = -1; - if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2"); - if (has_avx2 && ((*(const size_t *)d) % 16 == 0)) { + if (vec_avx_caps.avx2 && ((*(const size_t *)d) % 16 == 0)) { return l2_sqr_float_avx(a, b, d); } #endif @@ -623,9 +659,7 @@ static f32 distance_cosine_float(const void *pVect1v, const void *pVect2v, } #endif #ifdef SQLITE_VEC_ENABLE_AVX - static int has_avx2 = -1; - if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2"); - if (has_avx2 && (*(const size_t *)qty_ptr) >= 16) { + if (vec_avx_caps.avx2 && (*(const size_t *)qty_ptr) >= 16) { return cosine_float_avx(pVect1v, pVect2v, qty_ptr); } #endif @@ -957,9 +991,7 @@ static f32 distance_hamming(const void *a, const void *b, const void *d) { } #endif #ifdef SQLITE_VEC_ENABLE_AVX - static int has_avx2 = -1; - if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2"); - if (has_avx2 && n_bytes >= 32) { + if (vec_avx_caps.avx2 && n_bytes >= 32) { return distance_hamming_avx2((const u8 *)a, (const u8 *)b, n_bytes); } #endif @@ -7811,6 +7843,21 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks, assert(numValueEntries == argc); int hasMetadataFilters = 0; int hasDistanceConstraints = 0; +#ifdef SQLITE_VEC_ENABLE_AVX + // The query's squared magnitude is the same for every row, so cosine + // computes it once per query rather than once per row. + int cosineAvxFma = 0; + f32 queryMagSq = 0; + if (vector_column->element_type == SQLITE_VEC_ELEMENT_TYPE_FLOAT32 && + vector_column->distance_metric == VEC0_DISTANCE_METRIC_COSINE && + vector_column->dimensions >= 16 && vec_avx_caps.avx2 && + vec_avx_caps.fma) { + cosineAvxFma = 1; + for (size_t j = 0; j < vector_column->dimensions; j++) { + queryMagSq += ((f32 *)queryVector)[j] * ((f32 *)queryVector)[j]; + } + } +#endif for(int i = 0; i < argc; i++) { int idx = 1 + (i * 4); char kind = idxStr[idx + 0]; @@ -7969,6 +8016,14 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks, break; } case VEC0_DISTANCE_METRIC_COSINE: { +#ifdef SQLITE_VEC_ENABLE_AVX + if (cosineAvxFma) { + result = cosine_float_avx_fma_qn(base_i, (f32 *)queryVector, + vector_column->dimensions, + queryMagSq); + break; + } +#endif result = distance_cosine_float(base_i, (f32 *)queryVector, &vector_column->dimensions); break; @@ -8062,6 +8117,18 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks, } } + // Once the global top k is full, a row that is not strictly nearer than + // the k-th (merge_sorted_lists keeps the earlier row on a tie, and ranks a + // NaN last) can never enter it, so it need not be a candidate. + if (k_used == k && !isnan(topk_distances[k - 1])) { + f32 worst = topk_distances[k - 1]; + for (int j = 0; j < p->chunk_size; j++) { + if (bitmap_get(b, j) && !(chunk_distances[j] < worst)) { + bitmap_set(b, j, 0); + } + } + } + int used1; min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs, min(k, p->chunk_size), &used1); @@ -11219,6 +11286,12 @@ SQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg, #endif int rc = SQLITE_OK; +#ifdef SQLITE_VEC_ENABLE_AVX + vec_avx_caps.avx = __builtin_cpu_supports("avx"); + vec_avx_caps.avx2 = __builtin_cpu_supports("avx2"); + vec_avx_caps.fma = __builtin_cpu_supports("fma"); +#endif + #define DEFAULT_FLAGS (SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC) rc = sqlite3_create_function_v2(db, "vec_version", 0, DEFAULT_FLAGS,