Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG-paperless.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ Changes carried by the `paperless` branch of `stumpylog/sqlite-vec` beyond upstr

Each upstream PR is carried as one squash commit. Fixes or follow-ups to it are separate commits listed beneath it. Entries read: upstream PR, fork commit, summary.

## v0.1.10-paperless.4 (unreleased)

Cheaper float32 cosine KNN on x86. Both changes are fork-only: they have no upstream PR.

- [`0a241f9`](https://github.com/stumpylog/sqlite-vec/commit/0a241f9) Centralize AVX CPU capability detection into a `vec_avx_caps` struct (`avx`, `avx2`, `fma`) filled once in `sqlite3_vec_init`. The three AVX dispatch sites (l2, cosine, hamming) read it instead of each caching `__builtin_cpu_supports("avx2")` in a function-local static. The flags are 0 until init has run, which selects the scalar path. The idea and struct are from `feature/simd-additional` (`277aed8`); the rest of that branch is not carried.
- [`43e3c80`](https://github.com/stumpylog/sqlite-vec/commit/43e3c80) Compute the cosine query norm once per KNN query, not once per row, with an AVX2+FMA kernel that accumulates only dot and |a|^2 (gated on `vec_avx_caps.avx2` and `.fma`), and drop rows that cannot beat the k-th distance before `min_idx()`. On a 768-dimension float vector the kernel takes 59 ns against 87 ns for the #364 kernel with the data in L1, and 68 ns against 108 ns at a 3 MB working set. A full scan of 60k vectors runs 110M instructions instead of 182M. Wall time moves about 0 to 10%, because a full scan is bound by memory and page reads, not by the kernel. Float32 cosine distances can differ from the scalar and #364 results in the last digits. NEON is unchanged.

Considered and not carried: the rest of `feature/simd-additional`. paperless-ngx only uses float32 cosine, and its `cosine_float_avx2` (`2c64ea3`) measured about 14% slower than the #364 kernel with the data in L1.

## v0.1.10-paperless.3

NEON on Linux arm64, plus memory-safety and leak fixes.
Expand Down
91 changes: 82 additions & 9 deletions sqlite-vec.c
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,16 @@ enum VectorElementType {
#define PORTABLE_ALIGN32 __attribute__((aligned(32)))
#define PORTABLE_ALIGN64 __attribute__((aligned(64)))

// Runtime CPU capability flags, filled in sqlite3_vec_init from CPUID, which
// is deterministic, so writing them on every connection open is harmless.
// Dispatch sites read them with no per-call check. They stay 0 until init has
// run, which selects the scalar path.
static struct {
int avx;
int avx2;
int fma;
} vec_avx_caps = {0, 0, 0};

__attribute__((target("avx,avx2")))
static f32 l2_sqr_float_avx(const void *pVect1v, const void *pVect2v,
const void *qty_ptr) {
Expand Down Expand Up @@ -251,6 +261,34 @@ static f32 cosine_float_avx(const void *pVect1v, const void *pVect2v,
return 1 - (dot / (sqrt(aMag) * sqrt(bMag)));
}
#undef VEC_AVX_MULADD

// Cosine distance with the query's squared magnitude (bMag) computed once by
// the caller, and fused multiply-add. Only dot and |a|^2 are accumulated, which
// is two thirds of the loads and arithmetic of cosine_float_avx. Callers must
// check vec_avx_caps.avx2 and vec_avx_caps.fma first.
__attribute__((target("avx,avx2,fma")))
static f32 cosine_float_avx_fma_qn(const f32 *a, const f32 *b, size_t qty,
f32 bMag) {
const f32 *aEnd = a + (qty & ~(size_t)15);
__m256 dot0 = _mm256_setzero_ps(), dot1 = _mm256_setzero_ps();
__m256 aa0 = _mm256_setzero_ps(), aa1 = _mm256_setzero_ps();
while (a < aEnd) {
__m256 a0 = _mm256_loadu_ps(a), a1 = _mm256_loadu_ps(a + 8);
dot0 = _mm256_fmadd_ps(a0, _mm256_loadu_ps(b), dot0);
dot1 = _mm256_fmadd_ps(a1, _mm256_loadu_ps(b + 8), dot1);
aa0 = _mm256_fmadd_ps(a0, a0, aa0);
aa1 = _mm256_fmadd_ps(a1, a1, aa1);
a += 16;
b += 16;
}
f32 dot = hsum_float_avx(_mm256_add_ps(dot0, dot1));
f32 aMag = hsum_float_avx(_mm256_add_ps(aa0, aa1));
for (size_t i = qty & ~(size_t)15; i < qty; i++, a++, b++) {
dot += *a * *b;
aMag += *a * *a;
}
return 1 - (dot / (sqrt(aMag) * sqrt(bMag)));
}
#endif

#ifdef SQLITE_VEC_ENABLE_NEON
Expand Down Expand Up @@ -549,9 +587,7 @@ static f32 distance_l2_sqr_float(const void *a, const void *b, const void *d) {
}
#endif
#ifdef SQLITE_VEC_ENABLE_AVX
static int has_avx2 = -1;
if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2");
if (has_avx2 && ((*(const size_t *)d) % 16 == 0)) {
if (vec_avx_caps.avx2 && ((*(const size_t *)d) % 16 == 0)) {
return l2_sqr_float_avx(a, b, d);
}
#endif
Expand Down Expand Up @@ -623,9 +659,7 @@ static f32 distance_cosine_float(const void *pVect1v, const void *pVect2v,
}
#endif
#ifdef SQLITE_VEC_ENABLE_AVX
static int has_avx2 = -1;
if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2");
if (has_avx2 && (*(const size_t *)qty_ptr) >= 16) {
if (vec_avx_caps.avx2 && (*(const size_t *)qty_ptr) >= 16) {
return cosine_float_avx(pVect1v, pVect2v, qty_ptr);
}
#endif
Expand Down Expand Up @@ -957,9 +991,7 @@ static f32 distance_hamming(const void *a, const void *b, const void *d) {
}
#endif
#ifdef SQLITE_VEC_ENABLE_AVX
static int has_avx2 = -1;
if (has_avx2 < 0) has_avx2 = __builtin_cpu_supports("avx2");
if (has_avx2 && n_bytes >= 32) {
if (vec_avx_caps.avx2 && n_bytes >= 32) {
return distance_hamming_avx2((const u8 *)a, (const u8 *)b, n_bytes);
}
#endif
Expand Down Expand Up @@ -7811,6 +7843,21 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks,
assert(numValueEntries == argc);
int hasMetadataFilters = 0;
int hasDistanceConstraints = 0;
#ifdef SQLITE_VEC_ENABLE_AVX
// The query's squared magnitude is the same for every row, so cosine
// computes it once per query rather than once per row.
int cosineAvxFma = 0;
f32 queryMagSq = 0;
if (vector_column->element_type == SQLITE_VEC_ELEMENT_TYPE_FLOAT32 &&
vector_column->distance_metric == VEC0_DISTANCE_METRIC_COSINE &&
vector_column->dimensions >= 16 && vec_avx_caps.avx2 &&
vec_avx_caps.fma) {
cosineAvxFma = 1;
for (size_t j = 0; j < vector_column->dimensions; j++) {
queryMagSq += ((f32 *)queryVector)[j] * ((f32 *)queryVector)[j];
}
}
#endif
for(int i = 0; i < argc; i++) {
int idx = 1 + (i * 4);
char kind = idxStr[idx + 0];
Expand Down Expand Up @@ -7969,6 +8016,14 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks,
break;
}
case VEC0_DISTANCE_METRIC_COSINE: {
#ifdef SQLITE_VEC_ENABLE_AVX
if (cosineAvxFma) {
result = cosine_float_avx_fma_qn(base_i, (f32 *)queryVector,
vector_column->dimensions,
queryMagSq);
break;
}
#endif
result = distance_cosine_float(base_i, (f32 *)queryVector,
&vector_column->dimensions);
break;
Expand Down Expand Up @@ -8062,6 +8117,18 @@ int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks,
}
}

// Once the global top k is full, a row that is not strictly nearer than
// the k-th (merge_sorted_lists keeps the earlier row on a tie, and ranks a
// NaN last) can never enter it, so it need not be a candidate.
if (k_used == k && !isnan(topk_distances[k - 1])) {
f32 worst = topk_distances[k - 1];
for (int j = 0; j < p->chunk_size; j++) {
if (bitmap_get(b, j) && !(chunk_distances[j] < worst)) {
bitmap_set(b, j, 0);
}
}
}

int used1;
min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs,
min(k, p->chunk_size), &used1);
Expand Down Expand Up @@ -11219,6 +11286,12 @@ SQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg,
#endif
int rc = SQLITE_OK;

#ifdef SQLITE_VEC_ENABLE_AVX
vec_avx_caps.avx = __builtin_cpu_supports("avx");
vec_avx_caps.avx2 = __builtin_cpu_supports("avx2");
vec_avx_caps.fma = __builtin_cpu_supports("fma");
#endif

#define DEFAULT_FLAGS (SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC)

rc = sqlite3_create_function_v2(db, "vec_version", 0, DEFAULT_FLAGS,
Expand Down
Loading