diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cc85e587..88025fa7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -82,6 +82,25 @@ jobs: path: dist/* # after the upload, so dist/test-unit stays out of the artifact - run: make test-unit + # The IVF index is compiled only on request, so the other builds skip its + # tests; this one compiles it in and runs every suite. + build-linux-x86_64-ivf: + needs: memory-gate + runs-on: ubuntu-latest + env: + CPPFLAGS: -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1 + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6 + with: + enable-cache: true + - run: ./scripts/vendor.sh + - run: make loadable static + - run: uv sync --locked --managed-python --directory tests + - run: make test-loadable python=./tests/.venv/bin/python + - run: make test-unit build-macos-x86_64-extension: runs-on: macos-15-intel steps: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 997cbf50..d05ad56c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -84,6 +84,49 @@ A table whose every vector column is indexed by `diskann()` keeps no chunks: its `xyz_chunks` stays empty, and its `xyz_rowids` rows have no `chunk_id` or `chunk_offset`. +#### `xyz_ivf_centroidsNN` + +The centroids of a column indexed by `ivf()`, held as its cells hold vectors: +`float32`, or quantized by the column's `quantizer`. The `ivf_trained_N` row of +`xyz_info` is 1 while the column has centroids. + +- `centroid_id INTEGER` +- `centroid BLOB` + +#### `xyz_ivf_cellsNN` + +The rows of a column indexed by `ivf()`, in cells of up to 64. A cell holds +rows of one centroid, or, with `centroid_id` -1, rows no clustering has placed. +`validity` holds a bit per slot, set when the slot holds a row, `rowids` the +row's rowid as a 64-bit integer in the machine's byte order, and `vectors` its +vector as the centroids hold them. `n_vectors` counts the set bits. A query +finds the cells of a centroid through the index of the +`UNIQUE (centroid_id, cell_id)` constraint. + +- `cell_id INTEGER` +- `centroid_id INTEGER` +- `n_vectors INTEGER` +- `validity BLOB` +- `rowids BLOB` +- `vectors BLOB` + +#### `xyz_ivf_rowid_mapNN` + +The cell and slot of each row of a column indexed by `ivf()`. + +- `rowid INTEGER` +- `cell_id INTEGER` +- `slot INTEGER` + +#### `xyz_ivf_vectorsNN` + +The `float32` vectors of a column indexed by `ivf()` with a `quantizer`, one +row per row of the table. Without a quantizer, the cells hold the vectors and +the table does not exist. + +- `rowid INTEGER` +- `vector BLOB` + #### `xyz_auxiliary` - `rowid INTEGER` @@ -106,11 +149,14 @@ The supported way to tell which tables belong to a `vec0` table is by name. A - `xyz_info`, `xyz_chunks`, `xyz_rowids`: always - `xyz_vector_chunksNN`: one per vector column, except those indexed by - `rescore()` or `diskann()` + `rescore()`, `diskann()` or `ivf()` - `xyz_rescore_chunksNN`, `xyz_rescore_vectorsNN`: one each per vector column indexed by `rescore()` - `xyz_vectorsNN`, `xyz_diskann_nodesNN`, `xyz_diskann_bufferNN`: one each per vector column indexed by `diskann()` +- `xyz_ivf_centroidsNN`, `xyz_ivf_cellsNN`, `xyz_ivf_rowid_mapNN`: one each per + vector column indexed by `ivf()`, and `xyz_ivf_vectorsNN` for each of those + with a `quantizer` - `xyz_auxiliary`: only if there are auxiliary columns - `xyz_metadatachunksNN`: one per metadata column - `xyz_metadatatextNN`: one per text metadata column @@ -132,6 +178,10 @@ join sqlite_master as s or substr(s.name, length(v.name) + 2) glob 'vectors[0-9][0-9]' or substr(s.name, length(v.name) + 2) glob 'diskann_nodes[0-9][0-9]' or substr(s.name, length(v.name) + 2) glob 'diskann_buffer[0-9][0-9]' + or substr(s.name, length(v.name) + 2) glob 'ivf_centroids[0-9][0-9]' + or substr(s.name, length(v.name) + 2) glob 'ivf_cells[0-9][0-9]' + or substr(s.name, length(v.name) + 2) glob 'ivf_rowid_map[0-9][0-9]' + or substr(s.name, length(v.name) + 2) glob 'ivf_vectors[0-9][0-9]' or substr(s.name, length(v.name) + 2) glob 'metadatachunks[0-9][0-9]' or substr(s.name, length(v.name) + 2) glob 'metadatatext[0-9][0-9]') where v.type = 'table' and v.rootpage = 0 @@ -149,7 +199,7 @@ relied on: - Before SQLite 3.37 there is no `PRAGMA table_list` and no prefix marking (see below), so the tables whose suffix has an underscore (`vector_chunksNN`, `rescore_chunksNN`, `rescore_vectorsNN`, `diskann_nodesNN`, - `diskann_bufferNN`) are never shadow tables. + `diskann_bufferNN`, and the four `ivf_` tables) are never shadow tables. - SQLite finds the virtual table that owns a newly created or newly parsed table by splitting its name at the **last** underscore ([documented](https://www.sqlite.org/vtab.html#the_xshadowname_method)). For diff --git a/CHANGELOG.md b/CHANGELOG.md index f6178b2b..e74a6fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - A vector column definition accepts `indexed by flat()`, as in `embedding float[768] indexed by flat()`. `flat()` is the exhaustive search that a column without the clause gets. Ported from [asg017@0de765f](https://github.com/asg017/sqlite-vec/commit/0de765f) ([asg017#273](https://github.com/asg017/sqlite-vec/pull/273)). - A `float` vector column can be declared `indexed by rescore(quantizer=bit)` or `indexed by rescore(quantizer=int8)`, with an optional `oversample=N` from 1 to 128 (8 by default). It also keeps its vectors quantized, in the new `_rescore_chunksNN` shadow table, and its `float` vectors by rowid in `_rescore_vectorsNN`. A KNN query scans the quantized vectors for the `k * oversample` nearest rows (`5 * k * oversample` with `mmr_lambda` below 1, at most 4096), then ranks those by the exact distances of their `float` vectors, to which distance constraints apply. The `oversample=N` command, as in `insert into t(t) values ('oversample=16')`, sets the oversample of the table's rescore columns for the connection's queries until SQLite reconnects the table, as after `VACUUM`, `ALTER TABLE ... RENAME`, or another connection's schema change. Rowid filters, MMR, auxiliary columns, `UPDATE`, `optimize`, and `ALTER TABLE ... RENAME` work on rescore columns; a table with one cannot have metadata or partition key columns. Ported from [asg017#276](https://github.com/asg017/sqlite-vec/pull/276) ([asg017@ba0db0b](https://github.com/asg017/sqlite-vec/commit/ba0db0b)) with its fixes [asg017@5e4c557](https://github.com/asg017/sqlite-vec/commit/5e4c557), [@82f4eb0](https://github.com/asg017/sqlite-vec/commit/82f4eb0), [@5522e86](https://github.com/asg017/sqlite-vec/commit/5522e86), [@b7fc459](https://github.com/asg017/sqlite-vec/commit/b7fc459), [@6e2c4c6](https://github.com/asg017/sqlite-vec/commit/6e2c4c6), the int8 quantizer of [@575371d](https://github.com/asg017/sqlite-vec/commit/575371d), and the distance constraints of the unmerged [asg017#309](https://github.com/asg017/sqlite-vec/pull/309). Unlike upstream, a point query for a rowid that a rescore table does not have returns no row instead of failing with `Could not fetch vector data`. - A `float` or `int8` vector column can be declared `indexed by diskann(neighbor_quantizer=binary)` or `indexed by diskann(neighbor_quantizer=int8)`, with optional `n_neighbors=N` (72 by default), `search_list_size=N` (128) or `search_list_size_search=N` and `search_list_size_insert=N`, and `buffer_threshold=N`. The column keeps a graph of its rows in the new `_diskann_nodesNN` shadow table, its vectors by rowid in `_vectorsNN`, and its buffered inserts in `_diskann_bufferNN`, and a KNN query walks the graph instead of reading every vector. The commands `search_list_size=N`, `search_list_size_search=N` and `search_list_size_insert=N` set those sizes for the connection's queries and inserts until SQLite reconnects the table, as after `VACUUM`, `ALTER TABLE ... RENAME`, or another connection's schema change. Rowid filters, distance constraints, MMR, auxiliary columns, `optimize`, and `ALTER TABLE ... RENAME` work on DiskANN columns; a table with one cannot have metadata or partition key columns, and a DiskANN column's vector cannot be updated. A table whose every vector column is indexed by DiskANN keeps no chunks. Ported from [asg017@575371d](https://github.com/asg017/sqlite-vec/commit/575371d) with [asg017@9df59b4](https://github.com/asg017/sqlite-vec/commit/9df59b4), [@82f4eb0](https://github.com/asg017/sqlite-vec/commit/82f4eb0), [@b008654](https://github.com/asg017/sqlite-vec/commit/b008654), [@c36a995](https://github.com/asg017/sqlite-vec/commit/c36a995), [@01b4b2a](https://github.com/asg017/sqlite-vec/commit/01b4b2a), [@b7fc459](https://github.com/asg017/sqlite-vec/commit/b7fc459) and [@fe94171](https://github.com/asg017/sqlite-vec/commit/fe94171). Unlike upstream, an `int8` column's vectors are quantized as `int8` values (upstream reads an `int8` query as floats, past its end), a result's distance is always its exact one, a point query for a rowid that the table does not have returns no row instead of failing with `Could not fetch vector data`, the row every search starts from is replaced, when deleted, by a row of the graph rather than one still waiting in the buffer, a search of a table whose start row upstream left in the buffer, or deleted, starts from a row of the graph, where upstream returns the start row twice or fails, and the table's next delete or buffer flush replaces that start row, a search allocates memory for the rows it reaches rather than for its whole search list (upstream allocates room for `search_list_size` candidates, and a visited set four times that size, before each search), rowid filters, distance constraints and MMR apply, errors and interrupts reach the query instead of being skipped, `VACUUM` works after a KNN query, the commands set every DiskANN column of the table, a KNN query takes `k` as every vec0 column does, an integer from 0 to 4096 (upstream returns no rows for a negative `k` and has no limit), CREATE refuses an abbreviated option such as `n_neigh=16` or `neighbor_quantizer=bin` and a comma before `)`, which connect reads as upstream does, and a command value such as `search_list_size=12abc` and a number past an `int` are refused. +- A build compiled with `-DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1` can declare a `float` vector column `indexed by ivf()`, with optional `nlist=N` (128 by default, or 0 to leave the centroids to `set-centroid`), `nprobe=N` (10, at most `nlist`), `quantizer=int8` or `quantizer=binary`, and `oversample=N` (1). The column keeps its rows in cells of up to 64 in the new `_ivf_cellsNN` shadow table, around the centroids in `_ivf_centroidsNN`, with each row's cell and slot in `_ivf_rowid_mapNN` and, with a quantizer, its `float` vector in `_ivf_vectorsNN`; a KNN query reads the cells of the `nprobe` centroids nearest its vector and the cells of rows no clustering has placed. The commands `compute-centroids`, `compute-centroids:{"nlist": N, "max_iterations": N, "seed": N}`, `set-centroid:N`, `assign-vectors` and `clear-centroids` build it, and the `nprobe=N` command sets the nprobe of a connection's queries until SQLite reconnects the table, as after `VACUUM`, `ALTER TABLE ... RENAME`, or another connection's schema change. Rowid filters, distance constraints, MMR, auxiliary columns, `optimize`, and `ALTER TABLE ... RENAME` work on IVF columns; a table with one cannot have metadata or partition key columns, and an IVF column's vector cannot be updated. The prebuilt binaries are built without it. Ported from [asg017@e2c38f3](https://github.com/asg017/sqlite-vec/commit/e2c38f3) with [asg017@4bee883](https://github.com/asg017/sqlite-vec/commit/4bee883), [@9df59b4](https://github.com/asg017/sqlite-vec/commit/9df59b4), [@b7fc459](https://github.com/asg017/sqlite-vec/commit/b7fc459), [@6e2c4c6](https://github.com/asg017/sqlite-vec/commit/6e2c4c6), [@8b81f40](https://github.com/asg017/sqlite-vec/commit/8b81f40) and [@fe94171](https://github.com/asg017/sqlite-vec/commit/fe94171), and the fixes of the unmerged [asg017#313](https://github.com/asg017/sqlite-vec/pull/313). Unlike upstream, a quantized column's point query and full scan return its `float` vectors (upstream returns the quantized bytes, or reads past them), a connection finds the centroids another connection computed (upstream caches the column's state per connection and finds no rows), rowid filters and distance constraints apply, a renamed table's cells are found through an index named after the new table, so a table created under the old name works, an `int8` or `bit` column is refused (upstream reads past an `int8` column's vectors), errors and interrupts reach the statement, so an interrupted command fails with `interrupted` and the connection's queries go on finding its rows (upstream reports `SQL logic error`, and its KNN queries then return no rows), `VACUUM` works after a KNN query, the commands apply to every IVF column of the table, and an abbreviated option, a command value such as `nprobe=5x`, and `compute-centroids` options that are not a JSON object of integers are refused. ### Changed @@ -21,7 +22,7 @@ - A command that a `vec0` table does not know, as in `insert into t(t) values ('compact')`, now fails with `unknown vec0 command: 'compact'` instead of `SQL logic error`, as upstream [asg017@6e2c4c6](https://github.com/asg017/sqlite-vec/commit/6e2c4c6) does. - `insert into t(t) values ('oversample=N')` now succeeds on every `vec0` table, as on upstream, and on a table without a rescore column changes nothing; it used to fail with `SQL logic error`. - When the same connection deletes a row that a running full scan or KNN query has found but not yet read, reading that row's vector in a KNN query, or its `text` primary key or partition key in either query, now fails with `SQLITE_ERROR` instead of `SQLITE_EMPTY`, a code SQLite reserves for its own use. The primary key read fails with `Could not find a row with rowid N` and the partition key read with `SQL logic error`; both used to fail with `unknown error`. -- `vec0`'s `xShadowName` also reports the `_rescore_chunksNN` and `_rescore_vectorsNN` tables of rescore columns and the `_vectorsNN`, `_diskann_nodesNN` and `_diskann_bufferNN` tables of DiskANN columns, which upstream's does not. Under `SQLITE_DBCONFIG_DEFENSIVE`, a table with one of those names beside any `vec0` table is read-only once SQLite marks it as a shadow table, as after a `VACUUM`, and `CREATE TABLE t_vectors00` beside a table `t` fails with `object name reserved for internal use`. +- `vec0`'s `xShadowName` also reports the `_rescore_chunksNN` and `_rescore_vectorsNN` tables of rescore columns, the `_vectorsNN`, `_diskann_nodesNN` and `_diskann_bufferNN` tables of DiskANN columns, and in a build with IVF the `_ivf_centroidsNN`, `_ivf_cellsNN`, `_ivf_rowid_mapNN` and `_ivf_vectorsNN` tables of IVF columns, which upstream's does not. Under `SQLITE_DBCONFIG_DEFENSIVE`, a table with one of those names beside any `vec0` table is read-only once SQLite marks it as a shadow table, as after a `VACUUM`, and `CREATE TABLE t_vectors00` beside a table `t` fails with `object name reserved for internal use`. - GitHub releases now wait for npm to serve the approved package, and carry these release notes. ### Fixed diff --git a/Makefile b/Makefile index 265c3cec..87b77bbb 100644 --- a/Makefile +++ b/Makefile @@ -211,7 +211,7 @@ test-loadable-watch: test-unit: | $(prefix) $(CC) tests/test-unit.c sqlite-vec.c vendor/sqlite3.c \ - -I./ -Ivendor -DSQLITE_CORE -o $(prefix)/test-unit -lm + -I./ -Ivendor -DSQLITE_CORE $(EXT_CFLAGS) -o $(prefix)/test-unit -lm $(prefix)/test-unit test-all: test test-loadable test-unit diff --git a/README.md b/README.md index 5e9321b1..ecc56621 100644 --- a/README.md +++ b/README.md @@ -295,8 +295,9 @@ VACUUM; `VACUUM` should not corrupt vec tables; a checkpoint first is recommended when using WAL so the rewrite starts from a clean state. -`optimize`, like the `oversample=N` command of a rescore index and the -`search_list_size=N` commands of a DiskANN index, runs through a hidden column +`optimize`, like the `oversample=N` command of a rescore index, the +`search_list_size=N` commands of a DiskANN index and the commands of an IVF +index, runs through a hidden column named after the table, so a `vec0` table cannot have another column with its name. `CREATE` rejects tables such as `CREATE VIRTUAL TABLE emb USING vec0(emb float[4])`, and tables named @@ -451,6 +452,93 @@ where sample_embedding match '[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]' row, one of the row's other neighbors it lacks, and reads every node of the graph to remove the links to the row. +### IVF Index (experimental) + +A build compiled with `-DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1`, as by +`make loadable CPPFLAGS=-DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1`, has one more +index; the prebuilt binaries leave it out. A `float` vector column declared +`indexed by ivf()` clusters its rows around `nlist` centroids with k-means, and +a KNN query reads only the rows of the `nprobe` centroids nearest its vector. +Rows wait in cells of no centroid until `compute-centroids` clusters them, and +a query reads those cells in full, so until then it answers as a flat column +does: + +```sql +create virtual table vec_ivf using vec0( + sample_embedding float[8] indexed by ivf(nlist=4, nprobe=1) +); +insert into vec_ivf(rowid, sample_embedding) + select rowid, sample_embedding from vec_examples; +insert into vec_ivf(vec_ivf) values ('compute-centroids'); + +select rowid, distance +from vec_ivf +where sample_embedding match '[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]' + and k = 3; +/* +┌───────┬──────────────────┐ +│ rowid │ distance │ +├───────┼──────────────────┤ +│ 5 │ 1.16368770599365 │ +│ 7 │ 1.89339029788971 │ +│ 4 │ 2.10507702827454 │ +└───────┴──────────────────┘ +*/ +``` + +The query read one of the four clusters, and rows 13 and 11, the second and +third nearest, are in others. The `nprobe=N` command sets `nprobe` for every +IVF column, for the connection that runs it, until SQLite reconnects the table, +as after `VACUUM`, `ALTER TABLE ... RENAME`, or another connection's schema +change: + +```sql +insert into vec_ivf(vec_ivf) values ('nprobe=2'); + +select rowid, distance +from vec_ivf +where sample_embedding match '[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]' + and k = 3; +/* +┌───────┬──────────────────┐ +│ rowid │ distance │ +├───────┼──────────────────┤ +│ 5 │ 1.16368770599365 │ +│ 13 │ 1.75137972831726 │ +│ 11 │ 1.83941268920898 │ +└───────┴──────────────────┘ +*/ +``` + +- `nlist` is 128 unless the column sets it, up to 65536, and `nlist=0` leaves + the centroids to `set-centroid`. `nprobe` is 10, or `nlist` if that is less. +- `compute-centroids` clusters every row of each IVF column, with at most one + centroid per row, and moves each row to the cells of its nearest centroid. + `compute-centroids:{"nlist": 64, "max_iterations": 25, "seed": 1}` sets any + of the three as JSON integers; the same rows, read in the same order, and the + same seed give the same centroids. A row inserted later goes to the cells of + its nearest centroid, and the centroids stay as they are until the next + `compute-centroids`. +- `set-centroid:N` sets centroid `N` of each IVF column to the vector the same + `INSERT` gives it, as in + `insert into vec_ivf(vec_ivf, sample_embedding) values ('set-centroid:0', '[...]')`. + `assign-vectors` then moves the rows in cells of no centroid to their nearest + centroids, and `clear-centroids` removes the centroids and moves every row to + cells of no centroid. Every command applies to each IVF column of the table. +- With `quantizer=int8` or `quantizer=binary`, the cells and centroids hold the + vectors quantized and the `float` vectors are kept by rowid besides. A query + keeps the `k * oversample` rows nearest by quantized distance and ranks them + by their exact distances. `int8` maps [-1, 1] onto [-127, 127]; `binary` keeps + a bit for each value > 0 and needs a number of dimensions divisible by 8. + `oversample` is 1 unless the column sets it, and needs a quantizer. +- With `rowid = ?` or `rowid in (...)`, a KNN query ranks the rows they name by + their exact distances, without the clusters. Distance constraints apply to + the exact distances of the rows the query read, and MMR reranks them as it + does any KNN query. +- A table with an IVF column cannot have metadata or partition key columns, and + only `float` columns can be indexed by IVF. Auxiliary columns work. +- `UPDATE` of an IVF column's vector fails; delete the row and insert it again. + ## Sponsors > [!NOTE] diff --git a/sqlite-vec.c b/sqlite-vec.c index 3012ac24..ee0d5dcb 100644 --- a/sqlite-vec.c +++ b/sqlite-vec.c @@ -87,6 +87,10 @@ typedef size_t usize; #define SQLITE_VEC_ENABLE_DISKANN 1 #endif +#ifndef SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +#define SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE 0 +#endif + // VEC0_UNPREDICTABLE(cond) tells clang that cond is hard to predict, so it // emits a conditional move instead of a branch. Other compilers get cond as is. #if defined(__has_builtin) @@ -2815,6 +2819,9 @@ enum Vec0IndexType { #if SQLITE_VEC_ENABLE_RESCORE VEC0_INDEX_TYPE_RESCORE = 2, #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + VEC0_INDEX_TYPE_IVF = 3, +#endif #if SQLITE_VEC_ENABLE_DISKANN VEC0_INDEX_TYPE_DISKANN = 4, #endif @@ -2879,6 +2886,44 @@ struct Vec0DiskannConfig { }; #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +// `indexed by ivf(nlist=N, nprobe=N, quantizer=none|int8|binary, +// oversample=N)` keeps the column's vectors in cells, each holding the rows +// nearest one centroid of a k-means clustering, and a KNN query reads only the +// cells of the nprobe centroids nearest the query, with the cells of the rows +// no clustering has placed yet. +// https://github.com/asg017/sqlite-vec/commit/e2c38f3 +#define VEC0_IVF_DEFAULT_NLIST 128 +#define VEC0_IVF_DEFAULT_NPROBE 10 +#define VEC0_IVF_MAX_NLIST 65536 +// the rows one cell holds +#define VEC0_IVF_CELL_MAX_VECTORS 64 +// the centroid id of the cells holding the rows no clustering has placed +#define VEC0_IVF_UNASSIGNED_CENTROID_ID (-1) +#define VEC0_IVF_KMEANS_MAX_ITER 25 +#define VEC0_IVF_KMEANS_DEFAULT_SEED 0 + +enum Vec0IvfQuantizer { + // a cell holds each float32 vector as is + VEC0_IVF_QUANTIZER_NONE = 0, + // a byte per dimension + VEC0_IVF_QUANTIZER_INT8 = 1, + // a bit per dimension + VEC0_IVF_QUANTIZER_BINARY = 2, +}; + +struct Vec0IvfConfig { + // the centroids compute-centroids makes; 0 leaves them to set-centroid + int nlist; + // the centroids whose cells a KNN query reads + int nprobe; + enum Vec0IvfQuantizer quantizer; + // with a quantizer, a KNN query ranks the k * oversample rows nearest by + // quantized distance by their exact distances + int oversample; +}; +#endif + struct VectorColumnDefinition { char *name; int name_length; @@ -2892,6 +2937,9 @@ struct VectorColumnDefinition { #if SQLITE_VEC_ENABLE_DISKANN struct Vec0DiskannConfig diskann; #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + struct Vec0IvfConfig ivf; +#endif }; struct Vec0PartitionColumnDefinition { @@ -3009,15 +3057,15 @@ static int vec0_parse_rescore_options(struct Vec0Scanner *scanner, } #endif -#if SQLITE_VEC_ENABLE_DISKANN +#if SQLITE_VEC_ENABLE_DISKANN || SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE /** - * @brief Reads the value of a numeric diskann() option: all digits, from min - * to max. + * @brief Reads the value of a numeric option of diskann() or ivf(): all + * digits, from min to max. * * @return SQLITE_OK, or SQLITE_ERROR for any other token. */ -static int vec0_parse_diskann_int(const struct Vec0Token *token, long min, - long max, int *out) { +static int vec0_parse_option_int(const struct Vec0Token *token, long min, + long max, int *out) { if (token->token_type != TOKEN_TYPE_DIGIT) { return SQLITE_ERROR; } @@ -3030,7 +3078,9 @@ static int vec0_parse_diskann_int(const struct Vec0Token *token, long min, *out = (int)value; return SQLITE_OK; } +#endif +#if SQLITE_VEC_ENABLE_DISKANN /** * @brief Parses the options of `indexed by diskann(...)`, from after its '(' * through its ')', separated by commas: `neighbor_quantizer=binary` or @@ -3090,34 +3140,34 @@ static int vec0_parse_diskann_options(struct Vec0Scanner *scanner, } hasQuantizer = 1; } else if (vec0_token_is_keyword(&key, "n_neighbors", strict)) { - if (vec0_parse_diskann_int(&token, 8, VEC0_DISKANN_MAX_N_NEIGHBORS, - &outConfig->n_neighbors) != SQLITE_OK || + if (vec0_parse_option_int(&token, 8, VEC0_DISKANN_MAX_N_NEIGHBORS, + &outConfig->n_neighbors) != SQLITE_OK || outConfig->n_neighbors % 8 != 0) { return SQLITE_ERROR; } } else if (vec0_token_is_keyword(&key, "search_list_size", strict)) { - if (vec0_parse_diskann_int(&token, 1, INT_MAX, - &outConfig->search_list_size) != SQLITE_OK) { + if (vec0_parse_option_int(&token, 1, INT_MAX, + &outConfig->search_list_size) != SQLITE_OK) { return SQLITE_ERROR; } hasSearchListSize = 1; } else if (vec0_token_is_keyword(&key, "search_list_size_search", true)) { - if (vec0_parse_diskann_int(&token, 1, INT_MAX, - &outConfig->search_list_size_search) != + if (vec0_parse_option_int(&token, 1, INT_MAX, + &outConfig->search_list_size_search) != SQLITE_OK) { return SQLITE_ERROR; } hasSplitSearchListSize = 1; } else if (vec0_token_is_keyword(&key, "search_list_size_insert", true)) { - if (vec0_parse_diskann_int(&token, 1, INT_MAX, - &outConfig->search_list_size_insert) != + if (vec0_parse_option_int(&token, 1, INT_MAX, + &outConfig->search_list_size_insert) != SQLITE_OK) { return SQLITE_ERROR; } hasSplitSearchListSize = 1; } else if (vec0_token_is_keyword(&key, "buffer_threshold", strict)) { - if (vec0_parse_diskann_int(&token, 0, INT_MAX, - &outConfig->buffer_threshold) != SQLITE_OK) { + if (vec0_parse_option_int(&token, 0, INT_MAX, + &outConfig->buffer_threshold) != SQLITE_OK) { return SQLITE_ERROR; } } else { @@ -3144,6 +3194,102 @@ static int vec0_parse_diskann_options(struct Vec0Scanner *scanner, } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +/** + * @brief Parses the options of `indexed by ivf(...)`, from after its '(' + * through its ')', separated by commas, each optional: `nlist=N` from 0 to + * 65536, 128 when omitted; `nprobe=N` from 1 to nlist, 10 when omitted or + * nlist when that is less; `quantizer=none`, `quantizer=int8` or + * `quantizer=binary`, none when omitted; and `oversample=N`, 1 when omitted, + * more only with a quantizer. + * + * @param strict true to require whole keywords; false to accept a prefix of a + * keyword, as upstream does + * @return SQLITE_OK, or SQLITE_ERROR for any other text. + */ +static int vec0_parse_ivf_options(struct Vec0Scanner *scanner, + struct Vec0IvfConfig *outConfig, + bool strict) { + struct Vec0Token token; + int hasNprobe = 0; + outConfig->nlist = VEC0_IVF_DEFAULT_NLIST; + outConfig->nprobe = VEC0_IVF_DEFAULT_NPROBE; + outConfig->quantizer = VEC0_IVF_QUANTIZER_NONE; + outConfig->oversample = 1; + + int rc = vec0_scanner_next(scanner, &token); + int empty = + rc == VEC0_TOKEN_RESULT_SOME && token.token_type == TOKEN_TYPE_RPAREN; + while (!empty) { + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_ERROR; + } + struct Vec0Token key = token; + rc = vec0_scanner_next(scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME || token.token_type != TOKEN_TYPE_EQ) { + return SQLITE_ERROR; + } + rc = vec0_scanner_next(scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME) { + return SQLITE_ERROR; + } + if (vec0_token_is_keyword(&key, "nlist", strict)) { + if (vec0_parse_option_int(&token, 0, VEC0_IVF_MAX_NLIST, + &outConfig->nlist) != SQLITE_OK) { + return SQLITE_ERROR; + } + } else if (vec0_token_is_keyword(&key, "nprobe", strict)) { + if (vec0_parse_option_int(&token, 1, VEC0_IVF_MAX_NLIST, + &outConfig->nprobe) != SQLITE_OK) { + return SQLITE_ERROR; + } + hasNprobe = 1; + } else if (vec0_token_is_keyword(&key, "quantizer", strict)) { + if (token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_ERROR; + } + if (vec0_token_is_keyword(&token, "none", strict)) { + outConfig->quantizer = VEC0_IVF_QUANTIZER_NONE; + } else if (vec0_token_is_keyword(&token, "int8", strict)) { + outConfig->quantizer = VEC0_IVF_QUANTIZER_INT8; + } else if (vec0_token_is_keyword(&token, "binary", strict)) { + outConfig->quantizer = VEC0_IVF_QUANTIZER_BINARY; + } else { + return SQLITE_ERROR; + } + } else if (vec0_token_is_keyword(&key, "oversample", strict)) { + if (vec0_parse_option_int(&token, 1, INT_MAX, &outConfig->oversample) != + SQLITE_OK) { + return SQLITE_ERROR; + } + } else { + return SQLITE_ERROR; + } + + rc = vec0_scanner_next(scanner, &token); + if (rc == VEC0_TOKEN_RESULT_SOME && token.token_type == TOKEN_TYPE_RPAREN) { + break; + } + if (rc != VEC0_TOKEN_RESULT_SOME || token.token_type != TOKEN_TYPE_COMMA) { + return SQLITE_ERROR; + } + rc = vec0_scanner_next(scanner, &token); + } + if (outConfig->nlist > 0 && outConfig->nprobe > outConfig->nlist) { + if (hasNprobe) { + return SQLITE_ERROR; + } + outConfig->nprobe = outConfig->nlist; + } + if (outConfig->oversample > 1 && + outConfig->quantizer == VEC0_IVF_QUANTIZER_NONE) { + return SQLITE_ERROR; + } + return SQLITE_OK; +} +#endif + /** * @brief Whether vec0 2.0.2 and earlier scanned the token. They scanned no '(', * ')' or ',', so a definition they created has none where a non-strict parse @@ -3192,6 +3338,10 @@ int vec0_parse_vector_column(const char *source, int source_length, #if SQLITE_VEC_ENABLE_DISKANN struct Vec0DiskannConfig diskannConfig; memset(&diskannConfig, 0, sizeof(diskannConfig)); +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + struct Vec0IvfConfig ivfConfig; + memset(&ivfConfig, 0, sizeof(ivfConfig)); #endif int dimensions; @@ -3308,8 +3458,8 @@ int vec0_parse_vector_column(const char *source, int source_length, return SQLITE_ERROR; } } - // `indexed by flat()`, `indexed by rescore(...)` or - // `indexed by diskann(...)`. Connect matches the keywords by prefix, as + // `indexed by flat()`, `indexed by rescore(...)`, `indexed by diskann(...)` + // or `indexed by ivf(...)`. Connect matches the keywords by prefix, as // upstream does at CREATE too, so a table upstream created as // `index by f()` opens. else if (vec0_token_is_keyword(&token, "indexed", strict)) { @@ -3373,6 +3523,26 @@ int vec0_parse_vector_column(const char *source, int source_length, return SQLITE_ERROR; } } +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + else if (vec0_token_is_keyword(&token, "ivf", strict)) { + indexType = VEC0_INDEX_TYPE_IVF; + if (elementType != SQLITE_VEC_ELEMENT_TYPE_FLOAT32) { + return SQLITE_ERROR; + } + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_LPAREN) { + return SQLITE_ERROR; + } + if (vec0_parse_ivf_options(&scanner, &ivfConfig, strict) != SQLITE_OK) { + return SQLITE_ERROR; + } + if (ivfConfig.quantizer == VEC0_IVF_QUANTIZER_BINARY && + (dimensions % CHAR_BIT) != 0) { + return SQLITE_ERROR; + } + } #endif else { return SQLITE_ERROR; @@ -3398,6 +3568,9 @@ int vec0_parse_vector_column(const char *source, int source_length, #endif #if SQLITE_VEC_ENABLE_DISKANN outColumn->diskann = diskannConfig; +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + outColumn->ivf = ivfConfig; #endif return SQLITE_OK; } @@ -4581,6 +4754,18 @@ struct vec0_vtab { char *shadowDiskannBufferNames[VEC0_MAX_VECTOR_COLUMNS]; #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // Names of the shadow tables of each vector column indexed by ivf(), ie + // `_ivf_centroids00` with its centroids, `_ivf_cells00` with its vectors in + // cells, `_ivf_rowid_map00` with each row's cell and slot, and, with a + // quantizer, `_ivf_vectors00` with its float32 vectors by rowid. NULL for + // other columns; freed with sqlite3_free(). + char *shadowIvfCentroidsNames[VEC0_MAX_VECTOR_COLUMNS]; + char *shadowIvfCellsNames[VEC0_MAX_VECTOR_COLUMNS]; + char *shadowIvfRowidMapNames[VEC0_MAX_VECTOR_COLUMNS]; + char *shadowIvfVectorsNames[VEC0_MAX_VECTOR_COLUMNS]; +#endif + struct VectorColumnDefinition vector_columns[VEC0_MAX_VECTOR_COLUMNS]; struct Vec0PartitionColumnDefinition paritition_columns[VEC0_MAX_PARTITION_COLUMNS]; @@ -4658,6 +4843,31 @@ struct vec0_vtab { sqlite3_stmt *stmtDiskannVectorRead[VEC0_MAX_VECTOR_COLUMNS]; sqlite3_stmt *stmtDiskannVectorWrite[VEC0_MAX_VECTOR_COLUMNS]; #endif + +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // Statements the inserts, deletes and queries of each vector column indexed + // by ivf() run, each prepared on first use and reset after every use. + struct { + // the first cell of a centroid with a free slot + sqlite3_stmt *cellFind; + // the cells of a centroid + sqlite3_stmt *cellScan; + // adds to a cell's count of rows + sqlite3_stmt *cellCount; + // a row's cell and slot, in _ivf_rowid_mapNN + sqlite3_stmt *mapRead; + sqlite3_stmt *mapWrite; + sqlite3_stmt *mapDelete; + // a row's float32 vector, in _ivf_vectorsNN + sqlite3_stmt *vectorRead; + sqlite3_stmt *vectorWrite; + sqlite3_stmt *vectorDelete; + // every centroid + sqlite3_stmt *centroids; + // the column's ivf_trained_N row of _info + sqlite3_stmt *trained; + } ivfStmts[VEC0_MAX_VECTOR_COLUMNS]; +#endif }; /** @@ -4688,6 +4898,22 @@ void vec0_free_resources(vec0_vtab *p) { p->stmtDiskannVectorWrite[i] = NULL; } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) { + sqlite3_stmt **stmts[] = { + &p->ivfStmts[i].cellFind, &p->ivfStmts[i].cellScan, + &p->ivfStmts[i].cellCount, &p->ivfStmts[i].mapRead, + &p->ivfStmts[i].mapWrite, &p->ivfStmts[i].mapDelete, + &p->ivfStmts[i].vectorRead, &p->ivfStmts[i].vectorWrite, + &p->ivfStmts[i].vectorDelete, &p->ivfStmts[i].centroids, + &p->ivfStmts[i].trained, + }; + for (size_t j = 0; j < countof(stmts); j++) { + sqlite3_finalize(*stmts[j]); + *stmts[j] = NULL; + } + } +#endif } /** @@ -4725,6 +4951,16 @@ void vec0_free(vec0_vtab *p) { p->shadowDiskannNodesNames[i] = NULL; sqlite3_free(p->shadowDiskannBufferNames[i]); p->shadowDiskannBufferNames[i] = NULL; +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + sqlite3_free(p->shadowIvfCentroidsNames[i]); + p->shadowIvfCentroidsNames[i] = NULL; + sqlite3_free(p->shadowIvfCellsNames[i]); + p->shadowIvfCellsNames[i] = NULL; + sqlite3_free(p->shadowIvfRowidMapNames[i]); + p->shadowIvfRowidMapNames[i] = NULL; + sqlite3_free(p->shadowIvfVectorsNames[i]); + p->shadowIvfVectorsNames[i] = NULL; #endif sqlite3_free(p->vector_columns[i].name); p->vector_columns[i].name = NULL; @@ -4761,6 +4997,11 @@ static int diskann_vector_read(vec0_vtab *p, int vec_col_idx, i64 rowid, void **outVector, int *outVectorSize); #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +static int ivf_vector_read(vec0_vtab *p, int i, i64 rowid, void **outVector, + int *outVectorSize); +#endif + /** * @brief Returns the index of the distance hidden column for the given vec0 * table. @@ -5117,6 +5358,21 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // an ivf() column keeps each float32 vector in a cell, or, with a + // quantizer, in its row of _ivf_vectorsNN + if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF) { + rc = ivf_vector_read(p, vector_column_idx, rowid, outVector, outVectorSize); + if (rc == SQLITE_EMPTY) { + vtab_set_error(&pVtab->base, + "Could not fetch vector data for %lld from the IVF index", + rowid); + rc = SQLITE_ERROR; + } + goto cleanup; + } +#endif + size = vector_column_byte_size(pVtab->vector_columns[vector_column_idx]); #if SQLITE_VEC_ENABLE_RESCORE // a rescore() column keeps each float32 vector in a row of its own @@ -6324,6 +6580,28 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // a KNN query reads the cells of the centroids nearest its vector, so no + // filter of a metadata column or partition key could apply before its k + for (int i = 0; i < numVectorColumns; i++) { + if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + if (numMetadataColumns > 0) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "metadata columns are not supported with IVF indexes"); + goto error; + } + if (numPartitionColumns > 0) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "partition key columns are not supported with IVF indexes"); + goto error; + } + } +#endif + const char *schemaName = argv[1]; const char *tableName = argv[2]; @@ -6485,6 +6763,22 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, goto error; } } +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + if (pNew->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF) { + pNew->shadowIvfCentroidsNames[i] = + sqlite3_mprintf("%s_ivf_centroids%02d", tableName, i); + pNew->shadowIvfCellsNames[i] = + sqlite3_mprintf("%s_ivf_cells%02d", tableName, i); + pNew->shadowIvfRowidMapNames[i] = + sqlite3_mprintf("%s_ivf_rowid_map%02d", tableName, i); + pNew->shadowIvfVectorsNames[i] = + sqlite3_mprintf("%s_ivf_vectors%02d", tableName, i); + if (!pNew->shadowIvfCentroidsNames[i] || !pNew->shadowIvfCellsNames[i] || + !pNew->shadowIvfRowidMapNames[i] || !pNew->shadowIvfVectorsNames[i]) { + goto error; + } + } #endif } for (int i = 0; i < pNew->numMetadataColumns; i++) { @@ -6726,6 +7020,86 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + for (int i = 0; i < pNew->numVectorColumns; i++) { + if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + // asg017's declarations, except that the cells are found by centroid + // through the index of a UNIQUE constraint, which ALTER TABLE RENAME + // renames along with its table: an index from CREATE INDEX keeps its + // name through a rename, and nothing inside xRename can drop it, so a + // table created later under the old name could not create its own. + // Every statement on the cells names its columns and addresses a cell by + // rowid, so either implementation opens the other's tables. + const char *zCreate[] = { + "CREATE TABLE \"%w\".\"%w_ivf_centroids%02d\"" + " (centroid_id INTEGER PRIMARY KEY, centroid BLOB NOT NULL)", + "CREATE TABLE \"%w\".\"%w_ivf_cells%02d\"" + " (cell_id INTEGER PRIMARY KEY," + " centroid_id INTEGER NOT NULL," + " n_vectors INTEGER NOT NULL DEFAULT 0," + " validity BLOB NOT NULL," + " rowids BLOB NOT NULL," + " vectors BLOB NOT NULL," + " UNIQUE (centroid_id, cell_id))", + "CREATE TABLE \"%w\".\"%w_ivf_rowid_map%02d\"" + " (rowid INTEGER PRIMARY KEY, cell_id INTEGER NOT NULL," + " slot INTEGER NOT NULL)", + "CREATE TABLE \"%w\".\"%w_ivf_vectors%02d\"" + " (rowid INTEGER PRIMARY KEY, vector BLOB NOT NULL)", + }; + const char *zShadow[] = {"_ivf_centroids", "_ivf_cells", "_ivf_rowid_map", + "_ivf_vectors"}; + // without a quantizer, the cells hold the float32 vectors themselves + size_t nCreate = + pNew->vector_columns[i].ivf.quantizer == VEC0_IVF_QUANTIZER_NONE + ? countof(zCreate) - 1 + : countof(zCreate); + for (size_t j = 0; j < nCreate; j++) { + char *zSql = + sqlite3_mprintf(zCreate[j], pNew->schemaName, pNew->tableName, i); + if (!zSql) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not create '%s%02d' shadow table: %s", + zShadow[j], i, sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + } + + // set when the column has centroids + char *zSql = sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_INFO_NAME + "(key, value) VALUES (?, '0')", + pNew->schemaName, pNew->tableName); + char *zKey = sqlite3_mprintf("ivf_trained_%d", i); + if (!zSql || !zKey) { + sqlite3_free(zSql); + sqlite3_free(zKey); + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0); + sqlite3_free(zSql); + if (rc == SQLITE_OK) { + sqlite3_bind_text(stmt, 1, zKey, -1, sqlite3_free); + } else { + sqlite3_free(zKey); + } + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + } +#endif + for (int i = 0; i < pNew->numMetadataColumns; i++) { char *zSql = sqlite3_mprintf("CREATE TABLE " VEC0_SHADOW_METADATA_N_NAME @@ -6880,14 +7254,24 @@ static int vec0Destroy(sqlite3_vtab *pVtab) { p->shadowDiskannVectorsNames[i], p->shadowDiskannNodesNames[i], p->shadowDiskannBufferNames[i], +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // dropping _ivf_cellsNN drops its index + p->shadowIvfCentroidsNames[i], + p->shadowIvfCellsNames[i], + p->shadowIvfRowidMapNames[i], + p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF && + p->vector_columns[i].ivf.quantizer != VEC0_IVF_QUANTIZER_NONE + ? p->shadowIvfVectorsNames[i] + : NULL, #endif }; for (size_t j = 0; j < countof(shadowNames); j++) { if (!shadowNames[j]) { continue; } - // As upstream does, a missing rescore or DiskANN shadow table (j > 0) - // does not keep the table from dropping. + // As upstream does, a missing rescore, DiskANN or IVF shadow table + // (j > 0) does not keep the table from dropping. zSql = sqlite3_mprintf("DROP TABLE %s\"%w\".\"%w\"", j > 0 ? "IF EXISTS " : "", p->schemaName, shadowNames[j]); @@ -9904,7 +10288,8 @@ static int vec0_knn_rowids_append(vec0_vtab *p, sqlite3_value *value, return array_append(rowids, &rowid); } -#if SQLITE_VEC_ENABLE_RESCORE || SQLITE_VEC_ENABLE_DISKANN +#if SQLITE_VEC_ENABLE_RESCORE || SQLITE_VEC_ENABLE_DISKANN || \ + SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE /** * Keeps those of the first *n candidates, rowids and distances in step, whose * distance meets every distance constraint of the KNN query, in their order, @@ -9962,6 +10347,26 @@ static void vec0_knn_keep_within_distance(const char *idxStr, int argc, } #endif +#if SQLITE_VEC_ENABLE_DISKANN || SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +/** + * Prepares *pStmt from zFormat, which names the schema and then zTable, unless + * a previous call prepared it. + */ +static int vec0_cached_stmt(vec0_vtab *p, sqlite3_stmt **pStmt, + const char *zFormat, const char *zTable) { + if (*pStmt) { + return SQLITE_OK; + } + char *zSql = sqlite3_mprintf(zFormat, p->schemaName, zTable); + if (!zSql) { + return SQLITE_NOMEM; + } + int rc = sqlite3_prepare_v2(p->db, zSql, -1, pStmt, NULL); + sqlite3_free(zSql); + return rc; +} +#endif + #if SQLITE_VEC_ENABLE_RESCORE // A vector column indexed by rescore() keeps two shadow tables: // _rescore_chunksNN holds its vectors quantized, in the layout of @@ -10788,24 +11193,6 @@ static f32 diskann_distance_quantized(struct VectorColumnDefinition *col, return FLT_MAX; } -/** - * Prepares *pStmt from zFormat, which names the schema and then zTable, unless - * a previous call prepared it. - */ -static int diskann_stmt(vec0_vtab *p, sqlite3_stmt **pStmt, const char *zFormat, - const char *zTable) { - if (*pStmt) { - return SQLITE_OK; - } - char *zSql = sqlite3_mprintf(zFormat, p->schemaName, zTable); - if (!zSql) { - return SQLITE_NOMEM; - } - int rc = sqlite3_prepare_v2(p->db, zSql, -1, pStmt, NULL); - sqlite3_free(zSql); - return rc; -} - /** * Runs zFormat, a statement on table zTable of the schema whose one parameter * is rowid, to its end, and sets a message naming the row and the table when @@ -10939,11 +11326,11 @@ static int diskann_node_read(vec0_vtab *p, int i, i64 rowid, u8 **outValidity, struct VectorColumnDefinition *col = &p->vector_columns[i]; struct Vec0DiskannConfig *cfg = &col->diskann; sqlite3_stmt *stmt = NULL; - int rc = diskann_stmt(p, &p->stmtDiskannNodeRead[i], - "SELECT neighbors_validity, neighbor_ids, " - "neighbor_quantized_vectors FROM \"%w\".\"%w\" " - "WHERE rowid = ?", - p->shadowDiskannNodesNames[i]); + int rc = vec0_cached_stmt(p, &p->stmtDiskannNodeRead[i], + "SELECT neighbors_validity, neighbor_ids, " + "neighbor_quantized_vectors FROM \"%w\".\"%w\" " + "WHERE rowid = ?", + p->shadowDiskannNodesNames[i]); if (rc != SQLITE_OK) { goto failed; } @@ -11103,11 +11490,11 @@ static int diskann_node_write(vec0_vtab *p, int i, i64 rowid, const u8 *validity, int validitySize, const u8 *neighborIds, int neighborIdsSize, const u8 *qvecs, int qvecsSize) { - int rc = diskann_stmt(p, &p->stmtDiskannNodeWrite[i], - "INSERT OR REPLACE INTO \"%w\".\"%w\" (rowid, " - "neighbors_validity, neighbor_ids, " - "neighbor_quantized_vectors) VALUES (?, ?, ?, ?)", - p->shadowDiskannNodesNames[i]); + int rc = vec0_cached_stmt(p, &p->stmtDiskannNodeWrite[i], + "INSERT OR REPLACE INTO \"%w\".\"%w\" (rowid, " + "neighbors_validity, neighbor_ids, " + "neighbor_quantized_vectors) VALUES (?, ?, ?, ?)", + p->shadowDiskannNodesNames[i]); if (rc == SQLITE_OK) { sqlite3_stmt *stmt = p->stmtDiskannNodeWrite[i]; sqlite3_bind_int64(stmt, 1, rowid); @@ -11140,9 +11527,9 @@ static int diskann_node_write(vec0_vtab *p, int i, i64 rowid, static int diskann_vector_read(vec0_vtab *p, int i, i64 rowid, void **outVector, int *outVectorSize) { sqlite3_stmt *stmt = NULL; - int rc = diskann_stmt(p, &p->stmtDiskannVectorRead[i], - "SELECT vector FROM \"%w\".\"%w\" WHERE rowid = ?", - p->shadowDiskannVectorsNames[i]); + int rc = vec0_cached_stmt(p, &p->stmtDiskannVectorRead[i], + "SELECT vector FROM \"%w\".\"%w\" WHERE rowid = ?", + p->shadowDiskannVectorsNames[i]); if (rc != SQLITE_OK) { goto failed; } @@ -11201,10 +11588,11 @@ static int diskann_vector_read(vec0_vtab *p, int i, i64 rowid, void **outVector, */ static int diskann_vector_write(vec0_vtab *p, int i, i64 rowid, const void *vector) { - int rc = diskann_stmt(p, &p->stmtDiskannVectorWrite[i], - "INSERT OR REPLACE INTO \"%w\".\"%w\" (rowid, vector) " - "VALUES (?, ?)", - p->shadowDiskannVectorsNames[i]); + int rc = + vec0_cached_stmt(p, &p->stmtDiskannVectorWrite[i], + "INSERT OR REPLACE INTO \"%w\".\"%w\" (rowid, vector) " + "VALUES (?, ?)", + p->shadowDiskannVectorsNames[i]); if (rc == SQLITE_OK) { sqlite3_stmt *stmt = p->stmtDiskannVectorWrite[i]; sqlite3_bind_int64(stmt, 1, rowid); @@ -12660,48 +13048,2037 @@ static int vec0_all_columns_diskann(vec0_vtab *p) { } #endif -int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, - const char *idxStr, int argc, sqlite3_value **argv) { - assert(argc == (int)((strlen(idxStr) - 1) / 4)); - int rc; - struct vec0_query_knn_data *knn_data; - - int vectorColumnIdx = idxNum; - struct VectorColumnDefinition *vector_column = - &p->vector_columns[vectorColumnIdx]; +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +// A vector column indexed by ivf() keeps its vectors in cells of up to 64 +// rows. Its k-means centroids are in _ivf_centroidsNN, held as the cells hold +// vectors. _ivf_cellsNN has one row per cell: the centroid whose rows it +// holds, or -1 for rows no clustering has placed, its count of rows, and in +// slots a validity bit, the rowid and the vector of each row. _ivf_rowid_mapNN +// gives each row's cell and slot. With a quantizer, the cells hold the vectors +// quantized and _ivf_vectorsNN holds each float32 vector by rowid. The +// ivf_trained_N row of _info is 1 once the column has centroids. +// https://github.com/asg017/sqlite-vec/blob/04d28bd/sqlite-vec-ivf.c +// https://github.com/asg017/sqlite-vec/blob/04d28bd/sqlite-vec-ivf-kmeans.c +// with the fixes of https://github.com/asg017/sqlite-vec/pull/313 - struct Array *arrayRowidsIn = NULL; - sqlite3_stmt *stmtChunks = NULL; - void *queryVector; - size_t dimensions; - enum VectorElementType elementType; - vector_cleanup queryVectorCleanup = vector_cleanup_noop; - char *pzError; - knn_data = sqlite3_malloc(sizeof(*knn_data)); - if (!knn_data) { - return SQLITE_NOMEM; +/** + * The bytes of one vector in a cell of column col, and of one centroid. + */ +static int ivf_vector_size(struct VectorColumnDefinition *col) { + switch (col->ivf.quantizer) { + case VEC0_IVF_QUANTIZER_INT8: + return (int)col->dimensions; + case VEC0_IVF_QUANTIZER_BINARY: + return (int)(col->dimensions / CHAR_BIT); + case VEC0_IVF_QUANTIZER_NONE: + break; } - memset(knn_data, 0, sizeof(*knn_data)); - // array of `struct Vec0MetadataIn`, IF there are any `xxx in (...)` metadata - // constraints - struct Array *aMetadataIn = NULL; + return (int)(col->dimensions * sizeof(f32)); +} - int query_idx = -1; - int k_idx = -1; - int rowid_in_idx = -1; - int rowid_eq_idx = -1; - int mmr_lambda_idx = -1; - for (int i = 0; i < argc; i++) { - if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_MATCH) { - query_idx = i; - } - if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_K) { - k_idx = i; +/** + * Writes the int8 form of a float32 vector: each value clamped to [-1, 1] and + * scaled to [-127, 127], truncating toward zero, with NaN as 127. + */ +void ivf_quantize_int8(const f32 *src, i8 *dst, size_t dimensions) { + for (size_t i = 0; i < dimensions; i++) { + f32 v = src[i]; + if (!(v <= 1.0f)) { + v = 1.0f; } - if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_ROWID_IN) { - rowid_in_idx = i; + if (!(v >= -1.0f)) { + v = -1.0f; } - if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_ROWID_EQ) { + dst[i] = (i8)(v * 127.0f); + } +} + +/** + * Writes the binary form of a float32 vector: a bit per value, set when the + * value is > 0. dimensions is a multiple of 8. + */ +void ivf_quantize_binary(const f32 *src, u8 *dst, size_t dimensions) { + memset(dst, 0, dimensions / CHAR_BIT); + for (size_t i = 0; i < dimensions; i++) { + if (src[i] > 0.0f) { + dst[i / CHAR_BIT] |= (u8)(1 << (i % CHAR_BIT)); + } + } +} + +/** + * Writes the form of a float32 vector that the cells of column col hold. + */ +static void ivf_quantize(struct VectorColumnDefinition *col, const f32 *src, + void *dst) { + switch (col->ivf.quantizer) { + case VEC0_IVF_QUANTIZER_INT8: + ivf_quantize_int8(src, (i8 *)dst, col->dimensions); + return; + case VEC0_IVF_QUANTIZER_BINARY: + ivf_quantize_binary(src, (u8 *)dst, col->dimensions); + return; + case VEC0_IVF_QUANTIZER_NONE: + break; + } + memcpy(dst, src, col->dimensions * sizeof(f32)); +} + +/** + * The distance between two vectors as the cells of column col hold them: in + * the column's metric for float32 and int8 vectors, and hamming for binary. + */ +static f32 ivf_distance(struct VectorColumnDefinition *col, const void *a, + const void *b) { + size_t dimensions = col->dimensions; + switch (col->ivf.quantizer) { + case VEC0_IVF_QUANTIZER_INT8: + switch (col->distance_metric) { + case VEC0_DISTANCE_METRIC_L2: + return distance_l2_sqr_int8(a, b, &dimensions); + case VEC0_DISTANCE_METRIC_COSINE: + return distance_cosine_int8(a, b, &dimensions); + case VEC0_DISTANCE_METRIC_L1: + return (f32)distance_l1_int8(a, b, &dimensions); + } + break; + case VEC0_IVF_QUANTIZER_BINARY: + return distance_hamming(a, b, &dimensions); + case VEC0_IVF_QUANTIZER_NONE: + return vec0_compute_distance(col, a, b); + } + return FLT_MAX; +} + +static u32 ivf_xorshift32(u32 *state) { + u32 x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} + +static f32 ivf_l2_distance(const f32 *a, const f32 *b, size_t dimensions) { + f32 sum = 0.0f; + for (size_t d = 0; d < dimensions; d++) { + f32 diff = a[d] - b[d]; + sum += diff * diff; + } + return sum; +} + +/** + * The index of the nearest of the k centroids to vector by L2 distance. + */ +static int ivf_kmeans_nearest(const f32 *vector, const f32 *centroids, + size_t dimensions, int k) { + f32 minDistance = FLT_MAX; + int best = 0; + for (int c = 0; c < k; c++) { + f32 distance = + ivf_l2_distance(vector, ¢roids[(size_t)c * dimensions], dimensions); + if (distance < minDistance) { + minDistance = distance; + best = c; + } + } + return best; +} + +/** + * k-means++: picks the first of k centroids from the n vectors at random, and + * each next one with probability in proportion to its squared distance from + * the nearest centroid picked. + */ +static int ivf_kmeans_init_plusplus(const f32 *vectors, int n, + size_t dimensions, int k, u32 seed, + f32 *centroids) { + if (seed == 0) { + seed = 42; + } + size_t size = dimensions * sizeof(f32); + int first = (int)(ivf_xorshift32(&seed) % (u32)n); + memcpy(centroids, &vectors[(size_t)first * dimensions], size); + if (k == 1) { + return SQLITE_OK; + } + f32 *distances = sqlite3_malloc64((sqlite3_uint64)n * sizeof(f32)); + if (!distances) { + return SQLITE_NOMEM; + } + for (int c = 1; c < k; c++) { + // the distance of each vector from its nearest centroid so far + double total = 0.0; + for (int i = 0; i < n; i++) { + f32 distance = + ivf_l2_distance(&vectors[(size_t)i * dimensions], + ¢roids[(size_t)(c - 1) * dimensions], dimensions); + if (c == 1 || distance < distances[i]) { + distances[i] = distance; + } + total += distances[i]; + } + int pick = n - 1; + if (total <= 0.0) { + pick = (int)(ivf_xorshift32(&seed) % (u32)n); + } else { + double threshold = + ((double)ivf_xorshift32(&seed) / (double)0xFFFFFFFF) * total; + double cumulative = 0.0; + for (int i = 0; i < n; i++) { + cumulative += distances[i]; + if (cumulative >= threshold) { + pick = i; + break; + } + } + } + memcpy(¢roids[(size_t)c * dimensions], + &vectors[(size_t)pick * dimensions], size); + } + sqlite3_free(distances); + return SQLITE_OK; +} + +/** + * Lloyd's k-means of the n vectors of dimensions floats each, from a k-means++ + * start seeded with seed, for up to max_iter rounds or until no vector moves. + * A centroid that loses all its vectors moves to the vector farthest from its + * own centroid. Writes the k centroids to out_centroids, k <= n. + */ +int ivf_kmeans(const f32 *vectors, int n, size_t dimensions, int k, + int max_iter, u32 seed, f32 *out_centroids) { + if (n <= 0 || dimensions == 0 || k <= 0 || k > n) { + return SQLITE_MISUSE; + } + int *assignments = sqlite3_malloc64((sqlite3_uint64)n * sizeof(int)); + f32 *sums = sqlite3_malloc64((sqlite3_uint64)k * dimensions * sizeof(f32)); + int *counts = sqlite3_malloc64((sqlite3_uint64)k * sizeof(int)); + int rc = SQLITE_NOMEM; + if (!assignments || !sums || !counts) { + goto cleanup; + } + for (int i = 0; i < n; i++) { + assignments[i] = -1; + } + rc = ivf_kmeans_init_plusplus(vectors, n, dimensions, k, seed, out_centroids); + if (rc != SQLITE_OK) { + goto cleanup; + } + for (int iter = 0; iter < max_iter; iter++) { + int changed = 0; + for (int i = 0; i < n; i++) { + int nearest = ivf_kmeans_nearest(&vectors[(size_t)i * dimensions], + out_centroids, dimensions, k); + if (nearest != assignments[i]) { + assignments[i] = nearest; + changed++; + } + } + if (changed == 0) { + break; + } + memset(sums, 0, (size_t)k * dimensions * sizeof(f32)); + memset(counts, 0, (size_t)k * sizeof(int)); + for (int i = 0; i < n; i++) { + int c = assignments[i]; + counts[c]++; + for (size_t d = 0; d < dimensions; d++) { + sums[(size_t)c * dimensions + d] += vectors[(size_t)i * dimensions + d]; + } + } + for (int c = 0; c < k; c++) { + f32 *centroid = &out_centroids[(size_t)c * dimensions]; + if (counts[c] > 0) { + for (size_t d = 0; d < dimensions; d++) { + centroid[d] = sums[(size_t)c * dimensions + d] / (f32)counts[c]; + } + continue; + } + f32 maxDistance = -1.0f; + int farthest = 0; + for (int i = 0; i < n; i++) { + f32 distance = ivf_l2_distance( + &vectors[(size_t)i * dimensions], + &out_centroids[(size_t)assignments[i] * dimensions], dimensions); + if (distance > maxDistance) { + maxDistance = distance; + farthest = i; + } + } + memcpy(centroid, &vectors[(size_t)farthest * dimensions], + dimensions * sizeof(f32)); + } + } + rc = SQLITE_OK; + +cleanup: + sqlite3_free(assignments); + sqlite3_free(sums); + sqlite3_free(counts); + return rc; +} + +/** + * Sets a message saying the statement could not zWhat, as in "read the cells + * of", the IVF index of column i, with SQLite's reason, unless it was + * interrupted, and returns rc. + */ +static int ivf_failed(vec0_vtab *p, int i, const char *zWhat, int rc) { + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not %s the IVF index of vector column %d: %s", zWhat, + i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Reads whether column i has centroids into *outTrained, from its + * ivf_trained_N row of _info, which every connection's writes keep current. + */ +static int ivf_trained(vec0_vtab *p, int i, int *outTrained) { + char *zKey = sqlite3_mprintf("ivf_trained_%d", i); + if (!zKey) { + return SQLITE_NOMEM; + } + int rc = vec0_cached_stmt( + p, &p->ivfStmts[i].trained, + "SELECT value FROM \"%w\".\"%w_info\" WHERE key = ?", p->tableName); + if (rc != SQLITE_OK) { + sqlite3_free(zKey); + } else { + sqlite3_stmt *stmt = p->ivfStmts[i].trained; + sqlite3_bind_text(stmt, 1, zKey, -1, sqlite3_free); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + *outTrained = sqlite3_column_int(stmt, 0) == 1; + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_ROW) { + return SQLITE_OK; + } + if (rc == SQLITE_DONE) { + vtab_set_error(&p->base, + "the _info table has no ivf_trained_%d row for vector " + "column %d", + i, i); + return SQLITE_ERROR; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not read whether vector column %d has IVF " + "centroids: %s", + i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Sets the ivf_trained_N row of _info of column i. + */ +static int ivf_set_trained(vec0_vtab *p, int i, int trained) { + sqlite3_stmt *stmt = NULL; + char *zSql = sqlite3_mprintf("INSERT OR REPLACE INTO " VEC0_SHADOW_INFO_NAME + " (key, value) VALUES (?, ?)", + p->schemaName, p->tableName); + char *zKey = sqlite3_mprintf("ivf_trained_%d", i); + if (!zSql || !zKey) { + sqlite3_free(zSql); + sqlite3_free(zKey); + return SQLITE_NOMEM; + } + int rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + sqlite3_free(zKey); + } else { + sqlite3_bind_text(stmt, 1, zKey, -1, sqlite3_free); + sqlite3_bind_text(stmt, 2, trained ? "1" : "0", -1, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not record whether vector column %d has IVF " + "centroids: %s", + i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Sets or clears the validity bit of slot in cell cellId of column i, and + * adds delta to the cell's count of rows; sets a message naming the cell when + * it cannot. + */ +static int ivf_cell_mark(vec0_vtab *p, int i, i64 cellId, int slot, int valid, + int delta) { + sqlite3_blob *blob = NULL; + int rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowIvfCellsNames[i], + "validity", cellId, 1, &blob); + if (rc == SQLITE_OK) { + u8 byte; + if (sqlite3_blob_bytes(blob) != VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT) { + sqlite3_blob_close(blob); + vtab_set_error( + &p->base, "cell %lld of the IVF index of vector column %d is corrupt", + cellId, i); + return SQLITE_ERROR; + } + rc = sqlite3_blob_read(blob, &byte, 1, slot / CHAR_BIT); + if (rc == SQLITE_OK) { + if (valid) { + byte |= (u8)(1 << (slot % CHAR_BIT)); + } else { + byte &= (u8) ~(1 << (slot % CHAR_BIT)); + } + rc = sqlite3_blob_write(blob, &byte, 1, slot / CHAR_BIT); + } + int brc = sqlite3_blob_close(blob); + if (rc == SQLITE_OK) { + rc = brc; + } + } + if (rc != SQLITE_OK) { + rc = vec0_error_rc(rc); + goto failed; + } + rc = vec0_cached_stmt(p, &p->ivfStmts[i].cellCount, + "UPDATE \"%w\".\"%w\" SET n_vectors = n_vectors + ? " + "WHERE rowid = ?", + p->shadowIvfCellsNames[i]); + if (rc != SQLITE_OK) { + goto failed; + } + sqlite3_stmt *stmt = p->ivfStmts[i].cellCount; + sqlite3_bind_int(stmt, 1, delta); + sqlite3_bind_int64(stmt, 2, cellId); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + +failed: + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not write cell %lld of the IVF index of vector " + "column %d: %s", + cellId, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Writes n bytes of data at offset in column zColumn of row cellId of the + * cells of column i, and sets a message naming the cell when it cannot. + */ +static int ivf_cell_write(vec0_vtab *p, int i, i64 cellId, const char *zColumn, + const void *data, int n, int offset) { + sqlite3_blob *blob = NULL; + int rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowIvfCellsNames[i], + zColumn, cellId, 1, &blob); + if (rc == SQLITE_OK) { + if (sqlite3_blob_bytes(blob) < offset + n) { + sqlite3_blob_close(blob); + vtab_set_error( + &p->base, "cell %lld of the IVF index of vector column %d is corrupt", + cellId, i); + return SQLITE_ERROR; + } + rc = sqlite3_blob_write(blob, data, n, offset); + int brc = sqlite3_blob_close(blob); + if (rc == SQLITE_OK) { + rc = brc; + } + if (rc == SQLITE_OK) { + return SQLITE_OK; + } + } + rc = vec0_error_rc(rc); + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not write cell %lld of the IVF index of vector " + "column %d: %s", + cellId, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Records in _ivf_rowid_mapNN that row rowid of column i is in slot slot of + * cell cellId, and sets a message naming the row when it cannot. + */ +static int ivf_map_write(vec0_vtab *p, int i, i64 rowid, i64 cellId, int slot) { + int rc = vec0_cached_stmt(p, &p->ivfStmts[i].mapWrite, + "INSERT INTO \"%w\".\"%w\" (rowid, cell_id, slot) " + "VALUES (?, ?, ?)", + p->shadowIvfRowidMapNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].mapWrite; + sqlite3_bind_int64(stmt, 1, rowid); + sqlite3_bind_int64(stmt, 2, cellId); + sqlite3_bind_int(stmt, 3, slot); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not write row %lld of the IVF index of vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Places row rowid of column i, whose vector the cells hold as qvector, in the + * first free slot of a cell of centroid centroidId, making a cell when every + * cell of the centroid is full, and records the cell and slot in + * _ivf_rowid_mapNN. + */ +static int ivf_place(vec0_vtab *p, int i, i64 centroidId, i64 rowid, + const void *qvector) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int vectorSize = ivf_vector_size(col); + int found = 0; + i64 cellId = 0; + int slot = -1; + + int rc = vec0_cached_stmt(p, &p->ivfStmts[i].cellFind, + "SELECT rowid, validity FROM \"%w\".\"%w\" WHERE " + "centroid_id = ? AND n_vectors < ? LIMIT 1", + p->shadowIvfCellsNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].cellFind; + sqlite3_bind_int64(stmt, 1, centroidId); + sqlite3_bind_int(stmt, 2, VEC0_IVF_CELL_MAX_VECTORS); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + found = 1; + cellId = sqlite3_column_int64(stmt, 0); + const u8 *validity = sqlite3_column_blob(stmt, 1); + if (validity && sqlite3_column_bytes(stmt, 1) == + VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT) { + for (int s = 0; s < VEC0_IVF_CELL_MAX_VECTORS; s++) { + if (!(validity[s / CHAR_BIT] & (1 << (s % CHAR_BIT)))) { + slot = s; + break; + } + } + } + rc = SQLITE_OK; + } else if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + } else { + rc = vec0_error_rc(rc); + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + } + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "read the cells of", rc); + } + if (found && slot < 0) { + // the count says the cell has room, but no slot is free + vtab_set_error(&p->base, + "cell %lld of the IVF index of vector column %d is corrupt", + cellId, i); + return SQLITE_ERROR; + } + + if (!found) { + char *zSql = sqlite3_mprintf( + "INSERT INTO \"%w\".\"%w\" (centroid_id, n_vectors, validity, rowids, " + "vectors) VALUES (?, 0, ?, ?, ?)", + p->schemaName, p->shadowIvfCellsNames[i]); + if (!zSql) { + return SQLITE_NOMEM; + } + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc == SQLITE_OK) { + sqlite3_bind_int64(stmt, 1, centroidId); + sqlite3_bind_zeroblob(stmt, 2, VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT); + sqlite3_bind_zeroblob(stmt, 3, VEC0_IVF_CELL_MAX_VECTORS * sizeof(i64)); + sqlite3_bind_zeroblob(stmt, 4, VEC0_IVF_CELL_MAX_VECTORS * vectorSize); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + rc = rc == SQLITE_DONE ? SQLITE_OK : vec0_error_rc(rc); + } + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "add a cell to", rc); + } + cellId = sqlite3_last_insert_rowid(p->db); + slot = 0; + } + + rc = ivf_cell_write(p, i, cellId, "rowids", &rowid, sizeof(i64), + slot * (int)sizeof(i64)); + if (rc == SQLITE_OK) { + rc = ivf_cell_write(p, i, cellId, "vectors", qvector, vectorSize, + slot * vectorSize); + } + if (rc == SQLITE_OK) { + rc = ivf_cell_mark(p, i, cellId, slot, 1, 1); + } + if (rc == SQLITE_OK) { + rc = ivf_map_write(p, i, rowid, cellId, slot); + } + return rc; +} + +/** + * Reads the cell and slot of row rowid of column i from _ivf_rowid_mapNN. + * Returns SQLITE_EMPTY when the row has none, and otherwise sets a message + * naming the row when it cannot read it. + */ +static int ivf_map_read(vec0_vtab *p, int i, i64 rowid, i64 *outCellId, + int *outSlot) { + int rc = vec0_cached_stmt( + p, &p->ivfStmts[i].mapRead, + "SELECT cell_id, slot FROM \"%w\".\"%w\" WHERE rowid = ?", + p->shadowIvfRowidMapNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].mapRead; + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + *outCellId = sqlite3_column_int64(stmt, 0); + *outSlot = sqlite3_column_int(stmt, 1); + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_ROW) { + if (*outSlot < 0 || *outSlot >= VEC0_IVF_CELL_MAX_VECTORS) { + vtab_set_error(&p->base, + "row %lld of the IVF index of vector column %d is " + "corrupt", + rowid, i); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + if (rc == SQLITE_DONE) { + return SQLITE_EMPTY; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not read row %lld of the IVF index of vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Takes row rowid of column i out of its cell and out of _ivf_rowid_mapNN, + * if it is in one. + */ +static int ivf_unplace(vec0_vtab *p, int i, i64 rowid) { + i64 cellId; + int slot; + int rc = ivf_map_read(p, i, rowid, &cellId, &slot); + if (rc == SQLITE_EMPTY) { + return SQLITE_OK; + } + if (rc != SQLITE_OK) { + return rc; + } + rc = ivf_cell_mark(p, i, cellId, slot, 0, -1); + if (rc != SQLITE_OK) { + return rc; + } + rc = vec0_cached_stmt(p, &p->ivfStmts[i].mapDelete, + "DELETE FROM \"%w\".\"%w\" WHERE rowid = ?", + p->shadowIvfRowidMapNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].mapDelete; + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not remove row %lld from the IVF index of vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Reads the centroid of column i nearest qvector, a vector as the cells hold + * it, into *outCentroidId. Fails when the column has no centroid. + */ +static int ivf_nearest_centroid(vec0_vtab *p, int i, const void *qvector, + i64 *outCentroidId) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int vectorSize = ivf_vector_size(col); + int rc = vec0_cached_stmt(p, &p->ivfStmts[i].centroids, + "SELECT centroid_id, centroid FROM \"%w\".\"%w\"", + p->shadowIvfCentroidsNames[i]); + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "read the centroids of", rc); + } + sqlite3_stmt *stmt = p->ivfStmts[i].centroids; + f32 minDistance = FLT_MAX; + int found = 0; + int corrupt = 0; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const void *centroid = sqlite3_column_blob(stmt, 1); + if (!centroid || sqlite3_column_bytes(stmt, 1) != vectorSize) { + vtab_set_error(&p->base, + "centroid %lld of the IVF index of vector column %d is " + "corrupt", + sqlite3_column_int64(stmt, 0), i); + corrupt = 1; + break; + } + f32 distance = ivf_distance(col, qvector, centroid); + if (!found || distance < minDistance) { + minDistance = distance; + *outCentroidId = sqlite3_column_int64(stmt, 0); + found = 1; + } + } + sqlite3_reset(stmt); + if (corrupt) { + return SQLITE_ERROR; + } + if (rc == SQLITE_DONE) { + if (!found) { + vtab_set_error(&p->base, "vector column %d has no IVF centroids", i); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + return ivf_failed(p, i, "read the centroids of", vec0_error_rc(rc)); +} + +/** + * Adds row rowid of column i, with float32 vector, to the index: in a cell of + * the centroid nearest its quantized vector, or of no centroid while the + * column has none, and with a quantizer to _ivf_vectorsNN. + */ +static int ivf_insert(vec0_vtab *p, int i, i64 rowid, const void *vector) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + void *qvector = sqlite3_malloc(ivf_vector_size(col)); + if (!qvector) { + return SQLITE_NOMEM; + } + ivf_quantize(col, vector, qvector); + int trained = 0; + i64 centroidId = VEC0_IVF_UNASSIGNED_CENTROID_ID; + int rc = ivf_trained(p, i, &trained); + if (rc == SQLITE_OK && trained) { + rc = ivf_nearest_centroid(p, i, qvector, ¢roidId); + } + if (rc == SQLITE_OK) { + rc = ivf_place(p, i, centroidId, rowid, qvector); + } + sqlite3_free(qvector); + if (rc != SQLITE_OK || col->ivf.quantizer == VEC0_IVF_QUANTIZER_NONE) { + return rc; + } + + rc = vec0_cached_stmt( + p, &p->ivfStmts[i].vectorWrite, + "INSERT INTO \"%w\".\"%w\" (rowid, vector) VALUES (?, ?)", + p->shadowIvfVectorsNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].vectorWrite; + sqlite3_bind_int64(stmt, 1, rowid); + sqlite3_bind_blob(stmt, 2, vector, (int)(col->dimensions * sizeof(f32)), + SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not write the IVF vector of row %lld in vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Removes row rowid of column i from the index. + */ +static int ivf_delete(vec0_vtab *p, int i, i64 rowid) { + int rc = ivf_unplace(p, i, rowid); + if (rc != SQLITE_OK || + p->vector_columns[i].ivf.quantizer == VEC0_IVF_QUANTIZER_NONE) { + return rc; + } + rc = vec0_cached_stmt(p, &p->ivfStmts[i].vectorDelete, + "DELETE FROM \"%w\".\"%w\" WHERE rowid = ?", + p->shadowIvfVectorsNames[i]); + if (rc == SQLITE_OK) { + sqlite3_stmt *stmt = p->ivfStmts[i].vectorDelete; + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not remove the IVF vector of row %lld in vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Reads the float32 vector of row rowid of column i into *outVector, to be + * freed with sqlite3_free(), and its size into *outVectorSize unless that is + * NULL: from its cell, or with a quantizer from _ivf_vectorsNN. Returns + * SQLITE_EMPTY when the column has no vector with that rowid, and otherwise + * sets a message naming the vector when it cannot read it. + */ +static int ivf_vector_read(vec0_vtab *p, int i, i64 rowid, void **outVector, + int *outVectorSize) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int size = (int)(col->dimensions * sizeof(f32)); + void *vector = NULL; + int rc; + if (col->ivf.quantizer == VEC0_IVF_QUANTIZER_NONE) { + i64 cellId; + int slot; + rc = ivf_map_read(p, i, rowid, &cellId, &slot); + if (rc != SQLITE_OK) { + return rc; + } + sqlite3_blob *blob = NULL; + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowIvfCellsNames[i], + "vectors", cellId, 0, &blob); + if (rc != SQLITE_OK) { + rc = vec0_error_rc(rc); + goto failed; + } + if (sqlite3_blob_bytes(blob) < (slot + 1) * size) { + sqlite3_blob_close(blob); + vtab_set_error(&p->base, + "cell %lld of the IVF index of vector column %d is " + "corrupt", + cellId, i); + return SQLITE_ERROR; + } + vector = sqlite3_malloc(size); + if (!vector) { + sqlite3_blob_close(blob); + return SQLITE_NOMEM; + } + rc = sqlite3_blob_read(blob, vector, size, slot * size); + sqlite3_blob_close(blob); + if (rc != SQLITE_OK) { + sqlite3_free(vector); + rc = vec0_error_rc(rc); + goto failed; + } + } else { + rc = vec0_cached_stmt(p, &p->ivfStmts[i].vectorRead, + "SELECT vector FROM \"%w\".\"%w\" WHERE rowid = ?", + p->shadowIvfVectorsNames[i]); + if (rc != SQLITE_OK) { + goto failed; + } + sqlite3_stmt *stmt = p->ivfStmts[i].vectorRead; + sqlite3_bind_int64(stmt, 1, rowid); + int step = sqlite3_step(stmt); + if (step == SQLITE_ROW) { + const void *blob = sqlite3_column_blob(stmt, 0); + if (!blob || sqlite3_column_bytes(stmt, 0) != size) { + vtab_set_error(&p->base, + "the IVF vector of row %lld in vector column %d is " + "corrupt", + rowid, i); + rc = SQLITE_ERROR; + } else { + vector = sqlite3_malloc(size); + if (vector) { + memcpy(vector, blob, size); + } else { + rc = SQLITE_NOMEM; + } + } + } else if (step == SQLITE_DONE) { + rc = SQLITE_EMPTY; + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (step != SQLITE_ROW && step != SQLITE_DONE) { + rc = vec0_error_rc(step); + goto failed; + } + if (rc != SQLITE_OK) { + return rc; + } + } + *outVector = vector; + if (outVectorSize) { + *outVectorSize = size; + } + return SQLITE_OK; + +failed: + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not read the IVF vector of row %lld in vector " + "column %d: %s", + rowid, i, sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Runs zFormat, a statement without parameters on table zTable of the schema, + * to its end, and sets a message naming the table when it fails. + */ +static int ivf_exec(vec0_vtab *p, const char *zFormat, const char *zTable) { + sqlite3_stmt *stmt = NULL; + char *zSql = sqlite3_mprintf(zFormat, p->schemaName, zTable); + if (!zSql) { + return SQLITE_NOMEM; + } + int rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc == SQLITE_OK) { + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc == SQLITE_DONE) { + return SQLITE_OK; + } + rc = vec0_error_rc(rc); + } + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, "could not change %s: %s", zTable, + sqlite3_errmsg(p->db)); + } + return rc; +} + +/** + * Appends to rowids and vectors the rowid and the vector, as the cells of + * column i hold it, of each row in a cell the statement stmt reads, whose + * columns are a cell's rowid, validity, rowids and vectors. Leaves stmt reset, + * and sets a message naming the cells when it cannot read them. + */ +static int ivf_read_cells(vec0_vtab *p, int i, sqlite3_stmt *stmt, + struct Array *rowids, struct Array *vectors) { + int vectorSize = ivf_vector_size(&p->vector_columns[i]); + int rc = SQLITE_OK; + int step; + while ((step = sqlite3_step(stmt)) == SQLITE_ROW) { + const u8 *validity = sqlite3_column_blob(stmt, 1); + const u8 *cellRowids = sqlite3_column_blob(stmt, 2); + const u8 *cellVectors = sqlite3_column_blob(stmt, 3); + if (!validity || !cellRowids || !cellVectors || + sqlite3_column_bytes(stmt, 1) != VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT || + sqlite3_column_bytes(stmt, 2) != + VEC0_IVF_CELL_MAX_VECTORS * (int)sizeof(i64) || + sqlite3_column_bytes(stmt, 3) != + VEC0_IVF_CELL_MAX_VECTORS * vectorSize) { + vtab_set_error(&p->base, + "cell %lld of the IVF index of vector column %d is " + "corrupt", + sqlite3_column_int64(stmt, 0), i); + rc = SQLITE_ERROR; + break; + } + for (int s = 0; s < VEC0_IVF_CELL_MAX_VECTORS; s++) { + if (!(validity[s / CHAR_BIT] & (1 << (s % CHAR_BIT)))) { + continue; + } + i64 rowid; + memcpy(&rowid, cellRowids + (size_t)s * sizeof(i64), sizeof(i64)); + rc = array_append(rowids, &rowid); + if (rc == SQLITE_OK) { + rc = array_append(vectors, cellVectors + (size_t)s * vectorSize); + } + if (rc != SQLITE_OK) { + break; + } + } + if (rc != SQLITE_OK) { + break; + } + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc != SQLITE_OK) { + return rc; + } + if (step == SQLITE_DONE) { + return SQLITE_OK; + } + return ivf_failed(p, i, "read the cells of", vec0_error_rc(step)); +} + +/** + * Appends to rowids and vectors the rowid and float32 vector of every row of + * column i: from its cells, or with a quantizer from _ivf_vectorsNN. + */ +static int ivf_load_vectors(vec0_vtab *p, int i, struct Array *rowids, + struct Array *vectors) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + sqlite3_stmt *stmt = NULL; + int rc; + if (col->ivf.quantizer == VEC0_IVF_QUANTIZER_NONE) { + char *zSql = sqlite3_mprintf( + "SELECT rowid, validity, rowids, vectors FROM \"%w\".\"%w\"", + p->schemaName, p->shadowIvfCellsNames[i]); + if (!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "read the cells of", rc); + } + rc = ivf_read_cells(p, i, stmt, rowids, vectors); + sqlite3_finalize(stmt); + return rc; + } + + int size = (int)(col->dimensions * sizeof(f32)); + char *zSql = sqlite3_mprintf("SELECT rowid, vector FROM \"%w\".\"%w\"", + p->schemaName, p->shadowIvfVectorsNames[i]); + if (!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "read the vectors of", rc); + } + int step; + while ((step = sqlite3_step(stmt)) == SQLITE_ROW) { + i64 rowid = sqlite3_column_int64(stmt, 0); + const void *vector = sqlite3_column_blob(stmt, 1); + if (!vector || sqlite3_column_bytes(stmt, 1) != size) { + vtab_set_error(&p->base, + "the IVF vector of row %lld in vector column %d is " + "corrupt", + rowid, i); + rc = SQLITE_ERROR; + break; + } + rc = array_append(rowids, &rowid); + if (rc == SQLITE_OK) { + rc = array_append(vectors, vector); + } + if (rc != SQLITE_OK) { + break; + } + } + sqlite3_finalize(stmt); + if (rc != SQLITE_OK || step == SQLITE_DONE) { + return rc; + } + return ivf_failed(p, i, "read the vectors of", vec0_error_rc(step)); +} + +struct IvfPlacement { + i64 centroidId; + int row; +}; + +static int ivf_placement_cmp(const void *a, const void *b) { + const struct IvfPlacement *x = a; + const struct IvfPlacement *y = b; + if (x->centroidId != y->centroidId) { + return x->centroidId < y->centroidId ? -1 : 1; + } + return x->row < y->row ? -1 : (x->row > y->row); +} + +/** + * Replaces the centroids, cells and row map of column i: nCentroids float32 + * centroids with ids from 0, and n rows, rowids and float32 vectors, each in + * cells of the centroid centroidOf gives it. + */ +static int ivf_rebuild(vec0_vtab *p, int i, int n, const i64 *rowids, + const f32 *vectors, const i64 *centroidOf, + int nCentroids, const f32 *centroids) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int vectorSize = ivf_vector_size(col); + sqlite3_stmt *stmtCentroid = NULL; + sqlite3_stmt *stmtCell = NULL; + struct IvfPlacement *order = NULL; + u8 *validity = NULL; + i64 *cellRowids = NULL; + u8 *cellVectors = NULL; + void *qvector = NULL; + + int rc = + ivf_exec(p, "DELETE FROM \"%w\".\"%w\"", p->shadowIvfCentroidsNames[i]); + if (rc == SQLITE_OK) { + rc = ivf_exec(p, "DELETE FROM \"%w\".\"%w\"", p->shadowIvfCellsNames[i]); + } + if (rc == SQLITE_OK) { + rc = ivf_exec(p, "DELETE FROM \"%w\".\"%w\"", p->shadowIvfRowidMapNames[i]); + } + if (rc != SQLITE_OK) { + return rc; + } + + qvector = sqlite3_malloc(vectorSize); + validity = sqlite3_malloc(VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT); + cellRowids = sqlite3_malloc(VEC0_IVF_CELL_MAX_VECTORS * sizeof(i64)); + cellVectors = + sqlite3_malloc64((sqlite3_uint64)VEC0_IVF_CELL_MAX_VECTORS * vectorSize); + order = sqlite3_malloc64((sqlite3_uint64)(n > 0 ? n : 1) * sizeof(*order)); + if (!qvector || !validity || !cellRowids || !cellVectors || !order) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + char *zSql = sqlite3_mprintf( + "INSERT INTO \"%w\".\"%w\" (centroid_id, centroid) VALUES (?, ?)", + p->schemaName, p->shadowIvfCentroidsNames[i]); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmtCentroid, NULL); + sqlite3_free(zSql); + for (int c = 0; rc == SQLITE_OK && c < nCentroids; c++) { + ivf_quantize(col, ¢roids[(size_t)c * col->dimensions], qvector); + sqlite3_bind_int(stmtCentroid, 1, c); + sqlite3_bind_blob(stmtCentroid, 2, qvector, vectorSize, SQLITE_STATIC); + rc = sqlite3_step(stmtCentroid); + sqlite3_reset(stmtCentroid); + rc = rc == SQLITE_DONE ? SQLITE_OK : vec0_error_rc(rc); + } + if (rc != SQLITE_OK) { + rc = ivf_failed(p, i, "write the centroids of", rc); + goto cleanup; + } + + zSql = sqlite3_mprintf( + "INSERT INTO \"%w\".\"%w\" (centroid_id, n_vectors, validity, rowids, " + "vectors) VALUES (?, ?, ?, ?, ?)", + p->schemaName, p->shadowIvfCellsNames[i]); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmtCell, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + rc = ivf_failed(p, i, "add a cell to", rc); + goto cleanup; + } + + for (int r = 0; r < n; r++) { + order[r].centroidId = centroidOf[r]; + order[r].row = r; + } + qsort(order, n, sizeof(*order), ivf_placement_cmp); + + // fill cells of up to VEC0_IVF_CELL_MAX_VECTORS rows, one centroid at a time + for (int start = 0; start < n;) { + int count = 0; + memset(validity, 0, VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT); + memset(cellRowids, 0, VEC0_IVF_CELL_MAX_VECTORS * sizeof(i64)); + memset(cellVectors, 0, (size_t)VEC0_IVF_CELL_MAX_VECTORS * vectorSize); + while (start + count < n && count < VEC0_IVF_CELL_MAX_VECTORS && + order[start + count].centroidId == order[start].centroidId) { + int row = order[start + count].row; + validity[count / CHAR_BIT] |= (u8)(1 << (count % CHAR_BIT)); + cellRowids[count] = rowids[row]; + ivf_quantize(col, &vectors[(size_t)row * col->dimensions], + cellVectors + (size_t)count * vectorSize); + count++; + } + sqlite3_bind_int64(stmtCell, 1, order[start].centroidId); + sqlite3_bind_int(stmtCell, 2, count); + sqlite3_bind_blob(stmtCell, 3, validity, + VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT, SQLITE_STATIC); + sqlite3_bind_blob(stmtCell, 4, cellRowids, + VEC0_IVF_CELL_MAX_VECTORS * (int)sizeof(i64), + SQLITE_STATIC); + sqlite3_bind_blob(stmtCell, 5, cellVectors, + VEC0_IVF_CELL_MAX_VECTORS * vectorSize, SQLITE_STATIC); + rc = sqlite3_step(stmtCell); + sqlite3_reset(stmtCell); + if (rc != SQLITE_DONE) { + rc = ivf_failed(p, i, "add a cell to", vec0_error_rc(rc)); + goto cleanup; + } + i64 cellId = sqlite3_last_insert_rowid(p->db); + for (int s = 0; s < count; s++) { + rc = ivf_map_write(p, i, cellRowids[s], cellId, s); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + start += count; + } + rc = SQLITE_OK; + +cleanup: + sqlite3_finalize(stmtCentroid); + sqlite3_finalize(stmtCell); + sqlite3_free(order); + sqlite3_free(validity); + sqlite3_free(cellRowids); + sqlite3_free(cellVectors); + sqlite3_free(qvector); + return rc; +} + +/** + * compute-centroids for column i: clusters its rows with k-means into nlist + * centroids, the column's nlist when nlist is 0 and at most one per row, and + * moves each row to the cells of the centroid nearest it. + */ +static int ivf_compute_centroids(vec0_vtab *p, int i, int nlist, int maxIter, + u32 seed) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + size_t dimensions = col->dimensions; + struct Array rowids, vectors; + f32 *centroids = NULL; + i64 *centroidOf = NULL; + int rc = array_init(&rowids, sizeof(i64), 64); + if (rc != SQLITE_OK) { + return rc; + } + rc = array_init(&vectors, dimensions * sizeof(f32), 64); + if (rc != SQLITE_OK) { + array_cleanup(&rowids); + return rc; + } + rc = ivf_load_vectors(p, i, &rowids, &vectors); + if (rc != SQLITE_OK) { + goto cleanup; + } + int n = (int)rowids.length; + if (nlist <= 0) { + nlist = col->ivf.nlist; + } + if (nlist <= 0) { + vtab_set_error(&p->base, + "compute-centroids needs nlist for vector column %d, which " + "was created with nlist=0", + i); + rc = SQLITE_ERROR; + goto cleanup; + } + if (n == 0) { + vtab_set_error(&p->base, + "compute-centroids needs at least one row in vector " + "column %d", + i); + rc = SQLITE_ERROR; + goto cleanup; + } + if (nlist > n) { + nlist = n; + } + const f32 *data = (const f32 *)vectors.z; + centroids = + sqlite3_malloc64((sqlite3_uint64)nlist * dimensions * sizeof(f32)); + centroidOf = sqlite3_malloc64((sqlite3_uint64)n * sizeof(i64)); + if (!centroids || !centroidOf) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = ivf_kmeans(data, n, dimensions, nlist, maxIter, seed, centroids); + if (rc != SQLITE_OK) { + goto cleanup; + } + for (int r = 0; r < n; r++) { + f32 minDistance = FLT_MAX; + centroidOf[r] = 0; + for (int c = 0; c < nlist; c++) { + f32 distance = vec0_compute_distance(col, &data[(size_t)r * dimensions], + ¢roids[(size_t)c * dimensions]); + if (distance < minDistance) { + minDistance = distance; + centroidOf[r] = c; + } + } + } + rc = ivf_rebuild(p, i, n, (const i64 *)rowids.z, data, centroidOf, nlist, + centroids); + if (rc == SQLITE_OK) { + rc = ivf_set_trained(p, i, 1); + } + +cleanup: + sqlite3_free(centroids); + sqlite3_free(centroidOf); + array_cleanup(&rowids); + array_cleanup(&vectors); + return rc; +} + +/** + * clear-centroids for column i: removes its centroids and moves every row to + * cells of no centroid. + */ +static int ivf_clear_centroids(vec0_vtab *p, int i) { + struct Array rowids, vectors; + i64 *centroidOf = NULL; + int rc = array_init(&rowids, sizeof(i64), 64); + if (rc != SQLITE_OK) { + return rc; + } + rc = array_init(&vectors, p->vector_columns[i].dimensions * sizeof(f32), 64); + if (rc != SQLITE_OK) { + array_cleanup(&rowids); + return rc; + } + rc = ivf_load_vectors(p, i, &rowids, &vectors); + if (rc != SQLITE_OK) { + goto cleanup; + } + int n = (int)rowids.length; + centroidOf = sqlite3_malloc64((sqlite3_uint64)(n > 0 ? n : 1) * sizeof(i64)); + if (!centroidOf) { + rc = SQLITE_NOMEM; + goto cleanup; + } + for (int r = 0; r < n; r++) { + centroidOf[r] = VEC0_IVF_UNASSIGNED_CENTROID_ID; + } + rc = ivf_rebuild(p, i, n, (const i64 *)rowids.z, (const f32 *)vectors.z, + centroidOf, 0, NULL); + if (rc == SQLITE_OK) { + rc = ivf_set_trained(p, i, 0); + } + +cleanup: + sqlite3_free(centroidOf); + array_cleanup(&rowids); + array_cleanup(&vectors); + return rc; +} + +/** + * set-centroid:N for column i: sets its centroid N to the float32 vector. + */ +static int ivf_set_centroid(vec0_vtab *p, int i, i64 centroidId, + const f32 *vector) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int vectorSize = ivf_vector_size(col); + sqlite3_stmt *stmt = NULL; + void *qvector = sqlite3_malloc(vectorSize); + if (!qvector) { + return SQLITE_NOMEM; + } + ivf_quantize(col, vector, qvector); + char *zSql = sqlite3_mprintf("INSERT OR REPLACE INTO \"%w\".\"%w\" " + "(centroid_id, centroid) VALUES (?, ?)", + p->schemaName, p->shadowIvfCentroidsNames[i]); + if (!zSql) { + sqlite3_free(qvector); + return SQLITE_NOMEM; + } + int rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc == SQLITE_OK) { + sqlite3_bind_int64(stmt, 1, centroidId); + sqlite3_bind_blob(stmt, 2, qvector, vectorSize, SQLITE_STATIC); + rc = sqlite3_step(stmt); + rc = rc == SQLITE_DONE ? SQLITE_OK : vec0_error_rc(rc); + } + sqlite3_finalize(stmt); + sqlite3_free(qvector); + if (rc != SQLITE_OK) { + if (rc != SQLITE_INTERRUPT) { + vtab_set_error(&p->base, + "could not write centroid %lld of the IVF index of " + "vector column %d: %s", + centroidId, i, sqlite3_errmsg(p->db)); + } + return rc; + } + return ivf_set_trained(p, i, 1); +} + +/** + * assign-vectors for column i: moves each row in a cell of no centroid to the + * cells of the centroid nearest it. + */ +static int ivf_assign_vectors(vec0_vtab *p, int i) { + struct Array rowids, qvectors; + sqlite3_stmt *stmt = NULL; + int trained = 0; + int rc = ivf_trained(p, i, &trained); + if (rc != SQLITE_OK) { + return rc; + } + if (!trained) { + vtab_set_error(&p->base, "vector column %d has no IVF centroids", i); + return SQLITE_ERROR; + } + rc = array_init(&rowids, sizeof(i64), 64); + if (rc != SQLITE_OK) { + return rc; + } + rc = array_init(&qvectors, ivf_vector_size(&p->vector_columns[i]), 64); + if (rc != SQLITE_OK) { + array_cleanup(&rowids); + return rc; + } + char *zSql = sqlite3_mprintf( + "SELECT rowid, validity, rowids, vectors FROM \"%w\".\"%w\" WHERE " + "centroid_id = ?", + p->schemaName, p->shadowIvfCellsNames[i]); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + rc = ivf_failed(p, i, "read the cells of", rc); + goto cleanup; + } + sqlite3_bind_int64(stmt, 1, VEC0_IVF_UNASSIGNED_CENTROID_ID); + rc = ivf_read_cells(p, i, stmt, &rowids, &qvectors); + if (rc != SQLITE_OK) { + goto cleanup; + } + for (size_t r = 0; r < rowids.length; r++) { + i64 rowid = ((i64 *)rowids.z)[r]; + const void *qvector = (const u8 *)qvectors.z + r * qvectors.element_size; + i64 centroidId; + rc = ivf_nearest_centroid(p, i, qvector, ¢roidId); + if (rc == SQLITE_OK) { + rc = ivf_unplace(p, i, rowid); + } + if (rc == SQLITE_OK) { + rc = ivf_place(p, i, centroidId, rowid, qvector); + } + if (rc != SQLITE_OK) { + goto cleanup; + } + } + sqlite3_finalize(stmt); + stmt = NULL; + zSql = sqlite3_mprintf("DELETE FROM \"%w\".\"%w\" WHERE centroid_id = ?", + p->schemaName, p->shadowIvfCellsNames[i]); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc == SQLITE_OK) { + sqlite3_bind_int64(stmt, 1, VEC0_IVF_UNASSIGNED_CENTROID_ID); + rc = sqlite3_step(stmt); + rc = rc == SQLITE_DONE ? SQLITE_OK : vec0_error_rc(rc); + } + if (rc != SQLITE_OK) { + rc = ivf_failed(p, i, "remove the cells of no centroid from", rc); + } + +cleanup: + sqlite3_finalize(stmt); + array_cleanup(&rowids); + array_cleanup(&qvectors); + return rc; +} + +// a row, or a centroid, and its distance from the query +struct IvfCandidate { + i64 id; + f32 distance; +}; + +// nearest first, a NaN distance after every number, and by id among equals +static int ivf_candidate_cmp(const void *a, const void *b) { + const struct IvfCandidate *x = a; + const struct IvfCandidate *y = b; + int xNan = isnan(x->distance); + int yNan = isnan(y->distance); + if (xNan != yNan) { + return xNan - yNan; + } + if (!xNan && x->distance != y->distance) { + return x->distance < y->distance ? -1 : 1; + } + return x->id < y->id ? -1 : (x->id > y->id); +} + +/** + * Appends to candidates each row in a cell of centroid centroidId of column i, + * at its distance from qquery, the query as the cells hold vectors. + */ +static int ivf_scan_cells(vec0_vtab *p, int i, i64 centroidId, + const void *qquery, struct Array *candidates) { + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int vectorSize = ivf_vector_size(col); + int rc = vec0_cached_stmt(p, &p->ivfStmts[i].cellScan, + "SELECT rowid, validity, rowids, vectors FROM " + "\"%w\".\"%w\" WHERE centroid_id = ?", + p->shadowIvfCellsNames[i]); + if (rc != SQLITE_OK) { + return ivf_failed(p, i, "read the cells of", rc); + } + sqlite3_stmt *stmt = p->ivfStmts[i].cellScan; + sqlite3_bind_int64(stmt, 1, centroidId); + int step; + while ((step = sqlite3_step(stmt)) == SQLITE_ROW) { + const u8 *validity = sqlite3_column_blob(stmt, 1); + const u8 *cellRowids = sqlite3_column_blob(stmt, 2); + const u8 *cellVectors = sqlite3_column_blob(stmt, 3); + if (!validity || !cellRowids || !cellVectors || + sqlite3_column_bytes(stmt, 1) != VEC0_IVF_CELL_MAX_VECTORS / CHAR_BIT || + sqlite3_column_bytes(stmt, 2) != + VEC0_IVF_CELL_MAX_VECTORS * (int)sizeof(i64) || + sqlite3_column_bytes(stmt, 3) != + VEC0_IVF_CELL_MAX_VECTORS * vectorSize) { + vtab_set_error(&p->base, + "cell %lld of the IVF index of vector column %d is " + "corrupt", + sqlite3_column_int64(stmt, 0), i); + rc = SQLITE_ERROR; + break; + } + for (int s = 0; s < VEC0_IVF_CELL_MAX_VECTORS; s++) { + if (!(validity[s / CHAR_BIT] & (1 << (s % CHAR_BIT)))) { + continue; + } + struct IvfCandidate candidate; + memcpy(&candidate.id, cellRowids + (size_t)s * sizeof(i64), sizeof(i64)); + candidate.distance = + ivf_distance(col, qquery, cellVectors + (size_t)s * vectorSize); + rc = array_append(candidates, &candidate); + if (rc != SQLITE_OK) { + break; + } + } + if (rc != SQLITE_OK) { + break; + } + } + sqlite3_reset(stmt); + sqlite3_clear_bindings(stmt); + if (rc != SQLITE_OK || step == SQLITE_DONE) { + return rc; + } + return ivf_failed(p, i, "read the cells of", vec0_error_rc(step)); +} + +/** + * The KNN query of column i, indexed by ivf(). With rowid = or rowid IN, the + * candidates are the rows they name; otherwise they are the rows in the cells + * of the nprobe centroids nearest the query and in the cells of no centroid, + * and with a quantizer the k * oversample of them nearest by quantized + * distance. Returns the k nearest candidates by exact distance within the + * query's distance constraints, nearest first, as vec0Filter_knn_chunks_iter() + * does. + */ +static int ivf_knn(vec0_vtab *p, struct VectorColumnDefinition *col, int i, + struct Array *arrayRowidsIn, const char *idxStr, int argc, + sqlite3_value **argv, void *queryVector, i64 k, + i64 **out_topk_rowids, f32 **out_topk_distances, + i64 *out_used) { + struct Array candidates; + struct Array centroids; + void *qquery = NULL; + i64 *rowids = NULL; + f32 *distances = NULL; + int rc = array_init(&candidates, sizeof(struct IvfCandidate), 64); + if (rc != SQLITE_OK) { + return rc; + } + rc = array_init(¢roids, sizeof(struct IvfCandidate), 64); + if (rc != SQLITE_OK) { + array_cleanup(&candidates); + return rc; + } + + if (arrayRowidsIn) { + for (size_t j = 0; j < arrayRowidsIn->length; j++) { + i64 rowid = ((i64 *)arrayRowidsIn->z)[j]; + if (j > 0 && rowid == ((i64 *)arrayRowidsIn->z)[j - 1]) { + continue; + } + void *vector = NULL; + rc = ivf_vector_read(p, i, rowid, &vector, NULL); + if (rc == SQLITE_EMPTY) { + rc = SQLITE_OK; + continue; + } + if (rc != SQLITE_OK) { + goto cleanup; + } + struct IvfCandidate candidate = { + rowid, vec0_compute_distance(col, vector, queryVector)}; + sqlite3_free(vector); + rc = array_append(&candidates, &candidate); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + } else { + int vectorSize = ivf_vector_size(col); + qquery = sqlite3_malloc(vectorSize); + if (!qquery) { + rc = SQLITE_NOMEM; + goto cleanup; + } + ivf_quantize(col, queryVector, qquery); + + rc = ivf_scan_cells(p, i, VEC0_IVF_UNASSIGNED_CENTROID_ID, qquery, + &candidates); + int trained = 0; + if (rc == SQLITE_OK) { + rc = ivf_trained(p, i, &trained); + } + if (rc != SQLITE_OK) { + goto cleanup; + } + if (trained) { + rc = vec0_cached_stmt(p, &p->ivfStmts[i].centroids, + "SELECT centroid_id, centroid FROM \"%w\".\"%w\"", + p->shadowIvfCentroidsNames[i]); + if (rc != SQLITE_OK) { + rc = ivf_failed(p, i, "read the centroids of", rc); + goto cleanup; + } + sqlite3_stmt *stmt = p->ivfStmts[i].centroids; + int step; + while ((step = sqlite3_step(stmt)) == SQLITE_ROW) { + struct IvfCandidate centroid; + centroid.id = sqlite3_column_int64(stmt, 0); + const void *blob = sqlite3_column_blob(stmt, 1); + if (!blob || sqlite3_column_bytes(stmt, 1) != vectorSize) { + vtab_set_error(&p->base, + "centroid %lld of the IVF index of vector column %d " + "is corrupt", + centroid.id, i); + rc = SQLITE_ERROR; + break; + } + centroid.distance = ivf_distance(col, qquery, blob); + rc = array_append(¢roids, ¢roid); + if (rc != SQLITE_OK) { + break; + } + } + sqlite3_reset(stmt); + if (rc == SQLITE_OK && step != SQLITE_DONE) { + rc = ivf_failed(p, i, "read the centroids of", vec0_error_rc(step)); + } + if (rc != SQLITE_OK) { + goto cleanup; + } + qsort(centroids.z, centroids.length, sizeof(struct IvfCandidate), + ivf_candidate_cmp); + size_t nprobe = (size_t)col->ivf.nprobe; + for (size_t c = 0; c < centroids.length && c < nprobe; c++) { + i64 centroidId = ((struct IvfCandidate *)centroids.z)[c].id; + if (centroidId == VEC0_IVF_UNASSIGNED_CENTROID_ID) { + continue; + } + rc = ivf_scan_cells(p, i, centroidId, qquery, &candidates); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + } + + if (col->ivf.quantizer != VEC0_IVF_QUANTIZER_NONE) { + // rank the nearest by quantized distance by their exact distances + qsort(candidates.z, candidates.length, sizeof(struct IvfCandidate), + ivf_candidate_cmp); + i64 keep = k * col->ivf.oversample; + if ((i64)candidates.length > keep) { + candidates.length = (size_t)keep; + } + for (size_t j = 0; j < candidates.length; j++) { + struct IvfCandidate *candidate = + &((struct IvfCandidate *)candidates.z)[j]; + void *vector = NULL; + rc = ivf_vector_read(p, i, candidate->id, &vector, NULL); + if (rc == SQLITE_EMPTY) { + vtab_set_error(&p->base, + "row %lld of the IVF index of vector column %d has " + "no vector", + candidate->id, i); + rc = SQLITE_ERROR; + } + if (rc != SQLITE_OK) { + goto cleanup; + } + candidate->distance = vec0_compute_distance(col, vector, queryVector); + sqlite3_free(vector); + } + } + } + + qsort(candidates.z, candidates.length, sizeof(struct IvfCandidate), + ivf_candidate_cmp); + i64 used = (i64)candidates.length; + if (used > 0) { + rowids = sqlite3_malloc64(used * sizeof(i64)); + distances = sqlite3_malloc64(used * sizeof(f32)); + if (!rowids || !distances) { + rc = SQLITE_NOMEM; + goto cleanup; + } + for (i64 j = 0; j < used; j++) { + rowids[j] = ((struct IvfCandidate *)candidates.z)[j].id; + distances[j] = ((struct IvfCandidate *)candidates.z)[j].distance; + } + vec0_knn_keep_within_distance(idxStr, argc, argv, rowids, distances, &used); + } + if (used > k) { + used = k; + } + if (used == 0) { + sqlite3_free(rowids); + sqlite3_free(distances); + rowids = NULL; + distances = NULL; + } + *out_topk_rowids = rowids; + *out_topk_distances = distances; + *out_used = used; + rowids = NULL; + distances = NULL; + rc = SQLITE_OK; + +cleanup: + sqlite3_free(rowids); + sqlite3_free(distances); + sqlite3_free(qquery); + array_cleanup(&candidates); + array_cleanup(¢roids); + return rc; +} + +/** + * Reads the digits of zValue, from min to max, into *out. + * + * @return SQLITE_OK, or SQLITE_ERROR for any other text. + */ +static int ivf_command_int(const char *zValue, i64 min, i64 max, i64 *out) { + if (!(*zValue >= '0' && *zValue <= '9')) { + return SQLITE_ERROR; + } + char *zEnd; + errno = 0; + long long value = strtoll(zValue, &zEnd, 10); + if (*zEnd != '\0' || errno == ERANGE || value < min || value > max) { + return SQLITE_ERROR; + } + *out = value; + return SQLITE_OK; +} + +/** + * Reads the options of `compute-centroids:{...}`, a JSON object whose keys + * nlist, max_iterations and seed each take an integer. + */ +static int ivf_compute_options(vec0_vtab *p, const char *zJson, int *nlist, + int *maxIter, u32 *seed) { + sqlite3_stmt *stmt = NULL; + // the outer row makes a row of `{}` too, so its type is checked + int rc = sqlite3_prepare_v2(p->db, + "SELECT json_type(?1), key, type, value FROM " + "(SELECT 1) LEFT JOIN json_each(?1)", + -1, &stmt, NULL); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "compute-centroids options: %s", + sqlite3_errmsg(p->db)); + return vec0_error_rc(rc); + } + sqlite3_bind_text(stmt, 1, zJson, -1, SQLITE_STATIC); + static const struct { + const char *zKey; + i64 min; + i64 max; + } aOption[] = { + {"nlist", 1, VEC0_IVF_MAX_NLIST}, + {"max_iterations", 0, INT_MAX}, + {"seed", 0, UINT32_MAX}, + }; + int invalid = 0; + while (!invalid && (rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const char *zType = (const char *)sqlite3_column_text(stmt, 0); + if (!zType || strcmp(zType, "object") != 0) { + vtab_set_error(&p->base, + "compute-centroids options must be a JSON object"); + invalid = 1; + break; + } + if (sqlite3_column_type(stmt, 1) == SQLITE_NULL) { + continue; + } + const char *zKey = (const char *)sqlite3_column_text(stmt, 1); + const char *zValueType = (const char *)sqlite3_column_text(stmt, 2); + i64 value = sqlite3_column_int64(stmt, 3); + size_t o; + for (o = 0; o < countof(aOption); o++) { + if (zKey && strcmp(zKey, aOption[o].zKey) == 0) { + break; + } + } + if (o == countof(aOption)) { + vtab_set_error(&p->base, "unknown compute-centroids option: %s", + zKey ? zKey : ""); + invalid = 1; + } else if (!zValueType || strcmp(zValueType, "integer") != 0 || + value < aOption[o].min || value > aOption[o].max) { + vtab_set_error(&p->base, + "compute-centroids option %s must be an integer from " + "%lld to %lld", + aOption[o].zKey, aOption[o].min, aOption[o].max); + invalid = 1; + } else if (o == 0) { + *nlist = (int)value; + } else if (o == 1) { + *maxIter = (int)value; + } else { + *seed = (u32)value; + } + } + if (invalid) { + rc = SQLITE_ERROR; + } else if (rc == SQLITE_DONE) { + rc = SQLITE_OK; + } else { + vtab_set_error(&p->base, "compute-centroids options: %s", + sqlite3_errmsg(p->db)); + rc = vec0_error_rc(rc); + } + sqlite3_finalize(stmt); + return rc; +} + +/** + * The commands of columns indexed by ivf(), each applied to every such column + * of the table: `nprobe=N` sets the nprobe of this connection's queries, in + * place of the one its CREATE gave, until SQLite reconnects the table, as after + * VACUUM, ALTER TABLE ... RENAME, or another connection's schema change; + * `compute-centroids` clusters the rows with k-means into nlist centroids and + * moves each row to the cells of the nearest, and `compute-centroids:{...}` + * does so with the nlist, max_iterations and seed of a JSON object; + * `set-centroid:N` sets centroid N to the vector the INSERT gives the column, + * of each column given one; `assign-vectors` moves the rows no clustering has + * placed to the cells of their nearest centroids; and `clear-centroids` + * removes the centroids and moves every row to cells of no centroid. Returns + * SQLITE_EMPTY for any other command, and for these on a table with no such + * column, as upstream does. + */ +static int ivf_handle_command(vec0_vtab *p, const char *command, int argc, + sqlite3_value **argv) { + static const char zNprobe[] = "nprobe="; + static const char zCompute[] = "compute-centroids"; + static const char zSetCentroid[] = "set-centroid:"; + int rc; + int hasIvf = 0; + for (int i = 0; i < p->numVectorColumns; i++) { + hasIvf |= p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF; + } + if (!hasIvf) { + return SQLITE_EMPTY; + } + + if (strncmp(command, zNprobe, sizeof(zNprobe) - 1) == 0) { + i64 nprobe; + if (ivf_command_int(command + sizeof(zNprobe) - 1, 1, INT_MAX, &nprobe) != + SQLITE_OK) { + vtab_set_error(&p->base, "nprobe must be a whole number from 1 to %d", + INT_MAX); + return SQLITE_ERROR; + } + for (int i = 0; i < p->numVectorColumns; i++) { + if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF) { + p->vector_columns[i].ivf.nprobe = (int)nprobe; + } + } + return SQLITE_OK; + } + + size_t nCompute = sizeof(zCompute) - 1; + if (strncmp(command, zCompute, nCompute) == 0 && + (command[nCompute] == '\0' || command[nCompute] == ':')) { + int nlist = 0; + int maxIter = VEC0_IVF_KMEANS_MAX_ITER; + u32 seed = VEC0_IVF_KMEANS_DEFAULT_SEED; + if (command[nCompute] == ':') { + rc = ivf_compute_options(p, command + nCompute + 1, &nlist, &maxIter, + &seed); + if (rc != SQLITE_OK) { + return rc; + } + } + for (int i = 0; i < p->numVectorColumns; i++) { + if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + rc = ivf_compute_centroids(p, i, nlist, maxIter, seed); + if (rc != SQLITE_OK) { + return rc; + } + } + return SQLITE_OK; + } + + if (strncmp(command, zSetCentroid, sizeof(zSetCentroid) - 1) == 0) { + i64 centroidId; + if (ivf_command_int(command + sizeof(zSetCentroid) - 1, 0, INT_MAX, + ¢roidId) != SQLITE_OK) { + vtab_set_error(&p->base, "set-centroid needs a centroid id from 0 to %d", + INT_MAX); + return SQLITE_ERROR; + } + int set = 0; + for (int u = 0; u < vec0_num_defined_user_columns(p); u++) { + if (p->user_column_kinds[u] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) { + continue; + } + int i = p->user_column_idxs[u]; + struct VectorColumnDefinition *col = &p->vector_columns[i]; + int argvIdx = 2 + VEC0_COLUMN_USERN_START + u; + if (col->index_type != VEC0_INDEX_TYPE_IVF || argvIdx >= argc || + sqlite3_value_type(argv[argvIdx]) == SQLITE_NULL) { + continue; + } + void *vector; + size_t dimensions; + enum VectorElementType elementType; + vector_cleanup cleanup; + char *zError; + rc = vector_from_value(argv[argvIdx], &vector, &dimensions, &elementType, + &cleanup, &zError); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "set-centroid vector for \"%.*s\": %z", + col->name_length, col->name, zError); + return SQLITE_ERROR; + } + if (elementType != SQLITE_VEC_ELEMENT_TYPE_FLOAT32 || + dimensions != col->dimensions) { + cleanup(vector); + vtab_set_error(&p->base, + "set-centroid vector for \"%.*s\" must be a float32 " + "vector of %lld dimensions", + col->name_length, col->name, (i64)col->dimensions); + return SQLITE_ERROR; + } + rc = ivf_set_centroid(p, i, centroidId, vector); + cleanup(vector); + if (rc != SQLITE_OK) { + return rc; + } + set++; + } + if (!set) { + vtab_set_error(&p->base, + "set-centroid needs a vector for an IVF-indexed column"); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + + int (*apply)(vec0_vtab *, int) = NULL; + if (strcmp(command, "assign-vectors") == 0) { + apply = ivf_assign_vectors; + } else if (strcmp(command, "clear-centroids") == 0) { + apply = ivf_clear_centroids; + } else { + return SQLITE_EMPTY; + } + for (int i = 0; i < p->numVectorColumns; i++) { + if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + rc = apply(p, i); + if (rc != SQLITE_OK) { + return rc; + } + } + return SQLITE_OK; +} + +#endif + +int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, + const char *idxStr, int argc, sqlite3_value **argv) { + assert(argc == (int)((strlen(idxStr) - 1) / 4)); + int rc; + struct vec0_query_knn_data *knn_data; + + int vectorColumnIdx = idxNum; + struct VectorColumnDefinition *vector_column = + &p->vector_columns[vectorColumnIdx]; + + struct Array *arrayRowidsIn = NULL; + sqlite3_stmt *stmtChunks = NULL; + void *queryVector; + size_t dimensions; + enum VectorElementType elementType; + vector_cleanup queryVectorCleanup = vector_cleanup_noop; + char *pzError; + knn_data = sqlite3_malloc(sizeof(*knn_data)); + if (!knn_data) { + return SQLITE_NOMEM; + } + memset(knn_data, 0, sizeof(*knn_data)); + // array of `struct Vec0MetadataIn`, IF there are any `xxx in (...)` metadata + // constraints + struct Array *aMetadataIn = NULL; + + int query_idx = -1; + int k_idx = -1; + int rowid_in_idx = -1; + int rowid_eq_idx = -1; + int mmr_lambda_idx = -1; + for (int i = 0; i < argc; i++) { + if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_MATCH) { + query_idx = i; + } + if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_K) { + k_idx = i; + } + if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_ROWID_IN) { + rowid_in_idx = i; + } + if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_ROWID_EQ) { rowid_eq_idx = i; } if (idxStr[1 + (i * 4)] == VEC0_IDXSTR_KIND_KNN_MMR_LAMBDA) { @@ -12985,15 +15362,19 @@ int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, // a diskann() column keeps no chunks to scan if (vector_column->index_type != VEC0_INDEX_TYPE_DISKANN) #endif - { - rc = vec0_chunks_iter(p, idxStr, argc, argv, &stmtChunks); - if (rc != SQLITE_OK) { - // IMP: V06942_23781 - vtab_set_error(&p->base, "Error preparing stmtChunk: %s", - sqlite3_errmsg(p->db)); - goto cleanup; +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // an ivf() column's query reads its cells + if (vector_column->index_type != VEC0_INDEX_TYPE_IVF) +#endif + { + rc = vec0_chunks_iter(p, idxStr, argc, argv, &stmtChunks); + if (rc != SQLITE_OK) { + // IMP: V06942_23781 + vtab_set_error(&p->base, "Error preparing stmtChunk: %s", + sqlite3_errmsg(p->db)); + goto cleanup; + } } - } i64 *topk_rowids = NULL; f32 *topk_distances = NULL; @@ -13005,6 +15386,12 @@ int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, &k_used); } else #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + if (vector_column->index_type == VEC0_INDEX_TYPE_IVF) { + rc = ivf_knn(p, vector_column, vectorColumnIdx, arrayRowidsIn, idxStr, argc, + argv, queryVector, k, &topk_rowids, &topk_distances, &k_used); + } else +#endif #if SQLITE_VEC_ENABLE_RESCORE if (vector_column->index_type == VEC0_INDEX_TYPE_RESCORE) { rc = rescore_knn(p, stmtChunks, vector_column, vectorColumnIdx, @@ -14539,6 +16926,18 @@ int vec0Update_Insert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + for (int i = 0; i < p->numVectorColumns; i++) { + if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + rc = ivf_insert(p, i, rowid, vectorDatas[i]); + if (rc != SQLITE_OK) { + goto cleanup; + } + } +#endif + if (p->numAuxiliaryColumns > 0) { sqlite3_stmt *stmt; sqlite3_str *s = sqlite3_str_new(NULL); @@ -15065,6 +17464,19 @@ int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) { } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // 1c. take the row out of the cells of each column indexed by ivf() + for (int i = 0; i < p->numVectorColumns; i++) { + if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) { + continue; + } + rc = ivf_delete(p, i, rowid); + if (rc != SQLITE_OK) { + return rc; + } + } +#endif + // a table whose every vector column is indexed by diskann() keeps no chunks int hasChunks = !vec0_all_columns_diskann(p); int chunkEmpty = 0; @@ -15397,6 +17809,18 @@ int vec0Update_Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv) { return SQLITE_ERROR; } #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + // the row's cell belongs to the centroid nearest the vector it has + // https://github.com/asg017/sqlite-vec/commit/9df59b4 + if (p->vector_columns[vector_idx].index_type == VEC0_INDEX_TYPE_IVF) { + vtab_set_error(&p->base, + "UPDATE on vector column \"%.*s\" is not supported for " + "IVF indexes.", + p->vector_columns[vector_idx].name_length, + p->vector_columns[vector_idx].name); + return SQLITE_ERROR; + } +#endif rc = vec0Update_UpdateVectorColumn(p, chunk_id, chunk_offset, vector_idx, valueVector, rowid); @@ -15550,6 +17974,17 @@ done: { } } +// flat() and rescore() keep a column's vectors in the chunks, at its row's +// slot, which the optimize moves; diskann() and ivf() keep them by rowid +static int vec0_index_in_chunks(enum Vec0IndexType index_type) { +#if SQLITE_VEC_ENABLE_RESCORE + if (index_type == VEC0_INDEX_TYPE_RESCORE) { + return 1; + } +#endif + return index_type == VEC0_INDEX_TYPE_FLAT; +} + int vec0Update_SpecialInsert_Optimize(vec0_vtab *p) { sqlite3_stmt *stmt = NULL, *partition_key_stmt = NULL; int rc; @@ -15673,12 +18108,9 @@ int vec0Update_SpecialInsert_Optimize(vec0_vtab *p) { } // get the vector data from all vector columns of a row for (int i = 0; i < p->numVectorColumns; i++) { -#if SQLITE_VEC_ENABLE_DISKANN - // a diskann() column keeps its vectors by rowid, which stays - if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_DISKANN) { + if (!vec0_index_in_chunks(p->vector_columns[i].index_type)) { continue; } -#endif rc = vec0_get_vector_data(p, rowid, i, &vectorDatas[i], NULL); if (rc != SQLITE_OK) { goto cleanup; @@ -15779,11 +18211,9 @@ int vec0Update_SpecialInsert_Optimize(vec0_vtab *p) { // 4) clean up old vector chunks for (int i = 0; i < p->numVectorColumns; i++) { -#if SQLITE_VEC_ENABLE_DISKANN - if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_DISKANN) { + if (!vec0_index_in_chunks(p->vector_columns[i].index_type)) { continue; } -#endif #if SQLITE_VEC_ENABLE_RESCORE if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_RESCORE) { zSql = sqlite3_mprintf("DELETE FROM \"%w\".\"%w\" WHERE rowid <= ?", @@ -15846,7 +18276,8 @@ int vec0Update_SpecialInsert_Optimize(vec0_vtab *p) { return rc; } -int vec0Update_SpecialInsert(sqlite3_vtab *pVTab, sqlite3_value *pVal) { +int vec0Update_SpecialInsert(sqlite3_vtab *pVTab, sqlite3_value *pVal, int argc, + sqlite3_value **argv) { vec0_vtab *p = (vec0_vtab *)pVTab; const char *cmd = (const char *)sqlite3_value_text(pVal); @@ -15869,6 +18300,15 @@ int vec0Update_SpecialInsert(sqlite3_vtab *pVTab, sqlite3_value *pVal) { if (diskannRc != SQLITE_EMPTY) { return diskannRc; } +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + int ivfRc = ivf_handle_command(p, cmd, argc, argv); + if (ivfRc != SQLITE_EMPTY) { + return ivfRc; + } +#else + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); #endif vtab_set_error(pVTab, "unknown vec0 command: '%s'", cmd); return SQLITE_ERROR; @@ -15885,7 +18325,8 @@ static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, return vec0_interrupt_result( pVTab, vec0Update_SpecialInsert( - pVTab, argv[2 + vec0_column_table_name_idx((vec0_vtab *)pVTab)])); + pVTab, argv[2 + vec0_column_table_name_idx((vec0_vtab *)pVTab)], + argc, argv)); } // DELETE operation if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) { @@ -15964,6 +18405,9 @@ static int vec0ShadowName(const char *zName) { #endif #if SQLITE_VEC_ENABLE_DISKANN "vectors", "diskann_nodes", "diskann_buffer", +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + "ivf_centroids", "ivf_cells", "ivf_rowid_map", "ivf_vectors", #endif }; for (size_t i = 0; i < countof(azPerColumn); i++) { @@ -16057,6 +18501,15 @@ static int vec0Rename(sqlite3_vtab *pVTab, const char *zName) { {p->shadowDiskannVectorsNames[i], "vectors"}, {p->shadowDiskannNodesNames[i], "diskann_nodes"}, {p->shadowDiskannBufferNames[i], "diskann_buffer"}, +#endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + {p->shadowIvfCentroidsNames[i], "ivf_centroids"}, + {p->shadowIvfCellsNames[i], "ivf_cells"}, + {p->shadowIvfRowidMapNames[i], "ivf_rowid_map"}, + {p->vector_columns[i].ivf.quantizer != VEC0_IVF_QUANTIZER_NONE + ? p->shadowIvfVectorsNames[i] + : NULL, + "ivf_vectors"}, #endif }; for (size_t j = 0; j < countof(aShadow); j++) { @@ -16861,6 +19314,12 @@ static sqlite3_module vec_static_blob_entriesModule = { #define SQLITE_VEC_DEBUG_BUILD_RESCORE "" #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +#define SQLITE_VEC_DEBUG_BUILD_IVF "ivf" +#else +#define SQLITE_VEC_DEBUG_BUILD_IVF "" +#endif + #if SQLITE_VEC_ENABLE_DISKANN #define SQLITE_VEC_DEBUG_BUILD_DISKANN "diskann" #else @@ -16868,9 +19327,9 @@ static sqlite3_module vec_static_blob_entriesModule = { #endif #define SQLITE_VEC_DEBUG_BUILD \ - SQLITE_VEC_DEBUG_BUILD_AVX " " SQLITE_VEC_DEBUG_BUILD_NEON \ - " " SQLITE_VEC_DEBUG_BUILD_RESCORE \ - " " SQLITE_VEC_DEBUG_BUILD_DISKANN + SQLITE_VEC_DEBUG_BUILD_AVX \ + " " SQLITE_VEC_DEBUG_BUILD_NEON " " SQLITE_VEC_DEBUG_BUILD_RESCORE \ + " " SQLITE_VEC_DEBUG_BUILD_IVF " " SQLITE_VEC_DEBUG_BUILD_DISKANN #define SQLITE_VEC_DEBUG_STRING \ "Version: " SQLITE_VEC_VERSION "\n" \ diff --git a/tests/conftest.py b/tests/conftest.py index db1c5656..c5eae2cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,26 @@ def get_extension_path(): return "dist/vec0" +def _has_build_flag(flag): + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + db.load_extension(get_extension_path()) + debug = db.execute("SELECT vec_debug()").fetchone()[0] + db.close() + return flag in debug.split("Build flags:")[-1].split() + + +def pytest_collection_modifyitems(config, items): + if _has_build_flag("ivf"): + return + skip_ivf = pytest.mark.skip( + reason="IVF not enabled (compile with -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1)" + ) + for item in items: + if item.fspath.basename.startswith("test-ivf"): + item.add_marker(skip_ivf) + + @pytest.fixture() def db(): db = sqlite3.connect(":memory:") diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 819463df..2045b04c 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -124,6 +124,33 @@ $(TARGET_DIR)/diskann_prune_direct: diskann-prune-direct.c $(FUZZ_SRCS) | $(TARG $(TARGET_DIR)/diskann_command_inject: diskann-command-inject.c $(FUZZ_SRCS) | $(TARGET_DIR) $(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@ +# the IVF index is compiled only on request +IVF_CFLAGS = -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1 + +$(TARGET_DIR)/ivf_create: ivf-create.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_operations: ivf-operations.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_quantize: ivf-quantize.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_kmeans: ivf-kmeans.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_shadow_corrupt: ivf-shadow-corrupt.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_knn_deep: ivf-knn-deep.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_cell_overflow: ivf-cell-overflow.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + +$(TARGET_DIR)/ivf_rescore: ivf-rescore.c $(FUZZ_SRCS) | $(TARGET_DIR) + $(FUZZ_CC) $(FUZZ_CFLAGS) $(IVF_CFLAGS) $(FUZZ_SRCS) $< -o $@ + FUZZ_TARGETS = vec0_create exec json numpy \ shadow_corrupt vec0_operations scalar_functions \ vec0_create_full metadata_columns vec_each vec_mismatch \ @@ -135,7 +162,9 @@ FUZZ_TARGETS = vec0_create exec json numpy \ diskann_deep_search diskann_blob_truncate \ diskann_delete_stress diskann_buffer_flush \ diskann_int8_quant diskann_prune_direct \ - diskann_command_inject + diskann_command_inject \ + ivf_create ivf_operations ivf_quantize ivf_kmeans \ + ivf_shadow_corrupt ivf_knn_deep ivf_cell_overflow ivf_rescore all: $(addprefix $(TARGET_DIR)/,$(FUZZ_TARGETS)) diff --git a/tests/fuzz/ivf-cell-overflow.c b/tests/fuzz/ivf-cell-overflow.c new file mode 100644 index 00000000..65ae6b2a --- /dev/null +++ b/tests/fuzz/ivf-cell-overflow.c @@ -0,0 +1,192 @@ +/** + * Fuzz target: IVF cell overflow and boundary conditions. + * + * Pushes cells past VEC0_IVF_CELL_MAX_VECTORS (64) to trigger cell + * splitting, then exercises blob I/O at slot boundaries. + * + * Targets: + * - Cell splitting when n_vectors reaches cap (64) + * - Blob offset arithmetic: slot * vecSize, slot / 8, slot % 8 + * - Validity bitmap at byte boundaries (slot 7->8, 15->16, etc.) + * - Insert into full cell -> create new cell path + * - Delete from various slot positions (first, last, middle) + * - Multiple cells per centroid + * - assign-vectors command with multi-cell centroids + */ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 8) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Use small dimensions for speed but enough vectors to overflow cells + int dim = (data[0] % 8) + 2; // 2..9 + int nlist = (data[1] % 4) + 1; // 1..4 + // We need >64 vectors to overflow a cell + int num_vecs = (data[2] % 64) + 65; // 65..128 + int delete_pattern = data[3]; // Controls which vectors to delete + + const uint8_t *payload = data + 4; + size_t payload_size = size - 4; + + char sql[256]; + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d))", + dim, nlist, nlist); + + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + // Insert enough vectors to overflow at least one cell + sqlite3_stmt *stmtInsert = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + if (!stmtInsert) { sqlite3_close(db); return 0; } + + size_t offset = 0; + for (int i = 0; i < num_vecs; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + for (int d = 0; d < dim; d++) { + if (offset < payload_size) { + vec[d] = ((float)(int8_t)payload[offset++]) / 50.0f; + } else { + // Cluster vectors near specific centroids to ensure some cells overflow + int cluster = i % nlist; + vec[d] = (float)cluster + (float)(i % 10) * 0.01f + d * 0.001f; + } + } + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, (int64_t)(i + 1)); + sqlite3_bind_blob(stmtInsert, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + sqlite3_free(vec); + } + sqlite3_finalize(stmtInsert); + + // Train to assign vectors to centroids (triggers cell building) + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Delete vectors at boundary positions based on fuzz data + // This tests validity bitmap manipulation at different slot positions + for (int i = 0; i < num_vecs; i++) { + int byte_idx = i / 8; + if (byte_idx < (int)payload_size && (payload[byte_idx] & (1 << (i % 8)))) { + // Use delete_pattern to thin deletions + if ((delete_pattern + i) % 3 == 0) { + char delsql[64]; + snprintf(delsql, sizeof(delsql), "DELETE FROM v WHERE rowid = %d", i + 1); + sqlite3_exec(db, delsql, NULL, NULL, NULL); + } + } + } + + // Insert more vectors after deletions (into cells with holes) + { + sqlite3_stmt *si = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &si, NULL); + if (si) { + for (int i = 0; i < 10; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + for (int d = 0; d < dim; d++) + vec[d] = (float)(i + 200) * 0.01f; + sqlite3_reset(si); + sqlite3_bind_int64(si, 1, (int64_t)(num_vecs + i + 1)); + sqlite3_bind_blob(si, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(si); + sqlite3_free(vec); + } + sqlite3_finalize(si); + } + } + + // KNN query that must scan multiple cells per centroid + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = 0.0f; + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 20"); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + // Test assign-vectors with multi-cell state + // First clear centroids + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('clear-centroids')", + NULL, NULL, NULL); + + // Set centroids manually, then assign + for (int c = 0; c < nlist; c++) { + float *cvec = sqlite3_malloc(dim * sizeof(float)); + if (!cvec) break; + for (int d = 0; d < dim; d++) cvec[d] = (float)c + d * 0.1f; + + char cmd[128]; + snprintf(cmd, sizeof(cmd), + "INSERT INTO v(v, emb) VALUES ('set-centroid:%d', ?)", c); + sqlite3_stmt *sc = NULL; + sqlite3_prepare_v2(db, cmd, -1, &sc, NULL); + if (sc) { + sqlite3_bind_blob(sc, 1, cvec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(sc); + sqlite3_finalize(sc); + } + sqlite3_free(cvec); + } + + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('assign-vectors')", + NULL, NULL, NULL); + + // Final query after assign-vectors + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = 1.0f; + sqlite3_stmt *sk = NULL; + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 5", + -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + // Full scan + sqlite3_exec(db, "SELECT * FROM v", NULL, NULL, NULL); + + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-create.c b/tests/fuzz/ivf-create.c new file mode 100644 index 00000000..222b67b2 --- /dev/null +++ b/tests/fuzz/ivf-create.c @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + int rc; + sqlite3 *db; + sqlite3_stmt *stmt; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + sqlite3_str *s = sqlite3_str_new(NULL); + assert(s); + sqlite3_str_appendall(s, "CREATE VIRTUAL TABLE v USING vec0(emb float[4] indexed by ivf("); + sqlite3_str_appendf(s, "%.*s", (int)size, data); + sqlite3_str_appendall(s, "))"); + const char *zSql = sqlite3_str_finish(s); + assert(zSql); + + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, NULL); + sqlite3_free((void *)zSql); + if (rc == SQLITE_OK) { + sqlite3_step(stmt); + } + sqlite3_finalize(stmt); + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-create.dict b/tests/fuzz/ivf-create.dict new file mode 100644 index 00000000..9a014e7d --- /dev/null +++ b/tests/fuzz/ivf-create.dict @@ -0,0 +1,16 @@ +"nlist" +"nprobe" +"quantizer" +"oversample" +"binary" +"int8" +"none" +"=" +"," +"(" +")" +"0" +"1" +"128" +"65536" +"65537" diff --git a/tests/fuzz/ivf-kmeans.c b/tests/fuzz/ivf-kmeans.c new file mode 100644 index 00000000..1d37184e --- /dev/null +++ b/tests/fuzz/ivf-kmeans.c @@ -0,0 +1,180 @@ +/** + * Fuzz target: IVF k-means clustering. + * + * Builds a table, inserts fuzz-controlled vectors, then runs + * compute-centroids with fuzz-controlled parameters (nlist, max_iter, seed). + * Targets: + * - kmeans with N < k (clamping), N == 1, k == 1 + * - kmeans with duplicate/identical vectors (all distances zero) + * - kmeans with NaN/Inf vectors + * - Empty cluster reassignment path (farthest-point heuristic) + * - Large nlist relative to N + * - The compute-centroids:{json} command parsing + * - clear-centroids followed by compute-centroids (round-trip) + */ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 10) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Parse fuzz header + // Byte 0-1: dimension (1..128) + // Byte 2: nlist for CREATE (1..64) + // Byte 3: nlist override for compute-centroids (0 = use default) + // Byte 4: max_iter (1..50) + // Byte 5-8: seed + // Byte 9: num_vectors (1..64) + // Remaining: vector float data + + int dim = (data[0] | (data[1] << 8)) % 128 + 1; + int nlist_create = (data[2] % 64) + 1; + int nlist_override = data[3] % 65; // 0 means use table default + int max_iter = (data[4] % 50) + 1; + uint32_t seed = (uint32_t)data[5] | ((uint32_t)data[6] << 8) | + ((uint32_t)data[7] << 16) | ((uint32_t)data[8] << 24); + int num_vecs = (data[9] % 64) + 1; + + const uint8_t *payload = data + 10; + size_t payload_size = size - 10; + + char sql[256]; + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d))", + dim, nlist_create, nlist_create); + + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + // Insert vectors + sqlite3_stmt *stmtInsert = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + if (!stmtInsert) { sqlite3_close(db); return 0; } + + size_t offset = 0; + for (int i = 0; i < num_vecs; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + + for (int d = 0; d < dim; d++) { + if (offset + 4 <= payload_size) { + memcpy(&vec[d], payload + offset, sizeof(float)); + offset += 4; + } else if (offset < payload_size) { + // Scale to interesting range including values > 1, < -1 + vec[d] = ((float)(int8_t)payload[offset++]) / 5.0f; + } else { + // Reuse earlier bytes to fill remaining dimensions + vec[d] = (float)(i * dim + d) * 0.01f; + } + } + + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, (int64_t)(i + 1)); + sqlite3_bind_blob(stmtInsert, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + sqlite3_free(vec); + } + sqlite3_finalize(stmtInsert); + + // Exercise compute-centroids with JSON options + { + char cmd[256]; + snprintf(cmd, sizeof(cmd), + "INSERT INTO v(rowid) VALUES " + "('compute-centroids:{\"nlist\":%d,\"max_iterations\":%d,\"seed\":%u}')", + nlist_override, max_iter, seed); + sqlite3_exec(db, cmd, NULL, NULL, NULL); + } + + // KNN query after training + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) { + qvec[d] = (d < 3) ? 1.0f : 0.0f; + } + sqlite3_stmt *stmtKnn = NULL; + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 5", + -1, &stmtKnn, NULL); + if (stmtKnn) { + sqlite3_bind_blob(stmtKnn, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(stmtKnn) == SQLITE_ROW) {} + sqlite3_finalize(stmtKnn); + } + sqlite3_free(qvec); + } + } + + // Clear centroids and re-compute to test round-trip + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('clear-centroids')", + NULL, NULL, NULL); + + // Insert a few more vectors in untrained state + { + sqlite3_stmt *si = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &si, NULL); + if (si) { + for (int i = 0; i < 3; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + for (int d = 0; d < dim; d++) vec[d] = (float)(i + 100) * 0.1f; + sqlite3_reset(si); + sqlite3_bind_int64(si, 1, (int64_t)(num_vecs + i + 1)); + sqlite3_bind_blob(si, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(si); + sqlite3_free(vec); + } + sqlite3_finalize(si); + } + } + + // Re-train + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Delete some rows after training, then query + sqlite3_exec(db, "DELETE FROM v WHERE rowid = 1", NULL, NULL, NULL); + sqlite3_exec(db, "DELETE FROM v WHERE rowid = 2", NULL, NULL, NULL); + + // Query after deletes + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = 0.5f; + sqlite3_stmt *stmtKnn = NULL; + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 10", + -1, &stmtKnn, NULL); + if (stmtKnn) { + sqlite3_bind_blob(stmtKnn, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(stmtKnn) == SQLITE_ROW) {} + sqlite3_finalize(stmtKnn); + } + sqlite3_free(qvec); + } + } + + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-knn-deep.c b/tests/fuzz/ivf-knn-deep.c new file mode 100644 index 00000000..f5adb1e7 --- /dev/null +++ b/tests/fuzz/ivf-knn-deep.c @@ -0,0 +1,199 @@ +/** + * Fuzz target: IVF KNN search deep paths. + * + * Exercises the full KNN pipeline with fuzz-controlled: + * - nprobe values (including > nlist, =1, =nlist) + * - Query vectors (including adversarial floats) + * - Mix of trained/untrained state + * - Oversample + rescore path (quantizer=int8 with oversample>1) + * - Multiple interleaved KNN queries + * - Candidate array realloc path (many vectors in probed cells) + * + * Targets: + * - ivf_scan_cells_from_stmt: candidate realloc, distance computation + * - ivf_query_knn: centroid sorting, nprobe selection + * - Oversample rescore: re-ranking with full-precision vectors + * - qsort with NaN distances + */ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +static uint16_t read_u16(const uint8_t *p) { + return (uint16_t)(p[0] | (p[1] << 8)); +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 16) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Header + int dim = (data[0] % 32) + 2; // 2..33 + int nlist = (data[1] % 16) + 1; // 1..16 + int nprobe_initial = (data[2] % 20) + 1; // 1..20 (can be > nlist) + int quantizer_type = data[3] % 3; // 0=none, 1=int8, 2=binary + int oversample = (data[4] % 4) + 1; // 1..4 + int num_vecs = (data[5] % 80) + 4; // 4..83 + int num_queries = (data[6] % 8) + 1; // 1..8 + int k_limit = (data[7] % 20) + 1; // 1..20 + + const uint8_t *payload = data + 8; + size_t payload_size = size - 8; + + // For binary quantizer, dimension must be multiple of 8 + if (quantizer_type == 2) { + dim = ((dim + 7) / 8) * 8; + if (dim == 0) dim = 8; + } + + const char *qname; + switch (quantizer_type) { + case 1: qname = "int8"; break; + case 2: qname = "binary"; break; + default: qname = "none"; break; + } + + // Oversample only valid with quantization + if (quantizer_type == 0) oversample = 1; + + // Cap nprobe to nlist for CREATE (parser rejects nprobe > nlist) + int nprobe_create = nprobe_initial <= nlist ? nprobe_initial : nlist; + + char sql[512]; + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d, quantizer=%s%s))", + dim, nlist, nprobe_create, qname, + oversample > 1 ? ", oversample=2" : ""); + + // If that fails (e.g. oversample with none), try without oversample + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d, quantizer=%s))", + dim, nlist, nprobe_create, qname); + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + } + + // Insert vectors + sqlite3_stmt *stmtInsert = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + if (!stmtInsert) { sqlite3_close(db); return 0; } + + size_t offset = 0; + for (int i = 0; i < num_vecs; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + for (int d = 0; d < dim; d++) { + if (offset < payload_size) { + vec[d] = ((float)(int8_t)payload[offset++]) / 20.0f; + } else { + vec[d] = (float)((i * dim + d) % 256 - 128) / 128.0f; + } + } + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, (int64_t)(i + 1)); + sqlite3_bind_blob(stmtInsert, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + sqlite3_free(vec); + } + sqlite3_finalize(stmtInsert); + + // Query BEFORE training (flat scan path) + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = 0.5f; + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + // Train + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Change nprobe at runtime (can exceed nlist -- tests clamping in query) + { + char cmd[64]; + snprintf(cmd, sizeof(cmd), + "INSERT INTO v(v) VALUES ('nprobe=%d')", nprobe_initial); + sqlite3_exec(db, cmd, NULL, NULL, NULL); + } + + // Multiple KNN queries with different fuzz-derived query vectors + for (int q = 0; q < num_queries; q++) { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (!qvec) break; + for (int d = 0; d < dim; d++) { + if (offset < payload_size) { + qvec[d] = ((float)(int8_t)payload[offset++]) / 10.0f; + } else { + qvec[d] = (q == 0) ? 1.0f : 0.0f; + } + } + + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + + // Delete half the vectors then query again + for (int i = 1; i <= num_vecs / 2; i++) { + char delsql[64]; + snprintf(delsql, sizeof(delsql), "DELETE FROM v WHERE rowid = %d", i); + sqlite3_exec(db, delsql, NULL, NULL, NULL); + } + + // Query after mass deletion + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = -0.5f; + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-operations.c b/tests/fuzz/ivf-operations.c new file mode 100644 index 00000000..c8d0c012 --- /dev/null +++ b/tests/fuzz/ivf-operations.c @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 6) return 0; + + int rc; + sqlite3 *db; + sqlite3_stmt *stmtInsert = NULL; + sqlite3_stmt *stmtDelete = NULL; + sqlite3_stmt *stmtKnn = NULL; + sqlite3_stmt *stmtScan = NULL; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + rc = sqlite3_exec(db, + "CREATE VIRTUAL TABLE v USING vec0(emb float[4] indexed by ivf(nlist=4, nprobe=4))", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + sqlite3_prepare_v2(db, + "DELETE FROM v WHERE rowid = ?", -1, &stmtDelete, NULL); + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 3", + -1, &stmtKnn, NULL); + sqlite3_prepare_v2(db, + "SELECT rowid FROM v", -1, &stmtScan, NULL); + + if (!stmtInsert || !stmtDelete || !stmtKnn || !stmtScan) goto cleanup; + + size_t i = 0; + while (i + 2 <= size) { + uint8_t op = data[i++] % 7; + uint8_t rowid_byte = data[i++]; + int64_t rowid = (int64_t)(rowid_byte % 32) + 1; + + switch (op) { + case 0: { + // INSERT: consume 16 bytes for 4 floats, or use what's left + float vec[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + for (int j = 0; j < 4 && i < size; j++, i++) { + vec[j] = (float)((int8_t)data[i]) / 10.0f; + } + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, rowid); + sqlite3_bind_blob(stmtInsert, 2, vec, sizeof(vec), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + break; + } + case 1: { + // DELETE + sqlite3_reset(stmtDelete); + sqlite3_bind_int64(stmtDelete, 1, rowid); + sqlite3_step(stmtDelete); + break; + } + case 2: { + // KNN query with a fixed query vector + float qvec[4] = {1.0f, 0.0f, 0.0f, 0.0f}; + sqlite3_reset(stmtKnn); + sqlite3_bind_blob(stmtKnn, 1, qvec, sizeof(qvec), SQLITE_STATIC); + while (sqlite3_step(stmtKnn) == SQLITE_ROW) {} + break; + } + case 3: { + // Full scan + sqlite3_reset(stmtScan); + while (sqlite3_step(stmtScan) == SQLITE_ROW) {} + break; + } + case 4: { + // compute-centroids command + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + break; + } + case 5: { + // clear-centroids command + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('clear-centroids')", + NULL, NULL, NULL); + break; + } + case 6: { + // nprobe=N command + if (i < size) { + uint8_t n = data[i++]; + int nprobe = (n % 4) + 1; + char buf[64]; + snprintf(buf, sizeof(buf), + "INSERT INTO v(v) VALUES ('nprobe=%d')", nprobe); + sqlite3_exec(db, buf, NULL, NULL, NULL); + } + break; + } + } + } + + // Final operations — must not crash regardless of prior state + sqlite3_exec(db, "SELECT * FROM v", NULL, NULL, NULL); + +cleanup: + sqlite3_finalize(stmtInsert); + sqlite3_finalize(stmtDelete); + sqlite3_finalize(stmtKnn); + sqlite3_finalize(stmtScan); + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-quantize.c b/tests/fuzz/ivf-quantize.c new file mode 100644 index 00000000..bc8800bf --- /dev/null +++ b/tests/fuzz/ivf-quantize.c @@ -0,0 +1,129 @@ +/** + * Fuzz target: IVF quantization functions. + * + * Directly exercises ivf_quantize_int8 and ivf_quantize_binary with + * fuzz-controlled dimensions and float data. Targets: + * - ivf_quantize_int8: clamping, int8 overflow boundary + * - ivf_quantize_binary: D not divisible by 8, memset(D/8) undercount + * - Round-trip through CREATE TABLE + INSERT with quantized IVF + */ +#include +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 8) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Byte 0: quantizer type (0=int8, 1=binary) + // Byte 1: dimension (1..64, but we test edge cases) + // Byte 2: nlist (1..8) + // Byte 3: num_vectors to insert (1..32) + // Remaining: float data + int qtype = data[0] % 2; + int dim = (data[1] % 64) + 1; + int nlist = (data[2] % 8) + 1; + int num_vecs = (data[3] % 32) + 1; + const uint8_t *payload = data + 4; + size_t payload_size = size - 4; + + // For binary quantizer, D must be multiple of 8 to avoid the D/8 bug + // in production. But we explicitly want to test non-multiples too to + // find the bug. Use dim as-is. + const char *quantizer = qtype ? "binary" : "int8"; + + // Binary quantizer needs D multiple of 8 in current code, but let's + // test both valid and invalid dimensions to see what happens. + // For binary with non-multiple-of-8, the code does memset(dst, 0, D/8) + // which underallocates when D%8 != 0. + char sql[256]; + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d, quantizer=%s))", + dim, nlist, nlist, quantizer); + + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + // Insert vectors with fuzz-controlled float values + sqlite3_stmt *stmtInsert = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + if (!stmtInsert) { sqlite3_close(db); return 0; } + + size_t offset = 0; + for (int i = 0; i < num_vecs && offset < payload_size; i++) { + // Build float vector from fuzz data + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + + for (int d = 0; d < dim; d++) { + if (offset + 4 <= payload_size) { + // Use raw bytes as float -- can produce NaN, Inf, denormals + memcpy(&vec[d], payload + offset, sizeof(float)); + offset += 4; + } else if (offset < payload_size) { + // Partial: use byte as scaled value + vec[d] = ((float)(int8_t)payload[offset++]) / 50.0f; + } else { + vec[d] = 0.0f; + } + } + + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, (int64_t)(i + 1)); + sqlite3_bind_blob(stmtInsert, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + sqlite3_free(vec); + } + sqlite3_finalize(stmtInsert); + + // Trigger compute-centroids to exercise kmeans + quantization together + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // KNN query with fuzz-derived query vector + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) { + if (offset < payload_size) { + qvec[d] = ((float)(int8_t)payload[offset++]) / 10.0f; + } else { + qvec[d] = 1.0f; + } + } + + sqlite3_stmt *stmtKnn = NULL; + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 5", + -1, &stmtKnn, NULL); + if (stmtKnn) { + sqlite3_bind_blob(stmtKnn, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(stmtKnn) == SQLITE_ROW) {} + sqlite3_finalize(stmtKnn); + } + sqlite3_free(qvec); + } + } + + // Full scan + sqlite3_exec(db, "SELECT * FROM v", NULL, NULL, NULL); + + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-rescore.c b/tests/fuzz/ivf-rescore.c new file mode 100644 index 00000000..3cddf88d --- /dev/null +++ b/tests/fuzz/ivf-rescore.c @@ -0,0 +1,182 @@ +/** + * Fuzz target: IVF oversample + rescore path. + * + * Specifically targets the code path where quantizer != none AND + * oversample > 1, which triggers: + * 1. Quantized KNN scan to collect oversample*k candidates + * 2. Full-precision vector lookup from _ivf_vectors table + * 3. Re-scoring with float32 distances + * 4. Re-sort and truncation + * + * This path has the most complex memory management in the KNN query: + * - Two separate distance computations (quantized + float) + * - Cross-table lookups (cells + vectors KV store) + * - Candidate array resizing + * - qsort over partially re-scored arrays + * + * Also tests the int8 + binary quantization round-trip fidelity + * under adversarial float inputs. + */ +#include +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 12) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Header + int quantizer_type = (data[0] % 2) + 1; // 1=int8, 2=binary (never none) + int dim = (data[1] % 32) + 8; // 8..39 + int nlist = (data[2] % 8) + 1; // 1..8 + int oversample = (data[3] % 4) + 2; // 2..5 (always > 1) + int num_vecs = (data[4] % 60) + 8; // 8..67 + int k_limit = (data[5] % 15) + 1; // 1..15 + + const uint8_t *payload = data + 6; + size_t payload_size = size - 6; + + // Binary quantizer needs D multiple of 8 + if (quantizer_type == 2) { + dim = ((dim + 7) / 8) * 8; + } + + const char *qname = (quantizer_type == 1) ? "int8" : "binary"; + + char sql[512]; + snprintf(sql, sizeof(sql), + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[%d] indexed by ivf(nlist=%d, nprobe=%d, quantizer=%s, oversample=%d))", + dim, nlist, nlist, qname, oversample); + + rc = sqlite3_exec(db, sql, NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + // Insert vectors with diverse values + sqlite3_stmt *stmtInsert = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &stmtInsert, NULL); + if (!stmtInsert) { sqlite3_close(db); return 0; } + + size_t offset = 0; + for (int i = 0; i < num_vecs; i++) { + float *vec = sqlite3_malloc(dim * sizeof(float)); + if (!vec) break; + for (int d = 0; d < dim; d++) { + if (offset + 4 <= payload_size) { + // Use raw bytes as float for adversarial values + memcpy(&vec[d], payload + offset, sizeof(float)); + offset += 4; + // Sanitize: replace NaN/Inf with bounded values to avoid + // poisoning the entire computation. We want edge values, + // not complete nonsense. + if (isnan(vec[d]) || isinf(vec[d])) { + vec[d] = (vec[d] > 0) ? 1e6f : -1e6f; + if (isnan(vec[d])) vec[d] = 0.0f; + } + } else if (offset < payload_size) { + vec[d] = ((float)(int8_t)payload[offset++]) / 30.0f; + } else { + vec[d] = (float)(i * dim + d) * 0.001f; + } + } + sqlite3_reset(stmtInsert); + sqlite3_bind_int64(stmtInsert, 1, (int64_t)(i + 1)); + sqlite3_bind_blob(stmtInsert, 2, vec, dim * sizeof(float), SQLITE_TRANSIENT); + sqlite3_step(stmtInsert); + sqlite3_free(vec); + } + sqlite3_finalize(stmtInsert); + + // Train + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Multiple KNN queries to exercise rescore path + for (int q = 0; q < 4; q++) { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (!qvec) break; + for (int d = 0; d < dim; d++) { + if (offset < payload_size) { + qvec[d] = ((float)(int8_t)payload[offset++]) / 10.0f; + } else { + qvec[d] = (q == 0) ? 1.0f : (q == 1) ? -1.0f : 0.0f; + } + } + + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + + // Delete some vectors, then query again (rescore with missing _ivf_vectors rows) + for (int i = 1; i <= num_vecs / 3; i++) { + char delsql[64]; + snprintf(delsql, sizeof(delsql), "DELETE FROM v WHERE rowid = %d", i); + sqlite3_exec(db, delsql, NULL, NULL, NULL); + } + + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = 0.5f; + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + // Retrain after deletions + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Query after retrain + { + float *qvec = sqlite3_malloc(dim * sizeof(float)); + if (qvec) { + for (int d = 0; d < dim; d++) qvec[d] = -0.3f; + sqlite3_stmt *sk = NULL; + snprintf(sql, sizeof(sql), + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT %d", k_limit); + sqlite3_prepare_v2(db, sql, -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, dim * sizeof(float), SQLITE_TRANSIENT); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + sqlite3_free(qvec); + } + } + + sqlite3_close(db); + return 0; +} diff --git a/tests/fuzz/ivf-shadow-corrupt.c b/tests/fuzz/ivf-shadow-corrupt.c new file mode 100644 index 00000000..34f3d87d --- /dev/null +++ b/tests/fuzz/ivf-shadow-corrupt.c @@ -0,0 +1,229 @@ +/** + * Fuzz target: IVF shadow table corruption. + * + * Creates a trained IVF table, then corrupts IVF shadow table blobs + * (centroids, cells validity/rowids/vectors, rowid_map) with fuzz data. + * Then exercises all read/write paths. Must not crash. + * + * Targets: + * - Cell validity bitmap with wrong size + * - Cell rowids blob with wrong size/alignment + * - Cell vectors blob with wrong size + * - Centroid blob with wrong size + * - n_vectors inconsistent with validity bitmap + * - Missing rowid_map entries + * - KNN scan over corrupted cells + * - Insert/delete with corrupted rowid_map + */ +#include +#include +#include +#include +#include +#include "sqlite-vec.h" +#include "sqlite3.h" +#include + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + if (size < 4) return 0; + + int rc; + sqlite3 *db; + + rc = sqlite3_open(":memory:", &db); + assert(rc == SQLITE_OK); + rc = sqlite3_vec_init(db, NULL, NULL); + assert(rc == SQLITE_OK); + + // Create IVF table and insert enough vectors to train + rc = sqlite3_exec(db, + "CREATE VIRTUAL TABLE v USING vec0(" + "emb float[8] indexed by ivf(nlist=2, nprobe=2))", + NULL, NULL, NULL); + if (rc != SQLITE_OK) { sqlite3_close(db); return 0; } + + // Insert 10 vectors + { + sqlite3_stmt *si = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &si, NULL); + if (!si) { sqlite3_close(db); return 0; } + for (int i = 0; i < 10; i++) { + float vec[8]; + for (int d = 0; d < 8; d++) { + vec[d] = (float)(i * 8 + d) * 0.1f; + } + sqlite3_reset(si); + sqlite3_bind_int64(si, 1, i + 1); + sqlite3_bind_blob(si, 2, vec, sizeof(vec), SQLITE_TRANSIENT); + sqlite3_step(si); + } + sqlite3_finalize(si); + } + + // Train + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // Now corrupt shadow tables based on fuzz input + uint8_t target = data[0] % 10; + const uint8_t *payload = data + 1; + int payload_size = (int)(size - 1); + + // Limit payload to avoid huge allocations + if (payload_size > 4096) payload_size = 4096; + + sqlite3_stmt *stmt = NULL; + + switch (target) { + case 0: { + // Corrupt cell validity blob + rc = sqlite3_prepare_v2(db, + "UPDATE v_ivf_cells00 SET validity = ? WHERE rowid = 1", + -1, &stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC); + sqlite3_step(stmt); sqlite3_finalize(stmt); + } + break; + } + case 1: { + // Corrupt cell rowids blob + rc = sqlite3_prepare_v2(db, + "UPDATE v_ivf_cells00 SET rowids = ? WHERE rowid = 1", + -1, &stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC); + sqlite3_step(stmt); sqlite3_finalize(stmt); + } + break; + } + case 2: { + // Corrupt cell vectors blob + rc = sqlite3_prepare_v2(db, + "UPDATE v_ivf_cells00 SET vectors = ? WHERE rowid = 1", + -1, &stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC); + sqlite3_step(stmt); sqlite3_finalize(stmt); + } + break; + } + case 3: { + // Corrupt centroid blob + rc = sqlite3_prepare_v2(db, + "UPDATE v_ivf_centroids00 SET centroid = ? WHERE centroid_id = 0", + -1, &stmt, NULL); + if (rc == SQLITE_OK) { + sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC); + sqlite3_step(stmt); sqlite3_finalize(stmt); + } + break; + } + case 4: { + // Set n_vectors to a bogus value (larger than cell capacity) + int bogus_n = 99999; + if (payload_size >= 4) { + uint32_t raw; + memcpy(&raw, payload, 4); + bogus_n = (int)(raw % 100000); + } + char sql[128]; + snprintf(sql, sizeof(sql), + "UPDATE v_ivf_cells00 SET n_vectors = %d WHERE rowid = 1", bogus_n); + sqlite3_exec(db, sql, NULL, NULL, NULL); + break; + } + case 5: { + // Delete rowid_map entries (orphan vectors) + sqlite3_exec(db, + "DELETE FROM v_ivf_rowid_map00 WHERE rowid IN (1, 2, 3)", + NULL, NULL, NULL); + break; + } + case 6: { + // Corrupt rowid_map slot values + char sql[128]; + int bogus_slot = payload_size > 0 ? (int)payload[0] * 1000 : 99999; + snprintf(sql, sizeof(sql), + "UPDATE v_ivf_rowid_map00 SET slot = %d WHERE rowid = 1", bogus_slot); + sqlite3_exec(db, sql, NULL, NULL, NULL); + break; + } + case 7: { + // Corrupt rowid_map cell_id values + sqlite3_exec(db, + "UPDATE v_ivf_rowid_map00 SET cell_id = 99999 WHERE rowid = 1", + NULL, NULL, NULL); + break; + } + case 8: { + // Delete all centroids (make trained but no centroids) + sqlite3_exec(db, + "DELETE FROM v_ivf_centroids00", + NULL, NULL, NULL); + break; + } + case 9: { + // Set validity to NULL + sqlite3_exec(db, + "UPDATE v_ivf_cells00 SET validity = NULL WHERE rowid = 1", + NULL, NULL, NULL); + break; + } + } + + // Exercise all read paths over corrupted state — must not crash + float qvec[8] = {1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; + + // KNN query + { + sqlite3_stmt *sk = NULL; + sqlite3_prepare_v2(db, + "SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 5", + -1, &sk, NULL); + if (sk) { + sqlite3_bind_blob(sk, 1, qvec, sizeof(qvec), SQLITE_STATIC); + while (sqlite3_step(sk) == SQLITE_ROW) {} + sqlite3_finalize(sk); + } + } + + // Full scan + sqlite3_exec(db, "SELECT * FROM v", NULL, NULL, NULL); + + // Point query + sqlite3_exec(db, "SELECT * FROM v WHERE rowid = 1", NULL, NULL, NULL); + sqlite3_exec(db, "SELECT * FROM v WHERE rowid = 5", NULL, NULL, NULL); + + // Delete + sqlite3_exec(db, "DELETE FROM v WHERE rowid = 3", NULL, NULL, NULL); + + // Insert after corruption + { + float newvec[8] = {0.5f, 0.5f, 0.5f, 0.5f, 0.5f, 0.5f, 0.5f, 0.5f}; + sqlite3_stmt *si = NULL; + sqlite3_prepare_v2(db, + "INSERT INTO v(v, emb) VALUES (?, ?)", -1, &si, NULL); + if (si) { + sqlite3_bind_int64(si, 1, 100); + sqlite3_bind_blob(si, 2, newvec, sizeof(newvec), SQLITE_STATIC); + sqlite3_step(si); + sqlite3_finalize(si); + } + } + + // compute-centroids over corrupted state + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('compute-centroids')", + NULL, NULL, NULL); + + // clear-centroids + sqlite3_exec(db, + "INSERT INTO v(v) VALUES ('clear-centroids')", + NULL, NULL, NULL); + + sqlite3_close(db); + return 0; +} diff --git a/tests/sqlite-vec-internal.h b/tests/sqlite-vec-internal.h index 458ee2eb..3c89a2ae 100644 --- a/tests/sqlite-vec-internal.h +++ b/tests/sqlite-vec-internal.h @@ -71,11 +71,18 @@ enum Vec0DistanceMetrics { #define SQLITE_VEC_ENABLE_DISKANN 1 #endif +#ifndef SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +#define SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE 0 +#endif + enum Vec0IndexType { VEC0_INDEX_TYPE_FLAT = 1, #if SQLITE_VEC_ENABLE_RESCORE VEC0_INDEX_TYPE_RESCORE = 2, #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + VEC0_INDEX_TYPE_IVF = 3, +#endif #if SQLITE_VEC_ENABLE_DISKANN VEC0_INDEX_TYPE_DISKANN = 4, #endif @@ -111,6 +118,21 @@ struct Vec0DiskannConfig { }; #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +enum Vec0IvfQuantizer { + VEC0_IVF_QUANTIZER_NONE = 0, + VEC0_IVF_QUANTIZER_INT8 = 1, + VEC0_IVF_QUANTIZER_BINARY = 2, +}; + +struct Vec0IvfConfig { + int nlist; + int nprobe; + enum Vec0IvfQuantizer quantizer; + int oversample; +}; +#endif + struct VectorColumnDefinition { char *name; int name_length; @@ -124,6 +146,9 @@ struct VectorColumnDefinition { #if SQLITE_VEC_ENABLE_DISKANN struct Vec0DiskannConfig diskann; #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + struct Vec0IvfConfig ivf; +#endif }; int vec0_parse_vector_column(const char *source, int source_length, @@ -226,4 +251,11 @@ int diskann_visited_set_insert(struct DiskannVisitedSet *set, long long rowid, int *outAdded); #endif +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +void ivf_quantize_int8(const float *src, int8_t *dst, size_t dimensions); +void ivf_quantize_binary(const float *src, uint8_t *dst, size_t dimensions); +int ivf_kmeans(const float *vectors, int n, size_t dimensions, int k, + int max_iter, uint32_t seed, float *out_centroids); +#endif + #endif /* SQLITE_VEC_INTERNAL_H */ diff --git a/tests/test-error-paths.py b/tests/test-error-paths.py index 4444f989..ca693d8e 100644 --- a/tests/test-error-paths.py +++ b/tests/test-error-paths.py @@ -14,7 +14,13 @@ import subprocess import sys -from conftest import get_extension_path +from conftest import _has_build_flag, get_extension_path + +HAS_IVF = _has_build_flag("ivf") +ivf_only = pytest.mark.skipif( + not HAS_IVF, + reason="IVF not enabled (compile with -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1)", +) def _raises(message, error=sqlite3.OperationalError): @@ -452,6 +458,39 @@ def interrupt_template(): "INSERT INTO vb(rowid, vector) VALUES (?, ?)", [(i, diskann_vectors[i]) for i in range(1, 8)], ) + if HAS_IVF: + # vi and viq are indexed by ivf(), with centroids: rows 1 to 9 are in + # the cells of their centroids, and rowid 3 left a free slot. viq + # quantizes, so it keeps its float32 vectors in _ivf_vectors00. viu has + # two centroids set by hand and its rows in cells of no centroid, for + # assign-vectors. + for table, options in ( + ("vi", "nlist=2, nprobe=2"), + ("viq", "nlist=2, quantizer=int8, oversample=4"), + ): + db.execute( + f"CREATE VIRTUAL TABLE {table} USING vec0(vector float[8] " + f"indexed by ivf({options}), +a text)" + ) + db.executemany( + f"INSERT INTO {table}(rowid, vector, a) VALUES (?, ?, ?)", + [(i, diskann_vectors[i], f"aux {i}") for i in range(1, 10)], + ) + db.execute(f"INSERT INTO {table}({table}) VALUES ('compute-centroids')") + db.execute(f"DELETE FROM {table} WHERE rowid = 3") + db.execute( + "CREATE VIRTUAL TABLE viu USING vec0(vector float[8] " + "indexed by ivf(nlist=0))" + ) + db.executemany( + "INSERT INTO viu(rowid, vector) VALUES (?, ?)", + [(i, diskann_vectors[i]) for i in range(1, 7)], + ) + for centroid in (0, 1): + db.execute( + f"INSERT INTO viu(viu, vector) VALUES ('set-centroid:{centroid}', ?)", + [diskann_vectors[centroid + 7]], + ) db.commit() return db.serialize() @@ -596,6 +635,92 @@ def handler(): pytest.param("DELETE FROM vd WHERE rowid = 1", id="delete-diskann-medoid"), pytest.param("DELETE FROM vb WHERE rowid = 6", id="delete-diskann-buffered"), pytest.param("INSERT INTO vd(vd) VALUES ('optimize')", id="optimize-diskann"), + pytest.param( + "SELECT rowid, distance, a FROM vi WHERE vector MATCH " + "'[0.1, 0.2, -0.3, 0.4, -0.1, 0.2, 0.3, -0.4]' AND k = 3 " + "AND distance < 10", + id="knn-ivf", + marks=ivf_only, + ), + pytest.param( + "SELECT rowid, distance FROM vi WHERE vector MATCH " + "'[0.1, 0.2, -0.3, 0.4, -0.1, 0.2, 0.3, -0.4]' AND k = 2 " + "AND rowid IN (1, 3, 5)", + id="knn-rowid-in-ivf", + marks=ivf_only, + ), + pytest.param( + "SELECT rowid, distance FROM viq WHERE vector MATCH " + "'[0.1, 0.2, -0.3, 0.4, -0.1, 0.2, 0.3, -0.4]' AND k = 3", + id="knn-quantized-ivf", + marks=ivf_only, + ), + pytest.param( + "SELECT rowid, vector, a FROM vi", id="fullscan-ivf", marks=ivf_only + ), + pytest.param( + "SELECT rowid, vector FROM viq", id="fullscan-quantized-ivf", marks=ivf_only + ), + pytest.param( + "SELECT rowid, vector, a FROM vi WHERE rowid = 4", + id="point-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO vi(rowid, vector, a) VALUES " + "(100, '[0.1, 0.1, 0.1, 0.1, -0.1, -0.1, -0.1, -0.1]', 'x')", + id="insert-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viq(rowid, vector, a) VALUES " + "(100, '[0.1, 0.1, 0.1, 0.1, -0.1, -0.1, -0.1, -0.1]', 'x')", + id="insert-quantized-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viu(rowid, vector) VALUES " + "(100, '[0.1, 0.1, 0.1, 0.1, -0.1, -0.1, -0.1, -0.1]')", + id="insert-unassigned-ivf", + marks=ivf_only, + ), + pytest.param( + "UPDATE vi SET a = 'y' WHERE rowid = 4", id="update-ivf", marks=ivf_only + ), + pytest.param("DELETE FROM vi WHERE rowid = 4", id="delete-ivf", marks=ivf_only), + pytest.param( + "DELETE FROM viq WHERE rowid = 4", id="delete-quantized-ivf", marks=ivf_only + ), + pytest.param( + "INSERT INTO vi(vi) VALUES ('optimize')", id="optimize-ivf", marks=ivf_only + ), + pytest.param( + "INSERT INTO vi(vi) VALUES ('compute-centroids')", + id="compute-centroids-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viq(viq) VALUES " + """('compute-centroids:{"nlist": 3, "seed": 2}')""", + id="compute-centroids-quantized-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viq(viq) VALUES ('clear-centroids')", + id="clear-centroids-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viu(viu, vector) VALUES " + "('set-centroid:5', '[0.1, 0.1, 0.1, 0.1, -0.1, -0.1, -0.1, -0.1]')", + id="set-centroid-ivf", + marks=ivf_only, + ), + pytest.param( + "INSERT INTO viu(viu) VALUES ('assign-vectors')", + id="assign-vectors-ivf", + marks=ivf_only, + ), ], ) def test_interrupt_is_reported_as_interrupt(interrupt_template, sql, mode): diff --git a/tests/test-ivf-mutations.py b/tests/test-ivf-mutations.py new file mode 100644 index 00000000..455d3003 --- /dev/null +++ b/tests/test-ivf-mutations.py @@ -0,0 +1,803 @@ +""" +Thorough IVF mutation tests: insert, delete, update, KNN correctness, +error cases, edge cases, and cell overflow scenarios. +""" + +import pytest +import sqlite3 +import struct +from collections import OrderedDict + + +def _f32(values): + return struct.pack("%sf" % len(values), *values) + + +def exec(db, sql, parameters=[]): + try: + rows = db.execute(sql, parameters).fetchall() + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + return { + "error": e.__class__.__name__, + "message": str(e), + } + a = [] + for row in rows: + o = OrderedDict() + for k in row.keys(): + o[k] = row[k] + a.append(o) + result = OrderedDict() + result["sql"] = sql + result["rows"] = a + return result + + +def ivf_total_vectors(db, table="t", col=0): + """Count total vectors across all IVF cells.""" + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d}" + ).fetchone()[0] + + +def ivf_unassigned_count(db, table="t", col=0): + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d} WHERE centroid_id = -1" + ).fetchone()[0] + + +def ivf_assigned_count(db, table="t", col=0): + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d} WHERE centroid_id >= 0" + ).fetchone()[0] + + +def knn(db, query, k, table="t", col="v"): + """Run a KNN query and return list of (rowid, distance) tuples.""" + rows = db.execute( + f"SELECT rowid, distance FROM {table} WHERE {col} MATCH ? AND k = ?", + [_f32(query), k], + ).fetchall() + return [(r[0], r[1]) for r in rows] + + +# ============================================================================ +# Single row insert + KNN +# ============================================================================ + + +def test_insert_single_row_knn(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf())") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + results = knn(db, [1, 0, 0, 0], 5) + assert len(results) == 1 + assert results[0][0] == 1 + assert results[0][1] < 0.001 + + +# ============================================================================ +# Batch insert + KNN recall +# ============================================================================ + + +def test_batch_insert_knn_recall(db): + """Insert 200 vectors, train, verify KNN recall with nprobe=nlist.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=8, nprobe=8))" + ) + for i in range(200): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + assert ivf_total_vectors(db) == 200 + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert ivf_assigned_count(db) == 200 + + # Query near 100 -- closest should be rowid 100 + results = knn(db, [100.0, 0, 0, 0], 10) + assert len(results) == 10 + assert results[0][0] == 100 + assert results[0][1] < 0.01 + + # All results should be near 100 + rowids = {r[0] for r in results} + assert all(95 <= r <= 105 for r in rowids) + + +# ============================================================================ +# Delete rows, verify they're gone from KNN +# ============================================================================ + + +def test_delete_rows_gone_from_knn(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + for i in range(20): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Delete rowid 10 + db.execute("DELETE FROM t WHERE rowid = 10") + + results = knn(db, [10.0, 0, 0, 0], 20) + rowids = {r[0] for r in results} + assert 10 not in rowids + + +def test_delete_all_rows_empty_results(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + for i in range(10): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + for i in range(10): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + + assert ivf_total_vectors(db) == 0 + results = knn(db, [5.0, 0, 0, 0], 10) + assert len(results) == 0 + + +# ============================================================================ +# Insert after delete (reuse rowids) +# ============================================================================ + + +def test_insert_after_delete_reuse_rowid(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + for i in range(10): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Delete rowid 5 + db.execute("DELETE FROM t WHERE rowid = 5") + + # Re-insert rowid 5 with a very different vector + db.execute("INSERT INTO t(rowid, v) VALUES (5, ?)", [_f32([999.0, 0, 0, 0])]) + + # KNN near 999 should find rowid 5 + results = knn(db, [999.0, 0, 0, 0], 1) + assert len(results) >= 1 + assert results[0][0] == 5 + + +# ============================================================================ +# Update vectors (INSERT OR REPLACE), verify KNN reflects new values +# ============================================================================ + + +def test_update_vector_via_delete_insert(db): + """vec0 IVF update: delete then re-insert with new vector.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + for i in range(10): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # "Update" rowid 3: delete and re-insert with new vector + db.execute("DELETE FROM t WHERE rowid = 3") + db.execute( + "INSERT INTO t(rowid, v) VALUES (3, ?)", + [_f32([100.0, 0, 0, 0])], + ) + + # KNN near 100 should find rowid 3 + results = knn(db, [100.0, 0, 0, 0], 1) + assert results[0][0] == 3 + + +# ============================================================================ +# Error cases: IVF + auxiliary/metadata/partition key columns +# ============================================================================ + + +def test_ivf_with_auxiliary_column(db): + """IVF should support auxiliary columns.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(), +extra text)" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE name LIKE 't_%' ORDER BY 1" + ).fetchall() + ] + assert "t_auxiliary" in tables + + +def test_error_ivf_with_metadata_column(db): + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(), genre text)", + ) + assert "error" in result + assert "metadata" in result.get("message", "").lower() + + +def test_error_ivf_with_partition_key(db): + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(), user_id integer partition key)", + ) + assert "error" in result + assert "partition" in result.get("message", "").lower() + + +def test_flat_with_auxiliary_still_works(db): + """Regression guard: flat-indexed tables with aux columns should still work.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4], +extra text)") + db.execute( + "INSERT INTO t(rowid, v, extra) VALUES (1, ?, 'hello')", + [_f32([1, 0, 0, 0])], + ) + row = db.execute("SELECT extra FROM t WHERE rowid = 1").fetchone() + assert row[0] == "hello" + + +def test_flat_with_metadata_still_works(db): + """Regression guard: flat-indexed tables with metadata columns should still work.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4], genre text)") + db.execute( + "INSERT INTO t(rowid, v, genre) VALUES (1, ?, 'rock')", + [_f32([1, 0, 0, 0])], + ) + row = db.execute("SELECT genre FROM t WHERE rowid = 1").fetchone() + assert row[0] == "rock" + + +def test_flat_with_partition_key_still_works(db): + """Regression guard: flat-indexed tables with partition key should still work.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4], user_id integer partition key)" + ) + db.execute( + "INSERT INTO t(rowid, v, user_id) VALUES (1, ?, 42)", + [_f32([1, 0, 0, 0])], + ) + row = db.execute("SELECT user_id FROM t WHERE rowid = 1").fetchone() + assert row[0] == 42 + + +# ============================================================================ +# Edge cases +# ============================================================================ + + +def test_zero_vectors(db): + """Insert zero vectors, verify KNN still works.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf())") + for i in range(5): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([0, 0, 0, 0])], + ) + results = knn(db, [0, 0, 0, 0], 5) + assert len(results) == 5 + # All distances should be 0 + for _, dist in results: + assert dist < 0.001 + + +def test_large_values(db): + """Insert vectors with very large and small values.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf())") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1e6, 1e6, 1e6, 1e6])]) + db.execute( + "INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([1e-6, 1e-6, 1e-6, 1e-6])] + ) + db.execute( + "INSERT INTO t(rowid, v) VALUES (3, ?)", [_f32([-1e6, -1e6, -1e6, -1e6])] + ) + + results = knn(db, [1e6, 1e6, 1e6, 1e6], 3) + assert results[0][0] == 1 + + +def test_single_row_compute_centroids(db): + """Single row table, compute-centroids should still work.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=1))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 2, 3, 4])]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert ivf_assigned_count(db) == 1 + + results = knn(db, [1, 2, 3, 4], 1) + assert len(results) == 1 + assert results[0][0] == 1 + + +# ============================================================================ +# Cell overflow (many vectors in one cell) +# ============================================================================ + + +def test_cell_overflow_many_vectors(db): + """Insert >64 vectors that all go to same centroid. Should create multiple cells.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=0))") + # Insert 100 very similar vectors + for i in range(100): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([1.0 + i * 0.001, 0, 0, 0])], + ) + + # Set a single centroid so all vectors go there + db.execute( + "INSERT INTO t(t, v) VALUES ('set-centroid:0', ?)", + [_f32([1.0, 0, 0, 0])], + ) + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + + assert ivf_assigned_count(db) == 100 + + # Should have more than 1 cell (64 max per cell) + cell_count = db.execute( + "SELECT count(*) FROM t_ivf_cells00 WHERE centroid_id = 0" + ).fetchone()[0] + assert cell_count >= 2 # 100 / 64 = 2 cells needed + + # All vectors should be queryable + results = knn(db, [1.0, 0, 0, 0], 100) + assert len(results) == 100 + + +# ============================================================================ +# Large batch with training +# ============================================================================ + + +def test_large_batch_with_training(db): + """Insert 500, train, insert 500 more, verify total is 1000.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=16, nprobe=16))" + ) + for i in range(500): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + for i in range(500, 1000): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + assert ivf_total_vectors(db) == 1000 + + # KNN should still work + results = knn(db, [750.0, 0, 0, 0], 5) + assert len(results) == 5 + assert results[0][0] == 750 + + +# ============================================================================ +# KNN after interleaved insert/delete +# ============================================================================ + + +def test_knn_after_interleaved_insert_delete(db): + """Insert 20, train, delete 10 closest to query, verify remaining.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4, nprobe=4))" + ) + for i in range(20): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Delete rowids 0-9 (closest to query at 5.0) + for i in range(10): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + + results = knn(db, [5.0, 0, 0, 0], 10) + rowids = {r[0] for r in results} + # None of the deleted rowids should appear + assert all(r >= 10 for r in rowids) + assert len(results) == 10 + + +def test_knn_empty_centroids_after_deletes(db): + """Some centroids may become empty after deletes. Should not crash.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4, nprobe=2))" + ) + # Insert vectors clustered near 4 points + for i in range(40): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i % 10) * 10, 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Delete a bunch, potentially emptying some centroids + for i in range(30): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + + # Should not crash even with empty centroids + results = knn(db, [50.0, 0, 0, 0], 20) + assert len(results) <= 10 # only 10 left + + +# ============================================================================ +# KNN returns correct distances +# ============================================================================ + + +def test_knn_correct_distances(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([0, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([3, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v) VALUES (3, ?)", [_f32([0, 4, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + results = knn(db, [0, 0, 0, 0], 3) + result_map = {r[0]: r[1] for r in results} + + # L2 distances (sqrt of sum of squared differences) + assert abs(result_map[1] - 0.0) < 0.01 + assert abs(result_map[2] - 3.0) < 0.01 # sqrt(3^2) = 3 + assert abs(result_map[3] - 4.0) < 0.01 # sqrt(4^2) = 4 + + +# ============================================================================ +# Delete in flat mode leaves no orphan rowid_map entries +# ============================================================================ + + +def test_delete_flat_mode_rowid_map_count(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + for i in range(5): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("DELETE FROM t WHERE rowid = 0") + db.execute("DELETE FROM t WHERE rowid = 2") + db.execute("DELETE FROM t WHERE rowid = 4") + + assert db.execute("SELECT count(*) FROM t_ivf_rowid_map00").fetchone()[0] == 2 + assert ivf_unassigned_count(db) == 2 + + +# ============================================================================ +# Duplicate rowid insert +# ============================================================================ + + +def test_delete_reinsert_as_update(db): + """Simulate update via delete + insert on same rowid.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + + # Delete then re-insert as "update" + db.execute("DELETE FROM t WHERE rowid = 1") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([99, 0, 0, 0])]) + + results = knn(db, [99, 0, 0, 0], 1) + assert len(results) == 1 + assert results[0][0] == 1 + assert results[0][1] < 0.01 + + +def test_duplicate_rowid_insert_fails(db): + """Inserting a duplicate rowid should fail with a constraint error.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + + result = exec( + db, + "INSERT INTO t(rowid, v) VALUES (1, ?)", + [_f32([99, 0, 0, 0])], + ) + assert "error" in result + + +# ============================================================================ +# Interleaved insert/delete with KNN correctness +# ============================================================================ + + +def test_interleaved_ops_correctness(db): + """Complex sequence of inserts and deletes, verify KNN is always correct.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4, nprobe=4))" + ) + + # Phase 1: Insert 50 vectors + for i in range(50): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Phase 2: Delete even-numbered rowids + for i in range(0, 50, 2): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + + # Phase 3: Insert new vectors with higher rowids + for i in range(50, 75): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(i), 0, 0, 0])], + ) + + # Phase 4: Delete some of the new ones + for i in range(60, 70): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + + # KNN query: should only find existing vectors + results = knn(db, [25.0, 0, 0, 0], 50) + rowids = {r[0] for r in results} + + # Verify no deleted rowids appear + deleted = set(range(0, 50, 2)) | set(range(60, 70)) + assert len(rowids & deleted) == 0 + + # Verify we get the right count (25 odd + 15 new - 10 deleted new = 30) + expected_alive = set(range(1, 50, 2)) | set(range(50, 60)) | set(range(70, 75)) + assert rowids.issubset(expected_alive) + + +def test_ivf_update_vector_blocked(db): + """UPDATE on a vector column with IVF index should error (index would become stale).""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(emb float[4] indexed by ivf(nlist=2))" + ) + db.execute("INSERT INTO t(rowid, emb) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, emb) VALUES (2, ?)", [_f32([0, 1, 0, 0])]) + + with pytest.raises( + sqlite3.OperationalError, match="UPDATE on vector column.*not supported for IVF" + ): + db.execute("UPDATE t SET emb = ? WHERE rowid = 1", [_f32([0, 0, 1, 0])]) + + +# ============================================================================ +# Regression: slot allocation must reuse freed slots, not overwrite live ones +# ============================================================================ + + +def test_insert_after_delete_no_overwrite(db): + """Inserts after deletes must go into freed slots. Previously the insert + appended at slot=n_vectors, overwriting live vectors once a cell had + interior holes (data corruption + rows vanishing from KNN).""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=1))") + + # Fill exactly one cell to capacity (64) in untrained mode + for i in range(64): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([float(i), 0, 0, 0])] + ) + # Free slots 0..9 + for i in range(10): + db.execute("DELETE FROM t WHERE rowid = ?", [i]) + # These must land in the freed slots + for i in range(100, 110): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([float(i), 0, 0, 0])] + ) + + expected = set(range(10, 64)) | set(range(100, 110)) + rows = db.execute("SELECT rowid, vec_to_json(v) AS vj FROM t").fetchall() + assert set(r["rowid"] for r in rows) == expected + import json + + for r in rows: + vec = json.loads(r["vj"]) + assert vec[0] == pytest.approx( + float(r["rowid"]) + ), f"rowid {r['rowid']} vector corrupted: {vec}" + + # Every live row must be reachable via KNN + found = set(rowid for rowid, _ in knn(db, [50.0, 0, 0, 0], 100)) + assert found == expected + + # Bookkeeping: n_vectors must equal number of live rows + assert ivf_total_vectors(db) == len(expected) + + +def test_insert_delete_churn_consistency(db): + """Randomized insert/delete churn across multiple cells stays consistent.""" + import random + + rng = random.Random(42) + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + + alive = {} + next_rowid = 0 + for _ in range(600): + if alive and rng.random() < 0.4: + rid = rng.choice(list(alive)) + db.execute("DELETE FROM t WHERE rowid = ?", [rid]) + del alive[rid] + else: + vec = [rng.uniform(-1, 1) for _ in range(4)] + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [next_rowid, _f32(vec)]) + alive[next_rowid] = vec + next_rowid += 1 + + found = set(rowid for rowid, _ in knn(db, [0.0, 0, 0, 0], len(alive) + 10)) + assert found == set(alive) + assert ivf_total_vectors(db) == len(alive) + import json + + for rid, vec in alive.items(): + stored = json.loads( + db.execute( + "SELECT vec_to_json(v) FROM t WHERE rowid = ?", [rid] + ).fetchone()[0] + ) + assert stored == pytest.approx(vec, abs=1e-6) + + +# ============================================================================ +# Regression: assign-vectors centroid byte stride + non-contiguous ids +# ============================================================================ + + +def test_assign_vectors_correct_assignments(db): + """assign-vectors must place each vector in its nearest centroid's cell. + Previously the centroid array was indexed with a stride of D bytes instead + of D*sizeof(float), so all but centroid 0 were garbage.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + + truth = {} + rowid = 0 + for c in range(4): + for i in range(5): + truth[rowid] = c + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [rowid, _f32([c * 100.0 + i * 0.1] * 4)], + ) + rowid += 1 + + for c in range(4): + db.execute( + "INSERT INTO t(t, v) VALUES (?, ?)", + [f"set-centroid:{c}", _f32([c * 100.0] * 4)], + ) + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + + for rid, expected_c in truth.items(): + cell_id, _ = db.execute( + "SELECT cell_id, slot FROM t_ivf_rowid_map00 WHERE rowid = ?", [rid] + ).fetchone() + (centroid_id,) = db.execute( + "SELECT centroid_id FROM t_ivf_cells00 WHERE rowid = ?", [cell_id] + ).fetchone() + assert centroid_id == expected_c, f"rowid {rid} assigned to {centroid_id}" + assert ivf_unassigned_count(db) == 0 + + +def test_assign_vectors_non_contiguous_centroid_ids(db): + """set-centroid allows arbitrary centroid ids; assign-vectors must map + to the actual ids, not array indexes.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([0.0, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([100.0] * 4)]) + + db.execute( + "INSERT INTO t(t, v) VALUES ('set-centroid:10', ?)", [_f32([0.0, 0, 0, 0])] + ) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:20', ?)", [_f32([100.0] * 4)]) + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + + used = set( + r[0] + for r in db.execute( + "SELECT DISTINCT centroid_id FROM t_ivf_cells00 WHERE n_vectors > 0" + ).fetchall() + ) + assert used == {10, 20} + # Both rows must remain findable + found = set(rowid for rowid, _ in knn(db, [0.0, 0, 0, 0], 10)) + assert found == {1, 2} + + +@pytest.mark.parametrize("quantizer", ["none", "int8"]) +def test_scan_errors_on_a_row_deleted_while_reading(db, quantizer): + # As test_fullscan_errors_on_a_row_deleted_during_the_scan + # (test-fullscan.py) checks for the other index types: the scan found row + # 12, then the same connection deleted it before the scan read its vector. + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f"v float[2] indexed by ivf(quantizer={quantizer}))" + ) + for i in range(1, 17): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0])]) + scan = db.execute("SELECT rowid, v FROM t") + assert scan.fetchone()[0] == 1 + db.execute("DELETE FROM t WHERE rowid = 12") + with pytest.raises(sqlite3.OperationalError, match="rowid 12") as excinfo: + scan.fetchall() + assert excinfo.value.sqlite_errorname == "SQLITE_ERROR" + + +@pytest.mark.parametrize("quantizer", ["none", "int8"]) +def test_knn_errors_on_a_row_deleted_while_reading(db, quantizer): + # As test_knn_errors_on_a_row_deleted_while_reading (test-knn-topk.py) + # checks for the other index types: the KNN found row 14, then the same + # connection deleted it before the query read its vector. The int8 + # quantizer clamps values outside [-1, 1], so the vectors stay inside it. + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f"v float[2] indexed by ivf(quantizer={quantizer}))" + ) + for i in range(1, 17): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i / 16, 0])]) + knn = db.execute("SELECT rowid, v FROM t WHERE v MATCH ? AND k = 3", [_f32([1, 0])]) + assert knn.fetchone()[0] == 16 + db.execute("DELETE FROM t WHERE rowid = 14") + with pytest.raises(sqlite3.OperationalError, match="rowid 14") as excinfo: + knn.fetchall() + assert excinfo.value.sqlite_errorname == "SQLITE_ERROR" + + +@pytest.mark.parametrize("query", ["scan", "knn"]) +def test_reading_the_id_of_a_row_deleted_mid_read(db, query): + # As test_reading_the_id_or_partition_key_of_a_row_deleted_mid_read + # (test-fullscan.py) checks for the other index types; IVF refuses + # partition keys. The query found the row, then the same connection deleted + # it before the query read its id. Python reads one row ahead, so the + # deleted row is a few rows past the first. + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "id text primary key, v float[2] indexed by ivf())" + ) + for i in range(1, 17): + db.execute("INSERT INTO t(id, v) VALUES (?, ?)", [f"r{i}", _f32([i, 0])]) + if query == "scan": + cursor = db.execute("SELECT id, v FROM t") + deleted = 5 + else: + cursor = db.execute( + "SELECT id, v FROM t WHERE v MATCH ? AND k = 5", [_f32([16, 0])] + ) + deleted = 13 + cursor.fetchone() + db.execute("DELETE FROM t WHERE id = ?", [f"r{deleted}"]) + with pytest.raises(sqlite3.OperationalError, match=f"rowid {deleted}") as excinfo: + cursor.fetchall() + assert excinfo.value.sqlite_errorname == "SQLITE_ERROR" diff --git a/tests/test-ivf-quantization.py b/tests/test-ivf-quantization.py new file mode 100644 index 00000000..e51f194e --- /dev/null +++ b/tests/test-ivf-quantization.py @@ -0,0 +1,385 @@ +import pytest +import sqlite3 +import struct +from collections import OrderedDict + + +def _f32(values): + return struct.pack("%sf" % len(values), *values) + + +def exec(db, sql, parameters=[]): + try: + rows = db.execute(sql, parameters).fetchall() + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + return { + "error": e.__class__.__name__, + "message": str(e), + } + a = [] + for row in rows: + o = OrderedDict() + for k in row.keys(): + o[k] = row[k] + a.append(o) + result = OrderedDict() + result["sql"] = sql + result["rows"] = a + return result + + +# ============================================================================ +# Parser tests — quantizer and oversample options +# ============================================================================ + + +def test_ivf_quantizer_binary(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(nlist=64, quantizer=binary, oversample=10))" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + assert "t_ivf_cells00" in tables + + +def test_ivf_quantizer_int8(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(nlist=64, quantizer=int8))" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + + +def test_ivf_quantizer_none_explicit(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(quantizer=none))" + ) + # Should work — same as no quantizer + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + + +def test_ivf_quantizer_all_params(db): + """All params together: nlist, nprobe, quantizer, oversample.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] distance_metric=cosine " + "indexed by ivf(nlist=128, nprobe=16, quantizer=int8, oversample=4))" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + + +def test_ivf_error_oversample_without_quantizer(db): + """oversample > 1 without quantizer should error.""" + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(oversample=10))", + ) + assert "error" in result + + +def test_ivf_error_unknown_quantizer(db): + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(quantizer=pq))", + ) + assert "error" in result + + +def test_ivf_oversample_1_without_quantizer_ok(db): + """oversample=1 (default) is fine without quantizer.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" "v float[768] indexed by ivf(nlist=64))" + ) + # Should succeed — oversample defaults to 1 + + +# ============================================================================ +# Functional tests — insert, train, query with quantized IVF +# ============================================================================ + + +def test_ivf_int8_insert_and_query(db): + """int8 quantized IVF: insert, train, query.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[4] indexed by ivf(nlist=2, quantizer=int8, oversample=4))" + ) + for i in range(20): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # Should be able to query + rows = db.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 5", + [_f32([10.0, 0, 0, 0])], + ).fetchall() + assert len(rows) == 5 + # Top result should be close to 10 + assert rows[0][0] in range(8, 13) + + # Full vectors should be in _ivf_vectors table + fv_count = db.execute("SELECT count(*) FROM t_ivf_vectors00").fetchone()[0] + assert fv_count == 20 + + +def test_ivf_binary_insert_and_query(db): + """Binary quantized IVF: insert, train, query.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[8] indexed by ivf(nlist=2, quantizer=binary, oversample=4))" + ) + for i in range(20): + # Vectors with varying sign patterns + v = [(i * 0.1 - 1.0) + j * 0.3 for j in range(8)] + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32(v)]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 5", + [_f32([0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5])], + ).fetchall() + assert len(rows) == 5 + + # Full vectors stored + fv_count = db.execute("SELECT count(*) FROM t_ivf_vectors00").fetchone()[0] + assert fv_count == 20 + + +def test_ivf_int8_cell_sizes_smaller(db): + """Cell blobs should be smaller with int8 quantization.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(nlist=2, quantizer=int8, oversample=1))" + ) + for i in range(10): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(x) / 768 for x in range(768)])], + ) + + # Cell vectors blob: 10 vectors at int8 = 10 * 768 = 7680 bytes + # vs float32 = 10 * 768 * 4 = 30720 bytes + # But cells have capacity 64, so blob = 64 * 768 = 49152 (int8) vs 64*768*4=196608 (float32) + blob_size = db.execute( + "SELECT length(vectors) FROM t_ivf_cells00 LIMIT 1" + ).fetchone()[0] + # int8: 64 slots * 768 bytes = 49152 + assert blob_size == 64 * 768 + + +def test_ivf_binary_cell_sizes_smaller(db): + """Cell blobs should be much smaller with binary quantization.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[768] indexed by ivf(nlist=2, quantizer=binary, oversample=1))" + ) + for i in range(10): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([float(x) / 768 for x in range(768)])], + ) + + blob_size = db.execute( + "SELECT length(vectors) FROM t_ivf_cells00 LIMIT 1" + ).fetchone()[0] + # binary: 64 slots * 768/8 bytes = 6144 + assert blob_size == 64 * (768 // 8) + + +def test_ivf_int8_oversample_improves_recall(db): + """Oversample re-ranking should improve recall over oversample=1.""" + # Create two tables: one with oversample=1, one with oversample=10 + db.execute( + "CREATE VIRTUAL TABLE t1 USING vec0(" + "v float[4] indexed by ivf(nlist=4, quantizer=int8, oversample=1))" + ) + db.execute( + "CREATE VIRTUAL TABLE t2 USING vec0(" + "v float[4] indexed by ivf(nlist=4, quantizer=int8, oversample=10))" + ) + for i in range(100): + v = _f32([i * 0.1, (i * 0.1) % 3, (i * 0.3) % 5, i * 0.01]) + db.execute("INSERT INTO t1(rowid, v) VALUES (?, ?)", [i, v]) + db.execute("INSERT INTO t2(rowid, v) VALUES (?, ?)", [i, v]) + + db.execute("INSERT INTO t1(t1) VALUES ('compute-centroids')") + db.execute("INSERT INTO t2(t2) VALUES ('compute-centroids')") + db.execute("INSERT INTO t1(t1) VALUES ('nprobe=4')") + db.execute("INSERT INTO t2(t2) VALUES ('nprobe=4')") + + query = _f32([5.0, 1.5, 2.5, 0.5]) + r1 = db.execute("SELECT rowid FROM t1 WHERE v MATCH ? AND k=10", [query]).fetchall() + r2 = db.execute("SELECT rowid FROM t2 WHERE v MATCH ? AND k=10", [query]).fetchall() + + # Both should return 10 results + assert len(r1) == 10 + assert len(r2) == 10 + # oversample=10 should have at least as good recall (same or better ordering) + + +def test_ivf_quantized_delete(db): + """Delete should remove from both cells and _ivf_vectors.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[4] indexed by ivf(nlist=2, quantizer=int8, oversample=1))" + ) + for i in range(10): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert db.execute("SELECT count(*) FROM t_ivf_vectors00").fetchone()[0] == 10 + + db.execute("DELETE FROM t WHERE rowid = 5") + # _ivf_vectors should have 9 rows + assert db.execute("SELECT count(*) FROM t_ivf_vectors00").fetchone()[0] == 9 + + +def test_ivf_binary_rejects_non_multiple_of_8_dims(db): + """Binary quantizer requires dimensions divisible by 8.""" + with pytest.raises(sqlite3.OperationalError): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + " v float[12] indexed by ivf(quantizer=binary)" + ")" + ) + + # Dimensions divisible by 8 should work + db.execute( + "CREATE VIRTUAL TABLE t2 USING vec0(" + " v float[16] indexed by ivf(quantizer=binary)" + ")" + ) + + +# ============================================================================ +# Regression: commands on quantized indexes must keep the quantized +# representation consistent (cells + centroids both store quantized bytes) +# ============================================================================ + + +def _self_match_count(db, vecs): + ok = 0 + for rid, v in vecs.items(): + got = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 1", [_f32(v)] + ).fetchone()[0] + ok += got == rid + return ok + + +def _random_unit_vecs(n, d, seed=1): + import random + + rng = random.Random(seed) + return {rid: [rng.uniform(-1, 1) for _ in range(d)] for rid in range(n)} + + +def test_ivf_int8_clear_centroids_preserves_vectors(db): + """clear-centroids must re-quantize full-precision vectors when moving + them back to unassigned cells. Previously it wrote raw float32 bytes into + int8/binary cells, corrupting every vector.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[8] indexed by ivf(nlist=2, quantizer=int8))" + ) + vecs = _random_unit_vecs(20, 8) + for rid, v in vecs.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rid, _f32(v)]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert _self_match_count(db, vecs) == 20 + + db.execute("INSERT INTO t(t) VALUES ('clear-centroids')") + assert _self_match_count(db, vecs) == 20 + + +def test_ivf_binary_clear_centroids_preserves_vectors(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[16] indexed by ivf(nlist=2, quantizer=binary))" + ) + vecs = _random_unit_vecs(20, 16, seed=7) + for rid, v in vecs.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rid, _f32(v)]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + db.execute("INSERT INTO t(t) VALUES ('clear-centroids')") + # All rows must still be present and findable + found = set( + r[0] + for r in db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 50", [_f32(vecs[0])] + ).fetchall() + ) + assert found == set(vecs) + + +def test_ivf_int8_set_centroid_and_insert(db): + """set-centroid on a quantized index must store the quantized + representation; otherwise subsequent inserts find no size-matching + centroid and fail.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[8] indexed by ivf(nlist=2, quantizer=int8))" + ) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:0', ?)", [_f32([0.5] * 8)]) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:1', ?)", [_f32([-0.5] * 8)]) + + # Inserts after manual centroids must succeed and be assigned + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([0.4] * 8)]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([-0.4] * 8)]) + + found = set( + r[0] + for r in db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 10", [_f32([0.4] * 8)] + ).fetchall() + ) + assert found == {1, 2} + + +def test_ivf_int8_assign_vectors(db): + """assign-vectors on a quantized index must interpret cell blobs with the + quantized element size, not float32.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[8] indexed by ivf(nlist=2, quantizer=int8))" + ) + vecs = _random_unit_vecs(30, 8, seed=3) + for rid, v in vecs.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rid, _f32(v)]) + + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:0', ?)", [_f32([0.5] * 8)]) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:1', ?)", [_f32([-0.5] * 8)]) + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + + # No vectors left unassigned, all findable, none corrupted + unassigned = db.execute( + "SELECT COALESCE(SUM(n_vectors),0) FROM t_ivf_cells00 WHERE centroid_id = -1" + ).fetchone()[0] + assert unassigned == 0 + assert _self_match_count(db, vecs) == 30 diff --git a/tests/test-ivf-rename.py b/tests/test-ivf-rename.py new file mode 100644 index 00000000..957bd0c6 --- /dev/null +++ b/tests/test-ivf-rename.py @@ -0,0 +1,110 @@ +"""ALTER TABLE ... RENAME on vec0 tables with an IVF column. + +Ported from asg017/sqlite-vec@8b81f40. +""" + +import sqlite3 +import struct +import pytest + + +def _f32(values): + return struct.pack("%sf" % len(values), *values) + + +def _shadow_tables(db, prefix): + """Return sorted list of shadow table names for a given prefix.""" + return sorted( + [ + row[0] + for row in db.execute( + r"select name from sqlite_master where name like ? escape '\' and type='table' order by 1", + [f"{prefix}\\__%"], + ).fetchall() + ] + ) + + +def test_rename_ivf_no_quantizer(db): + """Rename should rename all IVF shadow tables (_ivf_centroids, _ivf_cells, + _ivf_rowid_map). quantizer=none — no _ivf_vectors table.""" + db.execute(""" + CREATE VIRTUAL TABLE v USING vec0( + a float[4] indexed by ivf(nlist=2, quantizer=none) + ) + """) + db.execute("insert into v(rowid, a) values (1, ?)", [_f32([0.1] * 4)]) + db.execute("insert into v(rowid, a) values (2, ?)", [_f32([0.9] * 4)]) + + before = _shadow_tables(db, "v") + assert "v_ivf_centroids00" in before + assert "v_ivf_cells00" in before + assert "v_ivf_rowid_map00" in before + assert "v_ivf_vectors00" not in before # quantizer=none -> no _ivf_vectors + assert "v_vector_chunks00" not in before + + db.execute("ALTER TABLE v RENAME TO v2") + + # Querying the renamed table should still work — it hits _ivf_cells, + # _ivf_centroids (when trained), and _ivf_rowid_map. + rows = db.execute( + "select rowid from v2 where a match ? and k=10", + [_f32([0.1] * 4)], + ).fetchall() + assert any(r[0] == 1 for r in rows) + + after = _shadow_tables(db, "v2") + assert "v2_ivf_centroids00" in after + assert "v2_ivf_cells00" in after + assert "v2_ivf_rowid_map00" in after + + # No old shadow tables should remain + assert _shadow_tables(db, "v") == [] + + +def test_rename_ivf_quantizer_binary(db): + """Rename should also rename _ivf_vectors when quantizer != none.""" + db.execute(""" + CREATE VIRTUAL TABLE v USING vec0( + a float[8] indexed by ivf(nlist=2, quantizer=binary) + ) + """) + db.execute("insert into v(rowid, a) values (1, ?)", [_f32([0.1] * 8)]) + + before = _shadow_tables(db, "v") + assert "v_ivf_centroids00" in before + assert "v_ivf_cells00" in before + assert "v_ivf_rowid_map00" in before + assert "v_ivf_vectors00" in before # quantizer=binary creates _ivf_vectors + + db.execute("ALTER TABLE v RENAME TO v2") + + rows = db.execute( + "select rowid from v2 where a match ? and k=10", + [_f32([0.1] * 8)], + ).fetchall() + assert rows[0][0] == 1 + + after = _shadow_tables(db, "v2") + assert "v2_ivf_centroids00" in after + assert "v2_ivf_cells00" in after + assert "v2_ivf_rowid_map00" in after + assert "v2_ivf_vectors00" in after + + assert _shadow_tables(db, "v") == [] + + +def test_rename_ivf_drop_after(db): + """DROP TABLE on a renamed IVF table must drop every shadow table — leftover + shadows from a half-renamed IVF index would orphan tables in the schema.""" + db.execute(""" + CREATE VIRTUAL TABLE v USING vec0( + a float[8] indexed by ivf(nlist=2, quantizer=binary) + ) + """) + db.execute("insert into v(rowid, a) values (1, ?)", [_f32([0.1] * 8)]) + db.execute("ALTER TABLE v RENAME TO v2") + db.execute("DROP TABLE v2") + + assert _shadow_tables(db, "v") == [] + assert _shadow_tables(db, "v2") == [] diff --git a/tests/test-ivf.py b/tests/test-ivf.py new file mode 100644 index 00000000..ff2b58aa --- /dev/null +++ b/tests/test-ivf.py @@ -0,0 +1,1321 @@ +import pytest +import random +import sqlite3 +import struct +from collections import OrderedDict +from conftest import get_extension_path + + +def _f32(values): + return struct.pack("%sf" % len(values), *values) + + +def exec(db, sql, parameters=[]): + try: + rows = db.execute(sql, parameters).fetchall() + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + return { + "error": e.__class__.__name__, + "message": str(e), + } + a = [] + for row in rows: + o = OrderedDict() + for k in row.keys(): + o[k] = row[k] + a.append(o) + result = OrderedDict() + result["sql"] = sql + result["rows"] = a + return result + + +def ivf_total_vectors(db, table="t", col=0): + """Count total vectors across all IVF cells.""" + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d}" + ).fetchone()[0] + + +def ivf_unassigned_count(db, table="t", col=0): + """Count vectors in unassigned cells (centroid_id=-1).""" + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d} WHERE centroid_id = -1" + ).fetchone()[0] + + +def ivf_assigned_count(db, table="t", col=0): + """Count vectors in trained cells (centroid_id >= 0).""" + return db.execute( + f"SELECT COALESCE(SUM(n_vectors), 0) FROM {table}_ivf_cells{col:02d} WHERE centroid_id >= 0" + ).fetchone()[0] + + +# ============================================================================ +# Parser tests +# ============================================================================ + + +def test_ivf_create_defaults(db): + """ivf() with no args uses defaults.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf())") + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + assert "t_ivf_cells00" in tables + assert "t_ivf_rowid_map00" in tables + + +def test_ivf_create_custom_params(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=64, nprobe=8))" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + assert "t_ivf_cells00" in tables + + +def test_ivf_create_with_distance_metric(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] distance_metric=cosine indexed by ivf(nlist=16))" + ) + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1" + ).fetchall() + ] + assert "t_ivf_centroids00" in tables + + +def test_ivf_create_error_unknown_key(db): + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(bogus=1))", + ) + assert "error" in result + + +def test_ivf_create_error_nprobe_gt_nlist(db): + result = exec( + db, + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4, nprobe=10))", + ) + assert "error" in result + + +# ============================================================================ +# Shadow table tests +# ============================================================================ + + +def test_ivf_shadow_tables_created(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=8))") + trained = db.execute( + "SELECT value FROM t_info WHERE key = 'ivf_trained_0'" + ).fetchone()[0] + assert str(trained) == "0" + + # No cells yet (created lazily on first insert) + count = db.execute("SELECT count(*) FROM t_ivf_cells00").fetchone()[0] + assert count == 0 + + +def test_shadow_ivf_tables(db): + # As in test_shadow_diskann_tables (test-diskann.py): after VACUUM, SQLite + # asks xShadowName about every backing table, and a name that only looks + # like an IVF table's stays an ordinary table. Upstream's xShadowName + # names none of the four. + db.execute( + "CREATE VIRTUAL TABLE v USING vec0(" + "a float[8] INDEXED BY ivf(nlist=2, quantizer=int8), b float[1])" + ) + db.execute("INSERT INTO v(rowid, a, b) VALUES (1, ?, '[1]')", [_f32([0.5] * 8)]) + db.commit() + for name in [ + "v_ivf_centroids1", + "v_ivf_cells", + "v_ivf_cells0", + "v_ivf_rowid_map000", + "v_ivf_vectors", + ]: + db.execute(f"CREATE TABLE {name}(x)") + db.execute("VACUUM") + types = { + row[0]: row[1] + for row in db.execute( + "SELECT name, type FROM pragma_table_list WHERE schema = 'main'" + ) + if row[0].startswith("v_ivf") + } + assert types == { + "v_ivf_centroids00": "shadow", + "v_ivf_cells00": "shadow", + "v_ivf_rowid_map00": "shadow", + "v_ivf_vectors00": "shadow", + "v_ivf_centroids1": "table", + "v_ivf_cells": "table", + "v_ivf_cells0": "table", + "v_ivf_rowid_map000": "table", + "v_ivf_vectors": "table", + } + + +def test_ivf_drop_cleans_up(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + db.execute("DROP TABLE t") + tables = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + ] + assert not any("ivf" in t for t in tables) + + +@pytest.mark.parametrize( + "missing", + ["t_ivf_centroids00", "t_ivf_cells00", "t_ivf_rowid_map00", "t_ivf_vectors00"], +) +def test_ivf_drop_table_without_a_shadow_table(db, missing): + # Upstream drops the IVF shadow tables with IF EXISTS, so a table that + # lost one still drops, with the rest of its shadow tables. + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + " v float[4] indexed by ivf(nlist=2, quantizer=int8)" + ")" + ) + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + db.execute(f"DROP TABLE {missing}") + db.execute("DROP TABLE t") + names = db.execute("SELECT name FROM sqlite_master WHERE name LIKE 't%'").fetchall() + assert names == [] + + +# ============================================================================ +# Insert tests (flat mode) +# ============================================================================ + + +def test_ivf_insert_flat_mode(db): + """Before training, vectors go to unassigned cell.""" + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 2, 3, 4])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([5, 6, 7, 8])]) + + assert ivf_unassigned_count(db) == 2 + assert ivf_assigned_count(db) == 0 + + # rowid_map should have 2 entries + assert db.execute("SELECT count(*) FROM t_ivf_rowid_map00").fetchone()[0] == 2 + + +def test_ivf_delete_flat_mode(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 2, 3, 4])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([5, 6, 7, 8])]) + db.execute("DELETE FROM t WHERE rowid = 1") + + assert ivf_unassigned_count(db) == 1 + assert db.execute("SELECT count(*) FROM t_ivf_rowid_map00").fetchone()[0] == 1 + + +# ============================================================================ +# KNN flat mode tests +# ============================================================================ + + +def test_ivf_knn_flat_mode(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v) VALUES (2, ?)", [_f32([2, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v) VALUES (3, ?)", [_f32([9, 0, 0, 0])]) + + rows = db.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 2", + [_f32([1.5, 0, 0, 0])], + ).fetchall() + assert len(rows) == 2 + rowids = {r[0] for r in rows} + assert rowids == {1, 2} + + +def test_ivf_knn_flat_empty(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 5", + [_f32([1, 0, 0, 0])], + ).fetchall() + assert len(rows) == 0 + + +# ============================================================================ +# compute-centroids tests +# ============================================================================ + + +def test_compute_centroids(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4))") + for i in range(40): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", + [i, _f32([i % 10, i // 10, 0, 0])], + ) + + assert ivf_unassigned_count(db) == 40 + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + # After training: unassigned cell should be gone (or empty), vectors in trained cells + assert ivf_unassigned_count(db) == 0 + assert ivf_assigned_count(db) == 40 + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 4 + trained = db.execute( + "SELECT value FROM t_info WHERE key='ivf_trained_0'" + ).fetchone()[0] + assert str(trained) == "1" + + +def test_compute_centroids_recompute(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + for i in range(20): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 2 + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 2 + assert ivf_assigned_count(db) == 20 + + +# ============================================================================ +# Insert after training (assigned mode) +# ============================================================================ + + +def test_ivf_insert_after_training(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + for i in range(20): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + db.execute("INSERT INTO t(rowid, v) VALUES (100, ?)", [_f32([5, 0, 0, 0])]) + + # Should be in a trained cell, not unassigned + row = db.execute( + "SELECT m.cell_id, c.centroid_id FROM t_ivf_rowid_map00 m " + "JOIN t_ivf_cells00 c ON c.rowid = m.cell_id " + "WHERE m.rowid = 100" + ).fetchone() + assert row is not None + assert row[1] >= 0 # centroid_id >= 0 means trained cell + + +# ============================================================================ +# KNN after training (IVF probe mode) +# ============================================================================ + + +def test_ivf_knn_after_training(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=4, nprobe=4))" + ) + for i in range(100): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + rows = db.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 5", + [_f32([50.0, 0, 0, 0])], + ).fetchall() + assert len(rows) == 5 + assert rows[0][0] == 50 + assert rows[0][1] < 0.01 + + +def test_ivf_knn_k_larger_than_n(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2, nprobe=2))" + ) + for i in range(5): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 100", + [_f32([0, 0, 0, 0])], + ).fetchall() + assert len(rows) == 5 + + +# ============================================================================ +# Manual centroid import (set-centroid, assign-vectors) +# ============================================================================ + + +def test_set_centroid_and_assign(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=0))") + for i in range(20): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute( + "INSERT INTO t(t, v) VALUES ('set-centroid:0', ?)", + [_f32([5, 0, 0, 0])], + ) + db.execute( + "INSERT INTO t(t, v) VALUES ('set-centroid:1', ?)", + [_f32([15, 0, 0, 0])], + ) + + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 2 + + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + + assert ivf_unassigned_count(db) == 0 + assert ivf_assigned_count(db) == 20 + + +# ============================================================================ +# clear-centroids +# ============================================================================ + + +def test_clear_centroids(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + for i in range(20): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 2 + + db.execute("INSERT INTO t(t) VALUES ('clear-centroids')") + assert db.execute("SELECT count(*) FROM t_ivf_centroids00").fetchone()[0] == 0 + assert ivf_unassigned_count(db) == 20 + trained = db.execute( + "SELECT value FROM t_info WHERE key='ivf_trained_0'" + ).fetchone()[0] + assert str(trained) == "0" + + +# ============================================================================ +# Delete after training +# ============================================================================ + + +def test_ivf_delete_after_training(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=2))") + for i in range(10): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert ivf_assigned_count(db) == 10 + + db.execute("DELETE FROM t WHERE rowid = 5") + assert ivf_assigned_count(db) == 9 + assert db.execute("SELECT count(*) FROM t_ivf_rowid_map00").fetchone()[0] == 9 + + +# ============================================================================ +# Recall test +# ============================================================================ + + +def test_ivf_recall_nprobe_equals_nlist(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[4] indexed by ivf(nlist=8, nprobe=8))" + ) + for i in range(100): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([i, 0, 0, 0])]) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 10", + [_f32([50.0, 0, 0, 0])], + ).fetchall() + rowids = {r[0] for r in rows} + + # 45 and 55 are equidistant from 50, so either may appear in top 10 + assert 50 in rowids + assert len(rowids) == 10 + assert all(45 <= r <= 55 for r in rowids) + + +# ============================================================================ +# Behavior this port defines where upstream's IVF differs +# ============================================================================ + + +def _random_rows(seed, n, dims): + rng = random.Random(seed) + return {i: [rng.uniform(-1, 1) for _ in range(dims)] for i in range(1, n + 1)} + + +def _connect(path): + db = sqlite3.connect(path) + db.enable_load_extension(True) + db.load_extension(get_extension_path()) + db.enable_load_extension(False) + return db + + +def _contents(db, table="t"): + """Every row of every table of the vec0 table, by table name.""" + names = [ + r[0] + for r in db.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + " AND name LIKE ? ESCAPE '\\' ORDER BY name", + [f"{table}\\_%"], + ) + ] + return { + name: db.execute(f'SELECT * FROM "{name}" ORDER BY rowid').fetchall() + for name in names + } + + +# With nprobe = nlist every cell is read, and an oversample as large as the +# table ranks every row by its exact distance, so the KNN of an IVF column must +# answer as a flat column does, constraint for constraint, before and after +# compute-centroids. +@pytest.mark.parametrize( + "options", + [ + "nlist=4, nprobe=4", + "nlist=4, nprobe=4, quantizer=int8, oversample=100", + "nlist=4, nprobe=4, quantizer=binary, oversample=100", + ], +) +@pytest.mark.parametrize( + "where", + [ + "k = 10", + "k = 10 AND distance > 0.4", + "k = 10 AND distance <= 0.8", + "k = 10 AND distance >= 0.2 AND distance < 0.9", + "k = 5 AND rowid IN (3, 7, 11, 19, 23, 31, 999)", + "k = 3 AND rowid = 17", + "k = 6 AND mmr_lambda = 0.5", + "k = 6 AND mmr_lambda = 0.2 AND distance > 0.3", + ], +) +def test_ivf_knn_answers_as_flat_when_every_cell_is_read(db, options, where): + db.execute( + "CREATE VIRTUAL TABLE i USING vec0(" + f"emb float[16] distance_metric=cosine INDEXED BY ivf({options}))" + ) + db.execute( + "CREATE VIRTUAL TABLE f USING vec0(emb float[16] distance_metric=cosine)" + ) + rows = _random_rows(11, 60, 16) + for rowid, vec in rows.items(): + for table in ("i", "f"): + db.execute( + f"INSERT INTO {table}(rowid, emb) VALUES (?, ?)", [rowid, _f32(vec)] + ) + + def compare(): + rng = random.Random(12) + for _ in range(10): + query = _f32([rng.uniform(-1, 1) for _ in range(16)]) + sql = "SELECT rowid, distance FROM {} WHERE emb MATCH ? AND " + where + got = db.execute(sql.format("i"), [query]).fetchall() + want = db.execute(sql.format("f"), [query]).fetchall() + assert [tuple(r) for r in got] == [tuple(r) for r in want] + + compare() + db.execute("INSERT INTO i(i) VALUES ('compute-centroids')") + compare() + + +@pytest.mark.parametrize("quantizer", ["int8", "binary"]) +def test_ivf_quantized_column_returns_its_float_vectors(db, quantizer): + """The cells hold the vectors quantized; the column reads as float32.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f"v float[8] INDEXED BY ivf(nlist=2, quantizer={quantizer}, oversample=2))" + ) + rows = _random_rows(5, 20, 8) + for rowid, vec in rows.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + for command in (None, "compute-centroids", "clear-centroids"): + if command: + db.execute("INSERT INTO t(t) VALUES (?)", [command]) + assert db.execute("SELECT v FROM t WHERE rowid = 7").fetchone()[0] == _f32( + rows[7] + ) + assert {r: v for r, v in db.execute("SELECT rowid, v FROM t")} == { + r: _f32(v) for r, v in rows.items() + } + + +def test_ivf_point_query_for_a_missing_rowid(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf())") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([1] * 8)]) + assert db.execute("SELECT rowid, v FROM t WHERE rowid = 2").fetchall() == [] + assert [tuple(r) for r in db.execute("SELECT rowid FROM t WHERE rowid = 1")] == [ + (1,) + ] + + +def test_ivf_another_connection_reads_the_centroids(tmp_path): + """Whether a column has centroids is read from the database, not kept by + the connection, so a connection sees the centroids another one makes.""" + path = str(tmp_path / "ivf.db") + a = _connect(path) + b = _connect(path) + a.execute( + "CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=4, nprobe=1))" + ) + rows = _random_rows(9, 40, 8) + for rowid, vec in rows.items(): + a.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + a.commit() + sql = "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 5" + query = [_f32(rows[3])] + assert b.execute(sql, query).fetchall() == a.execute(sql, query).fetchall() + for command in ("compute-centroids", "clear-centroids", "compute-centroids"): + a.execute("INSERT INTO t(t) VALUES (?)", [command]) + a.commit() + got = b.execute(sql, query).fetchall() + assert got == a.execute(sql, query).fetchall() + assert got[0] == (3, 0.0) + + +# Upstream matches the keywords of `indexed by ivf(...)`, of its options and of +# its quantizers by prefix, at CREATE too, so its tables can store +# `index by i(nl=4, np=4)`. CREATE here wants whole keywords; connect opens such +# a table. +@pytest.mark.parametrize( + "created, stored", + [ + ("indexed by ivf(nlist=4, nprobe=4)", "index by i(nl=4, np=4)"), + ( + "indexed by ivf(nlist=4, nprobe=4, quantizer=int8, oversample=2)", + "indexed by iv(nli=4, npr=4, quant=i, over=2)", + ), + ], +) +def test_ivf_connect_accepts_keyword_prefix(db, tmp_path, created, stored): + with pytest.raises( + sqlite3.OperationalError, match="(?i)^vec0 constructor error: could not parse" + ): + db.execute(f"CREATE VIRTUAL TABLE t USING vec0(v float[8] {stored})") + + path = str(tmp_path / "ivf.db") + con = _connect(path) + con.execute(f"CREATE VIRTUAL TABLE t USING vec0(v float[8] {created})") + rows = _random_rows(5, 20, 8) + for rowid, vec in rows.items(): + con.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + con.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + con.execute("PRAGMA writable_schema = ON") + con.execute( + "UPDATE sqlite_master SET sql = ? WHERE name = 't'", + [f"CREATE VIRTUAL TABLE t USING vec0(v float[8] {stored})"], + ) + con.commit() + con.close() + + con = _connect(path) + got = con.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 1", [_f32(rows[3])] + ).fetchall() + assert got == [(3,)] + + +def test_ivf_int8_quantizer_ranks_by_the_column_metric(db): + """By cosine, the first two rows are both at distance 0 from the query.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[8] distance_metric=cosine INDEXED BY ivf(nlist=2, quantizer=int8))" + ) + for rowid, vec in ( + (1, [0.9, 0.9, 0, 0, 0, 0, 0, 0]), + (2, [0.1, 0.1, 0, 0, 0, 0, 0, 0]), + (3, [0.5, -0.5, 0, 0, 0, 0, 0, 0]), + ): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + rows = db.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 2", + [_f32([0.2, 0.2, 0, 0, 0, 0, 0, 0])], + ).fetchall() + assert {r[0] for r in rows} == {1, 2} + assert all(r[1] == pytest.approx(0.0, abs=1e-6) for r in rows) + + +def test_ivf_rename_then_create_under_the_old_name(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=2))") + rows = _random_rows(4, 20, 8) + for rowid, vec in rows.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + db.execute("ALTER TABLE t RENAME TO u") + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=2))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32(rows[2])]) + knn = "SELECT rowid FROM {} WHERE v MATCH ? AND k = 1" + assert db.execute(knn.format("u"), [_f32(rows[5])]).fetchone()[0] == 5 + assert db.execute(knn.format("t"), [_f32(rows[2])]).fetchone()[0] == 1 + indexed = sorted( + r[0] + for r in db.execute("SELECT tbl_name FROM sqlite_master WHERE type = 'index'") + if "ivf" in r[0] + ) + assert indexed == ["t_ivf_cells00", "u_ivf_cells00"] + + +@pytest.mark.parametrize( + "columns, message", + [ + ("v int8[8] INDEXED BY ivf()", "could not parse vector column"), + ("v bit[8] INDEXED BY ivf()", "could not parse vector column"), + ( + "v float[8] INDEXED BY ivf(), m integer", + "metadata columns are not supported with IVF indexes", + ), + ( + "p integer partition key, v float[8] INDEXED BY ivf()", + "partition key columns are not supported with IVF indexes", + ), + ], +) +def test_ivf_create_refuses(db, columns, message): + with pytest.raises(sqlite3.OperationalError, match=message): + db.execute(f"CREATE VIRTUAL TABLE t USING vec0({columns})") + + +@pytest.mark.parametrize( + "option", + [ + "quantizer=bin", + "nli=8", + "nlist=8x", + "nlist=99999999999", + "nlist=65537", + "nprobe=0", + "quantizer=int8, oversample=0", + "nlist=8,", + ], +) +def test_ivf_create_option_must_be_whole(db, option): + result = exec( + db, f"CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf({option}))" + ) + assert "error" in result + + +def test_ivf_with_auxiliary_and_text_primary_key(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "id text primary key, v float[8] INDEXED BY ivf(nlist=2), +label text)" + ) + rows = _random_rows(6, 20, 8) + for rowid, vec in rows.items(): + db.execute( + "INSERT INTO t(id, v, label) VALUES (?, ?, ?)", + [f"id{rowid}", _f32(vec), f"label{rowid}"], + ) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + db.execute("DELETE FROM t WHERE id = 'id4'") + for rowid, vec in rows.items(): + got = db.execute( + "SELECT id, label, distance FROM t WHERE v MATCH ? AND k = 1", + [_f32(vec)], + ).fetchone() + if rowid == 4: + assert got[0] != "id4" + else: + assert tuple(got) == (f"id{rowid}", f"label{rowid}", 0.0) + + +@pytest.mark.parametrize( + "command", + ["compute-centroids", "clear-centroids", "assign-vectors"], +) +def test_ivf_interrupted_command_changes_nothing(db, command): + """A command stopped part way leaves every table as it was, and the + connection goes on answering queries.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[8] INDEXED BY ivf(nlist=4, quantizer=int8, oversample=4))" + ) + rows = _random_rows(8, 100, 8) + for rowid, vec in rows.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + if command == "clear-centroids": + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + if command == "assign-vectors": + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:3', ?)", [_f32(rows[1])]) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:8', ?)", [_f32(rows[2])]) + db.commit() + image = db.serialize() + + def fresh(): + c = sqlite3.connect(":memory:") + c.deserialize(image) + c.enable_load_extension(True) + c.load_extension(get_extension_path()) + c.enable_load_extension(False) + return c + + counted = fresh() + steps = 0 + + def count(): + nonlocal steps + steps += 1 + return 0 + + counted.set_progress_handler(count, 1) + counted.execute("INSERT INTO t(t) VALUES (?)", [command]) + counted.set_progress_handler(None, 1) + after = _contents(counted) + + before = _contents(fresh()) + assert after != before + # A progress handler also runs once the statement has halted, when SQLite + # reports the interrupt but has nothing left to roll back, so the points + # stop short of the last steps. + for n in [1 + steps * tenth // 10 for tenth in range(10)]: + c = fresh() + seen = 0 + + def stop(): + nonlocal seen + seen += 1 + return seen >= n + + c.set_progress_handler(stop, 1) + with pytest.raises(sqlite3.OperationalError, match="interrupted"): + c.execute("INSERT INTO t(t) VALUES (?)", [command]) + c.set_progress_handler(None, 1) + assert _contents(c) == before + got = c.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 3", + [_f32(rows[50])], + ).fetchall() + assert got[0] == (50, 0.0) + + +def test_ivf_commands_apply_to_every_ivf_column(db): + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "a float[8] INDEXED BY ivf(nlist=2), " + "f float[8], " + "b float[8] INDEXED BY ivf(nlist=3, quantizer=int8))" + ) + rows = _random_rows(7, 30, 8) + for rowid, vec in rows.items(): + db.execute( + "INSERT INTO t(rowid, a, f, b) VALUES (?, ?, ?, ?)", + [rowid, _f32(vec), _f32(vec), _f32([-x for x in vec])], + ) + + def trained(): + return [ + (key, str(value)) + for key, value in db.execute( + "SELECT key, value FROM t_info WHERE key LIKE 'ivf_trained_%' " + "ORDER BY key" + ) + ] + + def centroids(col): + return db.execute(f"SELECT count(*) FROM t_ivf_centroids{col:02d}").fetchone()[ + 0 + ] + + def unassigned(col): + return ivf_unassigned_count(db, col=col) + + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + assert trained() == [("ivf_trained_0", "1"), ("ivf_trained_2", "1")] + assert (centroids(0), centroids(2)) == (2, 3) + assert (unassigned(0), unassigned(2)) == (0, 0) + + db.execute("INSERT INTO t(t) VALUES ('clear-centroids')") + assert trained() == [("ivf_trained_0", "0"), ("ivf_trained_2", "0")] + assert (centroids(0), centroids(2)) == (0, 0) + assert (unassigned(0), unassigned(2)) == (30, 30) + + db.execute( + "INSERT INTO t(t, a, b) VALUES ('set-centroid:4', ?, ?)", + [_f32([0.5] * 8), _f32([-0.5] * 8)], + ) + assert trained() == [("ivf_trained_0", "1"), ("ivf_trained_2", "1")] + db.execute("INSERT INTO t(t) VALUES ('assign-vectors')") + assert (unassigned(0), unassigned(2)) == (0, 0) + for rowid, vec in rows.items(): + for column, value in (("a", vec), ("b", [-x for x in vec])): + got = db.execute( + f"SELECT rowid, distance FROM t WHERE {column} MATCH ? AND k = 1", + [_f32(value)], + ).fetchone() + assert tuple(got) == (rowid, 0.0) + + +def test_ivf_nprobe_command_lasts_until_the_table_reconnects(db): + # Row 1 is in the cells of centroid 0, the one nearest the query; row 2 is + # nearer the query but in the cells of centroid 1, which nprobe=1 skips. + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[2] indexed by ivf(nlist=2, nprobe=1))" + ) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:0', ?)", [_f32([0, 0])]) + db.execute("INSERT INTO t(t, v) VALUES ('set-centroid:1', ?)", [_f32([10, 0])]) + db.execute( + "INSERT INTO t(rowid, v) VALUES (1, ?), (2, ?)", [_f32([1, 0]), _f32([6, 0])] + ) + + def knn(): + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 1", [_f32([4, 0])] + ) + return [r[0] for r in rows] + + assert knn() == [1] + db.execute("INSERT INTO t(t) VALUES ('nprobe=2')") + assert knn() == [2] + # VACUUM makes SQLite reconnect the table, with the nprobe its CREATE gave. + db.commit() + db.execute("VACUUM") + assert knn() == [1] + + +@pytest.mark.parametrize( + "command, message", + [ + ("nprobe=0", "nprobe must be a whole number from 1 to 2147483647"), + ("nprobe=5x", "nprobe must be a whole number from 1 to 2147483647"), + ("nprobe= 5", "nprobe must be a whole number from 1 to 2147483647"), + ("nprobe=", "nprobe must be a whole number from 1 to 2147483647"), + ("nprobe=2147483648", "nprobe must be a whole number from 1 to 2147483647"), + ( + 'compute-centroids:{"nlist": 0}', + "option nlist must be an integer from 1 to 65536", + ), + ( + 'compute-centroids:{"nlist": 2.5}', + "option nlist must be an integer from 1 to 65536", + ), + ( + 'compute-centroids:{"nlist": "2"}', + "option nlist must be an integer from 1 to 65536", + ), + ( + 'compute-centroids:{"seed": -1}', + "option seed must be an integer from 0 to 4294967295", + ), + ('compute-centroids:{"nlists": 2}', "unknown compute-centroids option: nlists"), + ("compute-centroids:[2]", "options must be a JSON object"), + ("compute-centroids:[]", "options must be a JSON object"), + ("compute-centroids:2", "options must be a JSON object"), + ("compute-centroids:{nlist: 2", "compute-centroids options: malformed JSON"), + ("compute-centroids:", "compute-centroids options: malformed JSON"), + ("set-centroid:-1", "set-centroid needs a centroid id from 0 to 2147483647"), + ("set-centroid:x", "set-centroid needs a centroid id from 0 to 2147483647"), + ("set-centroid:1", "set-centroid needs a vector for an IVF-indexed column"), + ("assign-vectors", "vector column 0 has no IVF centroids"), + ("compute-centroid", "unknown vec0 command"), + ("compute-centroids2", "unknown vec0 command"), + ], +) +def test_ivf_command_errors(db, command, message): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=2))") + db.execute("INSERT INTO t(rowid, v) VALUES (1, ?)", [_f32([0.5] * 8)]) + with pytest.raises(sqlite3.OperationalError, match=message): + db.execute("INSERT INTO t(t) VALUES (?)", [command]) + + +@pytest.mark.parametrize( + "column", + [ + "v float[8]", + "v float[8] INDEXED BY rescore(quantizer=bit)", + "v float[8] INDEXED BY diskann(neighbor_quantizer=binary)", + ], +) +@pytest.mark.parametrize( + "command", + [ + "nprobe=2", + "nprobe=abc", + "compute-centroids", + 'compute-centroids:{"nlist": 2}', + "set-centroid:0", + "assign-vectors", + "clear-centroids", + ], +) +def test_ivf_commands_are_unknown_without_an_ivf_column(db, column, command): + """As upstream, a table with no IVF column has no such command.""" + db.execute(f"CREATE VIRTUAL TABLE t USING vec0({column})") + with pytest.raises(sqlite3.OperationalError) as excinfo: + db.execute("INSERT INTO t(t) VALUES (?)", [command]) + assert str(excinfo.value) == f"unknown vec0 command: '{command}'" + + +def test_ivf_compute_centroids_options(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=0))") + with pytest.raises( + sqlite3.OperationalError, + match="compute-centroids needs at least one row in vector column 0", + ): + db.execute("""INSERT INTO t(t) VALUES ('compute-centroids:{"nlist": 2}')""") + rows = _random_rows(10, 30, 8) + for rowid, vec in rows.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + with pytest.raises( + sqlite3.OperationalError, + match="compute-centroids needs nlist for vector column 0, which was " + "created with nlist=0", + ): + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + def centroids(table="t"): + return [ + tuple(r) + for r in db.execute( + f"SELECT centroid_id, centroid FROM {table}_ivf_centroids00 ORDER BY 1" + ) + ] + + # the same rows, read in the same order, and the same seed + db.execute("CREATE VIRTUAL TABLE u USING vec0(v float[8] INDEXED BY ivf(nlist=0))") + for rowid, vec in rows.items(): + db.execute("INSERT INTO u(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + for table in ("t", "u"): + db.execute( + f"INSERT INTO {table}({table}) VALUES " + """('compute-centroids:{"nlist": 3, "seed": 5}')""" + ) + assert len(centroids("t")) == 3 + assert centroids("t") == centroids("u") + + # without iterations, the centroids are the rows k-means++ picks + db.execute( + "INSERT INTO t(t) VALUES " + """('compute-centroids:{"nlist": 4, "max_iterations": 0, "seed": 9}')""" + ) + vectors = {_f32(vec) for vec in rows.values()} + assert len(centroids()) == 4 + assert all(c in vectors for _, c in centroids()) + + # at most one centroid per row + db.execute("""INSERT INTO t(t) VALUES ('compute-centroids:{"nlist": 64}')""") + assert len(centroids()) == 30 + assert ivf_assigned_count(db) == 30 + + +def test_ivf_corrupt_cell_errors(db): + db.execute("CREATE VIRTUAL TABLE t USING vec0(v float[8] INDEXED BY ivf(nlist=2))") + rows = _random_rows(2, 10, 8) + for rowid, vec in rows.items(): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32(vec)]) + db.execute("UPDATE t_ivf_rowid_map00 SET slot = 99 WHERE rowid = 3") + with pytest.raises( + sqlite3.OperationalError, + match="^row 3 of the IVF index of vector column 0 is corrupt$", + ): + db.execute("SELECT v FROM t WHERE rowid = 3").fetchall() + db.execute("UPDATE t_ivf_cells00 SET vectors = x'00'") + with pytest.raises(sqlite3.OperationalError, match="cell 1 .* is corrupt"): + db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 3", [_f32(rows[1])] + ).fetchall() + with pytest.raises(sqlite3.OperationalError, match="cell 1 .* is corrupt"): + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + + +def test_ivf_corrupt_vector_errors(db): + """A float vector of the wrong size fails each query that reads it, naming it.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[8] INDEXED BY ivf(nlist=2, quantizer=int8))" + ) + for rowid in (1, 2, 3): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32([rowid / 4] * 8)] + ) + db.execute("UPDATE t_ivf_vectors00 SET vector = x'00' WHERE rowid = 2") + for sql, parameters in [ + ("SELECT v FROM t WHERE rowid = 2", []), + ("SELECT rowid, v FROM t", []), + ("SELECT rowid FROM t WHERE v MATCH ? AND k = 3", [_f32([0.5] * 8)]), + ("INSERT INTO t(t) VALUES ('compute-centroids')", []), + ]: + with pytest.raises( + sqlite3.OperationalError, + match="^the IVF vector of row 2 in vector column 0 is corrupt$", + ): + db.execute(sql, parameters).fetchall() + + +@pytest.mark.parametrize( + "table, quantizer, statements, message", + [ + ( + "t_ivf_rowid_map00", + "none", + ["SELECT v FROM t WHERE rowid = 2", "DELETE FROM t WHERE rowid = 2"], + "could not read row 2 of the IVF index of vector column 0", + ), + ( + "t_ivf_cells00", + "none", + [ + "SELECT rowid FROM t WHERE v MATCH '[1, 1]' AND k = 2", + "INSERT INTO t(rowid, v) VALUES (9, '[1, 1]')", + "INSERT INTO t(t) VALUES ('compute-centroids')", + ], + "could not read the cells of the IVF index of vector column 0", + ), + ( + "t_ivf_vectors00", + "int8", + ["SELECT v FROM t WHERE rowid = 2"], + "could not read the IVF vector of row 2 in vector column 0", + ), + ( + "t_ivf_vectors00", + "int8", + ["INSERT INTO t(t) VALUES ('compute-centroids')"], + "could not read the vectors of the IVF index of vector column 0", + ), + ( + "t_ivf_centroids00", + "none", + [ + "SELECT rowid FROM t WHERE v MATCH '[1, 1]' AND k = 2", + "INSERT INTO t(rowid, v) VALUES (9, '[1, 1]')", + ], + "could not read the centroids of the IVF index of vector column 0", + ), + ( + "t_info", + "none", + ["SELECT rowid FROM t WHERE v MATCH '[1, 1]' AND k = 2"], + "could not read whether vector column 0 has IVF centroids", + ), + ], +) +def test_ivf_unreadable_table_errors(db, table, quantizer, statements, message): + """A failed read names what it read and SQLite's reason.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f"v float[2] INDEXED BY ivf(nlist=2, quantizer={quantizer}))" + ) + for rowid in (1, 2, 3): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32([rowid / 4] * 2)] + ) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + db.execute(f"DROP TABLE {table}") + for sql in statements: + with pytest.raises( + sqlite3.OperationalError, + match=f"^{message}: no such table: main.{table}$", + ): + db.execute(sql).fetchall() + + +@pytest.mark.parametrize( + "table, event, quantizer, setup, sql, message", + [ + ( + "t_ivf_rowid_map00", + "INSERT", + "none", + [], + "INSERT INTO t(rowid, v) VALUES (9, '[1, 1]')", + "could not write row 9 of the IVF index of vector column 0", + ), + ( + "t_ivf_rowid_map00", + "DELETE", + "none", + [], + "DELETE FROM t WHERE rowid = 2", + "could not remove row 2 from the IVF index of vector column 0", + ), + ( + "t_ivf_cells00", + "UPDATE", + "none", + [], + "DELETE FROM t WHERE rowid = 2", + "could not write cell 1 of the IVF index of vector column 0", + ), + ( + "t_ivf_vectors00", + "INSERT", + "int8", + [], + "INSERT INTO t(rowid, v) VALUES (9, '[1, 1]')", + "could not write the IVF vector of row 9 in vector column 0", + ), + ( + "t_ivf_vectors00", + "DELETE", + "int8", + [], + "DELETE FROM t WHERE rowid = 2", + "could not remove the IVF vector of row 2 in vector column 0", + ), + ( + "t_ivf_cells00", + "INSERT", + "none", + [], + "INSERT INTO t(t) VALUES ('compute-centroids')", + "could not add a cell to the IVF index of vector column 0", + ), + ( + "t_ivf_centroids00", + "INSERT", + "none", + [], + "INSERT INTO t(t) VALUES ('compute-centroids')", + "could not write the centroids of the IVF index of vector column 0", + ), + ( + "t_ivf_centroids00", + "INSERT", + "none", + [], + "INSERT INTO t(t, v) VALUES ('set-centroid:0', '[1, 1]')", + "could not write centroid 0 of the IVF index of vector column 0", + ), + ( + "t_ivf_centroids00", + "DELETE", + "none", + ["INSERT INTO t(t) VALUES ('compute-centroids')"], + "INSERT INTO t(t) VALUES ('compute-centroids')", + "could not change t_ivf_centroids00", + ), + ( + "t_info", + "INSERT", + "none", + [], + "INSERT INTO t(t) VALUES ('compute-centroids')", + "could not record whether vector column 0 has IVF centroids", + ), + ( + "t_ivf_cells00", + "DELETE", + "none", + ["INSERT INTO t(t, v) VALUES ('set-centroid:0', '[1, 1]')"], + "INSERT INTO t(t) VALUES ('assign-vectors')", + "could not remove the cells of no centroid from the IVF index of " + "vector column 0", + ), + ], +) +def test_ivf_refused_write_errors(db, table, event, quantizer, setup, sql, message): + """A failed write names what it changed and SQLite's reason.""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + f"v float[2] INDEXED BY ivf(nlist=2, quantizer={quantizer}))" + ) + for rowid in (1, 2, 3): + db.execute( + "INSERT INTO t(rowid, v) VALUES (?, ?)", [rowid, _f32([rowid / 4] * 2)] + ) + for statement in setup: + db.execute(statement) + db.execute(f""" + CREATE TRIGGER refuse BEFORE {event} ON {table} + BEGIN SELECT RAISE(ABORT, 'refused'); END + """) + with pytest.raises(sqlite3.OperationalError, match=f"^{message}: refused$"): + db.execute(sql) + + +@pytest.mark.parametrize("options", ["nlist=2", "nlist=2, quantizer=int8"]) +def test_ivf_insert_or_replace(db, options): + db.execute( + f"CREATE VIRTUAL TABLE t USING vec0(v float[4] INDEXED BY ivf({options}), " + "+a text)" + ) + db.execute("INSERT INTO t(rowid, v, a) VALUES (1, ?, 'x')", [_f32([1, 0, 0, 0])]) + db.execute("INSERT INTO t(rowid, v, a) VALUES (2, ?, 'y')", [_f32([0, 1, 0, 0])]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + db.execute( + "INSERT OR REPLACE INTO t(rowid, v, a) VALUES (1, ?, 'z')", + [_f32([0, 0, 1, 0])], + ) + rows = db.execute("SELECT rowid, v, a FROM t ORDER BY rowid") + assert [tuple(r) for r in rows] == [ + (1, _f32([0, 0, 1, 0]), "z"), + (2, _f32([0, 1, 0, 0]), "y"), + ] + got = db.execute( + "SELECT rowid, distance FROM t WHERE v MATCH ? AND k = 2", [_f32([0, 0, 1, 0])] + ).fetchall() + assert [tuple(r) for r in got] == [(1, 0.0), (2, pytest.approx(2**0.5))] + assert ivf_total_vectors(db) == 2 + + +def test_ivf_vacuum_after_knn(db): + """A KNN query leaves no statement in progress, so VACUUM works after it. + https://github.com/asg017/sqlite-vec/issues/295""" + db.execute( + "CREATE VIRTUAL TABLE t USING vec0(" + "v float[8] INDEXED BY ivf(nlist=2, quantizer=binary, oversample=2))" + ) + for i in range(1, 11): + db.execute("INSERT INTO t(rowid, v) VALUES (?, ?)", [i, _f32([0.1 * i] * 8)]) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + db.commit() + rows = db.execute( + "SELECT rowid FROM t WHERE v MATCH ? AND k = 3", [_f32([0.5] * 8)] + ).fetchall() + assert len(rows) == 3 + db.execute("VACUUM") + + +def test_ivf_optimize_beside_a_flat_column(db): + """optimize moves a flat column's vectors and leaves an IVF column's.""" + db.execute(""" + CREATE VIRTUAL TABLE t USING vec0( + f float[8], + i float[8] INDEXED BY ivf(nlist=3, quantizer=int8, oversample=8), + chunk_size=8 + ) + """) + rows = _random_rows(13, 40, 8) + for rowid, vec in rows.items(): + db.execute( + "INSERT INTO t(rowid, f, i) VALUES (?, ?, ?)", + [rowid, _f32(vec), _f32([-x for x in vec])], + ) + db.execute("INSERT INTO t(t) VALUES ('compute-centroids')") + for rowid in range(1, 41, 3): + db.execute("DELETE FROM t WHERE rowid = ?", [rowid]) + del rows[rowid] + db.execute("INSERT INTO t(t) VALUES ('optimize')") + for rowid, f, i in db.execute("SELECT rowid, f, i FROM t").fetchall(): + assert f == _f32(rows[rowid]) + assert i == _f32([-x for x in rows[rowid]]) + for rowid, vec in rows.items(): + for column, value in (("f", vec), ("i", [-x for x in vec])): + got = db.execute( + f"SELECT rowid FROM t WHERE {column} MATCH ? AND k = 1", [_f32(value)] + ).fetchone()[0] + assert got == rowid diff --git a/tests/test-rename.py b/tests/test-rename.py index 4d6cfabb..ea4f3921 100644 --- a/tests/test-rename.py +++ b/tests/test-rename.py @@ -1,7 +1,7 @@ """ALTER TABLE ... RENAME on vec0 tables. -Ported from asg017/sqlite-vec@c560710, @8105eee and @8b81f40, without the tests -for upstream's IVF index, which this fork does not have. +Ported from asg017/sqlite-vec@c560710, @8105eee and @8b81f40. The IVF tests of +@8b81f40 are in test-ivf-rename.py, which builds without IVF skip. """ import sqlite3 diff --git a/tests/test-unit.c b/tests/test-unit.c index caa9733b..08e517cf 100644 --- a/tests/test-unit.c +++ b/tests/test-unit.c @@ -1560,6 +1560,328 @@ void test_diskann_search_list_size_allocation() { #endif /* SQLITE_VEC_ENABLE_DISKANN */ +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + +void test_vec0_parse_vector_column_ivf() { + printf("Starting %s...\n", __func__); + struct VectorColumnDefinition col; + int rc; + + // Every default + { + const char *input = "v float[4] indexed by ivf()"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.index_type == VEC0_INDEX_TYPE_IVF); + assert(col.dimensions == 4); + assert(col.ivf.nlist == 128); + assert(col.ivf.nprobe == 10); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_NONE); + assert(col.ivf.oversample == 1); + sqlite3_free(col.name); + } + + // nprobe defaults to at most nlist + { + const char *input = "v float[4] indexed by ivf(nlist=8)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 8); + assert(col.ivf.nprobe == 8); + sqlite3_free(col.name); + } + + { + const char *input = "v float[4] indexed by ivf(nlist=64, nprobe=8)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 64); + assert(col.ivf.nprobe == 8); + sqlite3_free(col.name); + } + + // distance_metric before indexed by + { + const char *input = + "v float[4] distance_metric=cosine indexed by ivf(nlist=16)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.index_type == VEC0_INDEX_TYPE_IVF); + assert(col.distance_metric == VEC0_DISTANCE_METRIC_COSINE); + assert(col.ivf.nlist == 16); + sqlite3_free(col.name); + } + + // nlist=0 leaves the centroids to set-centroid + { + const char *input = "v float[4] indexed by ivf(nlist=0)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 0); + sqlite3_free(col.name); + } + + // nlist at its maximum + { + const char *input = "v float[4] indexed by ivf(nlist=65536)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 65536); + sqlite3_free(col.name); + } + + // Quantizers + { + const char *input = + "v float[768] indexed by ivf(nlist=128, quantizer=binary)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 128); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY); + assert(col.ivf.oversample == 1); + sqlite3_free(col.name); + } + + { + const char *input = "v float[768] indexed by ivf(nlist=64, quantizer=int8)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8); + sqlite3_free(col.name); + } + + { + const char *input = "v float[768] indexed by ivf(quantizer=none)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_NONE); + sqlite3_free(col.name); + } + + // oversample with a quantizer + { + const char *input = "v float[768] indexed by ivf(nlist=128, " + "quantizer=binary, oversample=10)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY); + assert(col.ivf.oversample == 10); + assert(col.ivf.nlist == 128); + sqlite3_free(col.name); + } + + // Every option, keywords in any case + { + const char *input = "v float[768] distance_metric=cosine INDEXED BY " + "IVF(NLIST=256, NPROBE=16, QUANTIZER=INT8, " + "OVERSAMPLE=4)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.distance_metric == VEC0_DISTANCE_METRIC_COSINE); + assert(col.ivf.nlist == 256); + assert(col.ivf.nprobe == 16); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8); + assert(col.ivf.oversample == 4); + sqlite3_free(col.name); + } + + { + const char *input = + "v float[768] indexed by ivf(quantizer=binary, oversample=5)"; + rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_OK); + assert(col.ivf.nlist == 128); + assert(col.ivf.nprobe == 10); + assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY); + assert(col.ivf.oversample == 5); + sqlite3_free(col.name); + } + + const char *errors[] = { + // an explicit nprobe past nlist + "v float[4] indexed by ivf(nlist=4, nprobe=10)", + "v float[4] indexed by ivf(bogus=1)", + "v float[4] indexed by hnsw()", + "v float[4] indexed by ivf(nlist=65537)", + "v float[4] indexed by ivf(nlist=-1)", + "v float[4] indexed by ivf(nprobe=0)", + // oversample only ranks a quantizer's candidates + "v float[768] indexed by ivf(oversample=10)", + "v float[768] indexed by ivf(quantizer=binary, oversample=0)", + "v float[768] indexed by ivf(quantizer=pq)", + // the binary quantizer packs 8 dimensions to a byte + "v float[12] indexed by ivf(quantizer=binary)", + // only float32 columns + "v int8[8] indexed by ivf()", + "v bit[8] indexed by ivf()", + // whole keywords and values, all digits, no trailing comma + "v float[8] indexed by ivf(quantizer=bin)", + "v float[8] indexed by ivf(nli=8)", + "v float[8] indexed by ivf(nlist=8x)", + "v float[8] indexed by ivf(nlist=8,)", + "v float[8] indexed by ivf(quantizer=int8, oversample=99999999999)", + "v float[8] indexed by ivf", + "v float[8] indexed by ivf(", + }; + for (size_t i = 0; i < countof(errors); i++) { + rc = vec0_parse_vector_column(errors[i], (int)strlen(errors[i]), &col, + true); + assert(rc == SQLITE_ERROR); + } + + printf(" All vec0_parse_vector_column_ivf tests passed.\n"); +} + +void test_ivf_quantize_int8() { + printf("Starting %s...\n", __func__); + + // Values in [-1, 1] scale to [-127, 127], truncating toward zero + { + float src[] = {0.0f, 1.0f, -1.0f, 0.5f}; + int8_t dst[4]; + ivf_quantize_int8(src, dst, 4); + assert(dst[0] == 0); + assert(dst[1] == 127); + assert(dst[2] == -127); + assert(dst[3] == 63); + } + + // Values past [-1, 1] clamp, and NaN is 127 + { + float src[] = {2.0f, -3.0f, 100.0f, -0.01f, NAN, INFINITY, -INFINITY}; + int8_t dst[7]; + ivf_quantize_int8(src, dst, 7); + assert(dst[0] == 127); + assert(dst[1] == -127); + assert(dst[2] == 127); + assert(dst[3] == (int8_t)(-0.01f * 127.0f)); + assert(dst[4] == 127); + assert(dst[5] == 127); + assert(dst[6] == -127); + } + + { + float src[] = {0.0f, -0.0f, 0.75f}; + int8_t dst[3]; + ivf_quantize_int8(src, dst, 3); + assert(dst[0] == 0); + assert(dst[1] == 0); + assert(dst[2] == (int8_t)(0.75f * 127.0f)); + } + + printf(" All ivf_quantize_int8 tests passed.\n"); +} + +void test_ivf_quantize_binary() { + printf("Starting %s...\n", __func__); + + // A value above 0 sets its bit, the first value in the lowest bit + { + float src[] = {1.0f, -1.0f, 0.5f, -0.5f, 0.0f, 0.1f, -0.1f, 2.0f}; + uint8_t dst[1]; + ivf_quantize_binary(src, dst, 8); + assert(dst[0] == 0xA5); + } + + { + float src[] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}; + uint8_t dst[1]; + ivf_quantize_binary(src, dst, 8); + assert(dst[0] == 0xFF); + } + + { + float src[] = {-1.0f, -2.0f, 0.0f, -0.0f, -5.0f, -6.0f, -7.0f, NAN}; + uint8_t dst[1] = {0xFF}; + ivf_quantize_binary(src, dst, 8); + assert(dst[0] == 0x00); + } + + { + float src[16]; + for (int i = 0; i < 16; i++) { + src[i] = (i % 2 == 0) ? 1.0f : -1.0f; + } + uint8_t dst[2]; + ivf_quantize_binary(src, dst, 16); + assert(dst[0] == 0x55); + assert(dst[1] == 0x55); + } + + printf(" All ivf_quantize_binary tests passed.\n"); +} + +void test_ivf_kmeans() { + printf("Starting %s...\n", __func__); + + // Three clusters of 2-dimensional points around (0, 0), (10, 0), (0, 10) + float vectors[30 * 2]; + for (int i = 0; i < 30; i++) { + float offset = (float)(i % 10) / 10.0f; + vectors[i * 2] = (i / 10 == 1 ? 10.0f : 0.0f) + offset; + vectors[i * 2 + 1] = (i / 10 == 2 ? 10.0f : 0.0f) - offset; + } + + float centroids[3 * 2]; + assert(ivf_kmeans(vectors, 30, 2, 3, 25, 0, centroids) == SQLITE_OK); + int found[3] = {0, 0, 0}; + for (int c = 0; c < 3; c++) { + float x = centroids[c * 2], y = centroids[c * 2 + 1]; + // each cluster's mean is its corner moved by (0.45, -0.45) + if (fabsf(x - 0.45f) < 1e-4 && fabsf(y + 0.45f) < 1e-4) { + found[0]++; + } else if (fabsf(x - 10.45f) < 1e-4 && fabsf(y + 0.45f) < 1e-4) { + found[1]++; + } else if (fabsf(x - 0.45f) < 1e-4 && fabsf(y - 9.55f) < 1e-4) { + found[2]++; + } + } + assert(found[0] == 1 && found[1] == 1 && found[2] == 1); + + // The same seed gives the same centroids + { + float a[3 * 2], b[3 * 2]; + assert(ivf_kmeans(vectors, 30, 2, 3, 25, 7, a) == SQLITE_OK); + assert(ivf_kmeans(vectors, 30, 2, 3, 25, 7, b) == SQLITE_OK); + assert(memcmp(a, b, sizeof(a)) == 0); + } + + // As many centroids as vectors: each vector is one + { + float few[3 * 2] = {1.0f, 1.0f, 5.0f, 5.0f, 9.0f, 9.0f}; + float out[3 * 2]; + assert(ivf_kmeans(few, 3, 2, 3, 25, 0, out) == SQLITE_OK); + for (int v = 0; v < 3; v++) { + int matches = 0; + for (int c = 0; c < 3; c++) { + matches += out[c * 2] == few[v * 2] && out[c * 2 + 1] == few[v * 2 + 1]; + } + assert(matches == 1); + } + } + + // More centroids than vectors, or none + assert(ivf_kmeans(vectors, 2, 2, 3, 25, 0, centroids) == SQLITE_MISUSE); + assert(ivf_kmeans(vectors, 30, 2, 0, 25, 0, centroids) == SQLITE_MISUSE); + assert(ivf_kmeans(vectors, 0, 2, 1, 25, 0, centroids) == SQLITE_MISUSE); + + printf(" All ivf_kmeans tests passed.\n"); +} + +#else + +void test_vec0_parse_vector_column_ivf() { + printf("Starting %s...\n", __func__); + struct VectorColumnDefinition col; + // a build without the IVF index has no ivf() to parse + const char *input = "v float[4] indexed by ivf()"; + int rc = vec0_parse_vector_column(input, (int)strlen(input), &col, true); + assert(rc == SQLITE_ERROR); + printf(" All vec0_parse_vector_column_ivf tests passed.\n"); +} + +#endif /* SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE */ + int main() { printf("Starting unit tests...\n"); test_vec0_token_next(); @@ -1586,6 +1908,12 @@ int main() { test_diskann_candidate_list_operations(); test_diskann_visited_set_operations(); test_diskann_search_list_size_allocation(); +#endif + test_vec0_parse_vector_column_ivf(); +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE + test_ivf_quantize_int8(); + test_ivf_quantize_binary(); + test_ivf_kmeans(); #endif printf("All unit tests passed.\n"); }