snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-22 18:17:39 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

with binary quantization

Browse Source
This commit is contained in:
Tamo 2024-07-08 20:12:56 +02:00
parent 05cc2d1fac
commit 42bfc67871
4 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Cargo.lock generated
View File

@ -398,6 +398,24 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "arroy"
version = "0.4.0"
source = "git+https://github.com/meilisearch/arroy?branch=binary-quantization#facc8575222d3f5da5b9a94288e44896911e701f"
dependencies = [
"bytemuck",
"byteorder",
"heed",
"log",
"memmap2 0.9.4",
"ordered-float",
"rand",
"rayon",
"roaring",
"tempfile",
"thiserror",
]
[[package]] [[package]]
name = "assert-json-diff" name = "assert-json-diff"
version = "2.0.2" version = "2.0.2"
@ -2455,7 +2473,7 @@ name = "index-scheduler"
version = "1.9.0" version = "1.9.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"arroy", "arroy 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"big_s", "big_s",
"bincode", "bincode",
"crossbeam", "crossbeam",
@ -3448,7 +3466,7 @@ dependencies = [
name = "milli" name = "milli"
version = "1.9.0" version = "1.9.0"
dependencies = [ dependencies = [
"arroy", "arroy 0.4.0 (git+https://github.com/meilisearch/arroy?branch=binary-quantization)",
"big_s", "big_s",
"bimap", "bimap",
"bincode", "bincode",

2
milli/Cargo.toml
View File

@ -79,7 +79,7 @@ hf-hub = { git = "https://github.com/dureuill/hf-hub.git", branch = "rust_tls",
] } ] }
tiktoken-rs = "0.5.8" tiktoken-rs = "0.5.8"
liquid = "0.26.4" liquid = "0.26.4"
arroy = "0.4.0" arroy = { git = "https://github.com/meilisearch/arroy", branch = "binary-quantization" }
rand = "0.8.5" rand = "0.8.5"
tracing = "0.1.40" tracing = "0.1.40"
ureq = { version = "2.9.7", features = ["json"] } ureq = { version = "2.9.7", features = ["json"] }

5
milli/src/index.rs
View File

@ -169,7 +169,7 @@ pub struct Index {
/// Maps an embedder name to its id in the arroy store. /// Maps an embedder name to its id in the arroy store.
pub embedder_category_id: Database<Str, U8>, pub embedder_category_id: Database<Str, U8>,
/// Vector store based on arroy™. /// Vector store based on arroy™.
pub vector_arroy: arroy::Database<arroy::distances::Angular>, pub vector_arroy: arroy::Database<arroy::distances::BinaryQuantizedEuclidean>,
/// Maps the document id to the document as an obkv store. /// Maps the document id to the document as an obkv store.
pub(crate) documents: Database<BEU32, ObkvCodec>, pub(crate) documents: Database<BEU32, ObkvCodec>,
@ -1605,7 +1605,8 @@ impl Index {
&'a self, &'a self,
rtxn: &'a RoTxn<'a>, rtxn: &'a RoTxn<'a>,
embedder_id: u8, embedder_id: u8,
) -> impl Iterator<Item = Result<arroy::Reader<arroy::distances::Angular>>> + 'a { ) -> impl Iterator<Item = Result<arroy::Reader<arroy::distances::BinaryQuantizedEuclidean>>> + 'a
{
crate::vector::arroy_db_range_for_embedder(embedder_id).map_while(move |k| { crate::vector::arroy_db_range_for_embedder(embedder_id).map_while(move |k| {
arroy::Reader::open(rtxn, k, self.vector_arroy) arroy::Reader::open(rtxn, k, self.vector_arroy)
.map(Some) .map(Some)

5
milli/src/update/index_documents/transform.rs
View File

@ -990,7 +990,10 @@ impl<'a, 'i> Transform<'a, 'i> {
}; };
let readers: Result< let readers: Result<
BTreeMap<&str, (Vec<arroy::Reader<arroy::distances::Angular>>, &RoaringBitmap)>, BTreeMap<
&str,
(Vec<arroy::Reader<arroy::distances::BinaryQuantizedEuclidean>>, &RoaringBitmap),
>,
> = settings_diff > = settings_diff
.embedding_config_updates .embedding_config_updates
.iter() .iter()