From 86d5e6d9ff22969980b553e37bd1baed052dc7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 25 Sep 2024 14:54:56 +0200 Subject: [PATCH] Use the new Lru --- Cargo.lock | 10 ---------- Cargo.toml | 21 --------------------- milli/Cargo.toml | 1 - milli/src/update/new/extract/cache.rs | 8 ++++---- milli/src/update/new/{ => extract}/lru.rs | 4 ++++ milli/src/update/new/extract/mod.rs | 8 +++----- milli/src/update/new/mod.rs | 1 - 7 files changed, 11 insertions(+), 42 deletions(-) rename milli/src/update/new/{ => extract}/lru.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 7b3de4a6a..06bd9c234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3312,15 +3312,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "lru" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" -dependencies = [ - "hashbrown 0.14.5", -] - [[package]] name = "lzma-rs" version = "0.3.0" @@ -3584,7 +3575,6 @@ dependencies = [ "json-depth-checker", "levenshtein_automata", "liquid", - "lru", "maplit", "md5", "meili-snap", diff --git a/Cargo.toml b/Cargo.toml index a73ac67ef..1d25b9795 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,26 +44,5 @@ opt-level = 3 [profile.dev.package.roaring] opt-level = 3 -[profile.dev.package.lindera-ipadic-builder] -opt-level = 3 -[profile.dev.package.encoding] -opt-level = 3 -[profile.dev.package.yada] -opt-level = 3 - -[profile.release.package.lindera-ipadic-builder] -opt-level = 3 -[profile.release.package.encoding] -opt-level = 3 -[profile.release.package.yada] -opt-level = 3 - -[profile.bench.package.lindera-ipadic-builder] -opt-level = 3 -[profile.bench.package.encoding] -opt-level = 3 -[profile.bench.package.yada] -opt-level = 3 - [patch.crates-io] roaring = { git = "https://github.com/RoaringBitmap/roaring-rs", branch = "clone-iter-slice" } diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 19986de01..bae3dd64b 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -40,7 +40,6 @@ heed = { version = "0.20.3", default-features = false, features = [ indexmap = { version = "2.2.6", features = ["serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } -lru = "0.12.3" memchr = "2.5.0" memmap2 = "0.9.4" obkv = { git = "https://github.com/kerollmops/obkv", branch = "unsized-kvreader" } diff --git a/milli/src/update/new/extract/cache.rs b/milli/src/update/new/extract/cache.rs index 572c81a55..4f6f30e70 100644 --- a/milli/src/update/new/extract/cache.rs +++ b/milli/src/update/new/extract/cache.rs @@ -2,16 +2,16 @@ use std::mem; use std::num::NonZeroUsize; use grenad::{MergeFunction, Sorter}; -use lru::LruCache; use roaring::RoaringBitmap; use smallvec::SmallVec; +use super::lru::Lru; use crate::update::del_add::{DelAdd, KvWriterDelAdd}; use crate::CboRoaringBitmapCodec; #[derive(Debug)] pub struct CboCachedSorter { - cache: lru::LruCache, DelAddRoaringBitmap>, + cache: Lru, DelAddRoaringBitmap>, sorter: Sorter, deladd_buffer: Vec, cbo_buffer: Vec, @@ -22,7 +22,7 @@ pub struct CboCachedSorter { impl CboCachedSorter { pub fn new(cap: NonZeroUsize, sorter: Sorter) -> Self { CboCachedSorter { - cache: lru::LruCache::new(cap), + cache: Lru::new(cap), sorter, deladd_buffer: Vec::new(), cbo_buffer: Vec::new(), @@ -171,7 +171,7 @@ impl CboCachedSorter { } pub fn into_sorter(mut self) -> grenad::Result, MF::Error> { - let default_arc = LruCache::new(NonZeroUsize::MIN); + let default_arc = Lru::new(NonZeroUsize::MIN); for (key, deladd) in mem::replace(&mut self.cache, default_arc) { self.write_entry(key, deladd)?; } diff --git a/milli/src/update/new/lru.rs b/milli/src/update/new/extract/lru.rs similarity index 99% rename from milli/src/update/new/lru.rs rename to milli/src/update/new/extract/lru.rs index fef108753..7c13d9350 100644 --- a/milli/src/update/new/lru.rs +++ b/milli/src/update/new/extract/lru.rs @@ -7,6 +7,7 @@ use std::num::NonZeroUsize; use hashbrown::hash_map::{DefaultHashBuilder, Entry}; use hashbrown::HashMap; +#[derive(Debug)] pub struct Lru { lookup: HashMap, storage: FixedSizeList>, @@ -104,17 +105,20 @@ impl Iterator for IntoIter { } } +#[derive(Debug)] struct LruNode { key: K, value: V, } +#[derive(Debug)] struct FixedSizeListNode { prev: usize, next: usize, data: T, } +#[derive(Debug)] struct FixedSizeList { nodes: Box<[Option>]>, // An un-ordered set of indices that are not in use in `nodes`. diff --git a/milli/src/update/new/extract/mod.rs b/milli/src/update/new/extract/mod.rs index 3836f9957..d1f6bb787 100644 --- a/milli/src/update/new/extract/mod.rs +++ b/milli/src/update/new/extract/mod.rs @@ -1,5 +1,6 @@ mod cache; mod faceted; +mod lru; mod searchable; use std::fs::File; @@ -9,12 +10,9 @@ use grenad::Merger; use rayon::iter::IntoParallelIterator; pub use searchable::*; -use crate::{ - update::{GrenadParameters, MergeDeladdCboRoaringBitmaps}, - GlobalFieldsIdsMap, Index, Result, -}; - use super::DocumentChange; +use crate::update::{GrenadParameters, MergeDeladdCboRoaringBitmaps}; +use crate::{GlobalFieldsIdsMap, Index, Result}; pub trait DocidsExtractor { fn run_extraction( diff --git a/milli/src/update/new/mod.rs b/milli/src/update/new/mod.rs index b4878a8fe..dedd89497 100644 --- a/milli/src/update/new/mod.rs +++ b/milli/src/update/new/mod.rs @@ -10,7 +10,6 @@ mod document_change; mod extract; pub mod indexer; mod items_pool; -mod lru; mod merger; mod top_level_map; mod word_fst_builder;