snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-30 09:04:59 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Make PathSet strongly typed

Browse Source
This commit is contained in:
Loïc Lecrenier 2023-03-16 09:58:59 +01:00
parent a49ddec9df
commit 7b1d8f4c6d
12 changed files with 102 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
milli/src/search/new/graph_based_ranking_rule.rs
View File

@ -50,7 +50,6 @@ use super::ranking_rule_graph::{
}; };
use super::small_bitmap::SmallBitmap; use super::small_bitmap::SmallBitmap;
use super::{QueryGraph, RankingRule, RankingRuleOutput, SearchContext}; use super::{QueryGraph, RankingRule, RankingRuleOutput, SearchContext};
use crate::search::new::interner::Interned;
use crate::search::new::query_graph::QueryNodeData; use crate::search::new::query_graph::QueryNodeData;
use crate::Result; use crate::Result;
@ -247,9 +246,8 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
let mut cached_edge_docids = vec![]; let mut cached_edge_docids = vec![];
// graph.conditions_interner.map(|_| RoaringBitmap::new()); // graph.conditions_interner.map(|_| RoaringBitmap::new());
for &condition_interned_raw in path { for &condition in path {
let condition = Interned::new(condition_interned_raw); visited_conditions.push(condition);
visited_conditions.push(condition_interned_raw);
let edge_docids = let edge_docids =
edge_docids_cache.get_edge_docids(ctx, condition, graph, &universe)?; edge_docids_cache.get_edge_docids(ctx, condition, graph, &universe)?;
@ -295,7 +293,7 @@ impl<'ctx, G: RankingRuleGraphTrait> RankingRule<'ctx, QueryGraph> for GraphBase
} }
assert!(!path_docids.is_empty()); assert!(!path_docids.is_empty());
for condition in path { for condition in path {
used_conditions.insert(Interned::new(*condition)); used_conditions.insert(*condition);
} }
bucket |= &path_docids; bucket |= &path_docids;
// Reduce the size of the universe so that we can more optimistically discard candidate paths // Reduce the size of the universe so that we can more optimistically discard candidate paths

42
milli/src/search/new/interner.rs
View File

@ -4,38 +4,27 @@ use std::marker::PhantomData;
use fxhash::FxHashMap; use fxhash::FxHashMap;
/// An index within a [`Interner<T>`] structure. /// An index within an interner ([`FixedSizeInterner`], [`DedupInterner`], or [`MappedInterner`]).
pub struct Interned<T> { pub struct Interned<T> {
idx: u16, idx: u16,
_phantom: PhantomData<T>, _phantom: PhantomData<T>,
} }
impl<T> Interned<T> { impl<T> Interned<T> {
pub fn new(idx: u16) -> Self { /// Create an interned value manually from its raw index within the interner.
pub fn from_raw(idx: u16) -> Self {
Self { idx, _phantom: PhantomData } Self { idx, _phantom: PhantomData }
} }
pub fn into_inner(self) -> u16 { /// Get the raw index from the interned value
pub fn into_raw(self) -> u16 {
self.idx self.idx
} }
} }
// TODO: the stable store should be replaced by a bump allocator /// A [`DedupInterner`] is used to store a unique copy of a value of type `T`. This value
// and the interned value should be a pointer wrapper
// then we can get its value with `interned.get()` instead of `interner.get(interned)`
// and as a bonus, its validity is tracked with Rust's lifetime system
// one problem is that we need two lifetimes: one for the bump allocator, one for the
// hashmap
// but that's okay, we can use:
// ```
// struct Interner<'bump> {
// bump: &'bump Bump,
// lookup: FxHashMap
// }
// ```
/// An [`Interner`] is used to store a unique copy of a value of type `T`. This value
/// is then identified by a lightweight index of type [`Interned<T>`], which can /// is then identified by a lightweight index of type [`Interned<T>`], which can
/// be copied, compared, and hashed efficiently. An immutable reference to the original value /// be copied, compared, and hashed efficiently. An immutable reference to the original value
/// can be retrieved using `self.get(interned)`. /// can be retrieved using `self.get(interned)`. A set of values within the interner can be
/// efficiently managed using [`SmallBitmap<T>`](super::small_bitmap::SmallBitmap).
#[derive(Clone)] #[derive(Clone)]
pub struct DedupInterner<T> { pub struct DedupInterner<T> {
stable_store: Vec<T>, stable_store: Vec<T>,
@ -47,6 +36,7 @@ impl<T> Default for DedupInterner<T> {
} }
} }
impl<T> DedupInterner<T> { impl<T> DedupInterner<T> {
///
pub fn freeze(self) -> FixedSizeInterner<T> { pub fn freeze(self) -> FixedSizeInterner<T> {
FixedSizeInterner { stable_store: self.stable_store } FixedSizeInterner { stable_store: self.stable_store }
} }
@ -62,7 +52,7 @@ where
} else { } else {
assert!(self.stable_store.len() < u16::MAX as usize); assert!(self.stable_store.len() < u16::MAX as usize);
self.stable_store.push(s.clone()); self.stable_store.push(s.clone());
let interned = Interned::new(self.stable_store.len() as u16 - 1); let interned = Interned::from_raw(self.stable_store.len() as u16 - 1);
self.lookup.insert(s, interned); self.lookup.insert(s, interned);
interned interned
} }
@ -87,7 +77,7 @@ impl<T> Interner<T> {
pub fn push(&mut self, s: T) -> Interned<T> { pub fn push(&mut self, s: T) -> Interned<T> {
assert!(self.stable_store.len() < u16::MAX as usize); assert!(self.stable_store.len() < u16::MAX as usize);
self.stable_store.push(s); self.stable_store.push(s);
Interned::new(self.stable_store.len() as u16 - 1) Interned::from_raw(self.stable_store.len() as u16 - 1)
} }
} }
@ -123,13 +113,13 @@ impl<T> FixedSizeInterner<T> {
} }
} }
pub fn indexes(&self) -> impl Iterator<Item = Interned<T>> { pub fn indexes(&self) -> impl Iterator<Item = Interned<T>> {
(0..self.stable_store.len()).map(|i| Interned::new(i as u16)) (0..self.stable_store.len()).map(|i| Interned::from_raw(i as u16))
} }
pub fn iter(&self) -> impl Iterator<Item = (Interned<T>, &T)> { pub fn iter(&self) -> impl Iterator<Item = (Interned<T>, &T)> {
self.stable_store.iter().enumerate().map(|(i, x)| (Interned::new(i as u16), x)) self.stable_store.iter().enumerate().map(|(i, x)| (Interned::from_raw(i as u16), x))
} }
pub fn iter_mut(&mut self) -> impl Iterator<Item = (Interned<T>, &mut T)> { pub fn iter_mut(&mut self) -> impl Iterator<Item = (Interned<T>, &mut T)> {
self.stable_store.iter_mut().enumerate().map(|(i, x)| (Interned::new(i as u16), x)) self.stable_store.iter_mut().enumerate().map(|(i, x)| (Interned::from_raw(i as u16), x))
} }
} }
#[derive(Clone)] #[derive(Clone)]
@ -152,10 +142,10 @@ impl<T, From> MappedInterner<T, From> {
} }
} }
pub fn iter(&self) -> impl Iterator<Item = (Interned<From>, &T)> { pub fn iter(&self) -> impl Iterator<Item = (Interned<From>, &T)> {
self.stable_store.iter().enumerate().map(|(i, x)| (Interned::new(i as u16), x)) self.stable_store.iter().enumerate().map(|(i, x)| (Interned::from_raw(i as u16), x))
} }
pub fn iter_mut(&mut self) -> impl Iterator<Item = (Interned<From>, &mut T)> { pub fn iter_mut(&mut self) -> impl Iterator<Item = (Interned<From>, &mut T)> {
self.stable_store.iter_mut().enumerate().map(|(i, x)| (Interned::new(i as u16), x)) self.stable_store.iter_mut().enumerate().map(|(i, x)| (Interned::from_raw(i as u16), x))
} }
} }
// Interned<T> boilerplate implementations // Interned<T> boilerplate implementations

14
milli/src/search/new/logger/detailed.rs
View File

@ -43,7 +43,7 @@ pub enum SearchEvents {
}, },
ProximityState { ProximityState {
graph: RankingRuleGraph<ProximityGraph>, graph: RankingRuleGraph<ProximityGraph>,
paths: Vec<Vec<u16>>, paths: Vec<Vec<Interned<ProximityCondition>>>,
empty_paths_cache: DeadEndPathCache<ProximityGraph>, empty_paths_cache: DeadEndPathCache<ProximityGraph>,
universe: RoaringBitmap, universe: RoaringBitmap,
distances: MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>, distances: MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,
@ -51,7 +51,7 @@ pub enum SearchEvents {
}, },
TypoState { TypoState {
graph: RankingRuleGraph<TypoGraph>, graph: RankingRuleGraph<TypoGraph>,
paths: Vec<Vec<u16>>, paths: Vec<Vec<Interned<TypoEdge>>>,
empty_paths_cache: DeadEndPathCache<TypoGraph>, empty_paths_cache: DeadEndPathCache<TypoGraph>,
universe: RoaringBitmap, universe: RoaringBitmap,
distances: MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>, distances: MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>,
@ -169,7 +169,7 @@ impl SearchLogger<QueryGraph> for DetailedSearchLogger {
fn log_proximity_state( fn log_proximity_state(
&mut self, &mut self,
query_graph: &RankingRuleGraph<ProximityGraph>, query_graph: &RankingRuleGraph<ProximityGraph>,
paths_map: &[Vec<u16>], paths_map: &[Vec<Interned<ProximityCondition>>],
empty_paths_cache: &DeadEndPathCache<ProximityGraph>, empty_paths_cache: &DeadEndPathCache<ProximityGraph>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,
@ -188,7 +188,7 @@ impl SearchLogger<QueryGraph> for DetailedSearchLogger {
fn log_typo_state( fn log_typo_state(
&mut self, &mut self,
query_graph: &RankingRuleGraph<TypoGraph>, query_graph: &RankingRuleGraph<TypoGraph>,
paths_map: &[Vec<u16>], paths_map: &[Vec<Interned<TypoEdge>>],
empty_paths_cache: &DeadEndPathCache<TypoGraph>, empty_paths_cache: &DeadEndPathCache<TypoGraph>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>,
@ -527,7 +527,7 @@ shape: class"
fn ranking_rule_graph_d2_description<R: RankingRuleGraphTrait>( fn ranking_rule_graph_d2_description<R: RankingRuleGraphTrait>(
ctx: &mut SearchContext, ctx: &mut SearchContext,
graph: &RankingRuleGraph<R>, graph: &RankingRuleGraph<R>,
paths: &[Vec<u16>], paths: &[Vec<Interned<R::EdgeCondition>>],
dead_end_paths_cache: &DeadEndPathCache<R>, dead_end_paths_cache: &DeadEndPathCache<R>,
distances: MappedInterner<Vec<(u16, SmallBitmap<R::EdgeCondition>)>, QueryNode>, distances: MappedInterner<Vec<(u16, SmallBitmap<R::EdgeCondition>)>, QueryNode>,
file: &mut File, file: &mut File,
@ -613,13 +613,13 @@ shape: class
fn paths_d2_description<R: RankingRuleGraphTrait>( fn paths_d2_description<R: RankingRuleGraphTrait>(
ctx: &mut SearchContext, ctx: &mut SearchContext,
graph: &RankingRuleGraph<R>, graph: &RankingRuleGraph<R>,
paths: &[Vec<u16>], paths: &[Vec<Interned<R::EdgeCondition>>],
file: &mut File, file: &mut File,
) { ) {
for (path_idx, condition_indexes) in paths.iter().enumerate() { for (path_idx, condition_indexes) in paths.iter().enumerate() {
writeln!(file, "{path_idx} {{").unwrap(); writeln!(file, "{path_idx} {{").unwrap();
for condition in condition_indexes.iter() { for condition in condition_indexes.iter() {
Self::condition_d2_description(ctx, graph, Interned::new(*condition), file); Self::condition_d2_description(ctx, graph, *condition, file);
} }
for couple_edges in condition_indexes.windows(2) { for couple_edges in condition_indexes.windows(2) {
let [src_edge_idx, dest_edge_idx] = couple_edges else { panic!() }; let [src_edge_idx, dest_edge_idx] = couple_edges else { panic!() };

10
milli/src/search/new/logger/mod.rs
View File

@ -3,7 +3,7 @@ pub mod detailed;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use super::interner::MappedInterner; use super::interner::{Interned, MappedInterner};
use super::query_graph::QueryNode; use super::query_graph::QueryNode;
use super::ranking_rule_graph::{ use super::ranking_rule_graph::{
DeadEndPathCache, ProximityCondition, ProximityGraph, RankingRuleGraph, TypoEdge, TypoGraph, DeadEndPathCache, ProximityCondition, ProximityGraph, RankingRuleGraph, TypoEdge, TypoGraph,
@ -65,7 +65,7 @@ pub trait SearchLogger<Q: RankingRuleQueryTrait> {
fn log_proximity_state( fn log_proximity_state(
&mut self, &mut self,
query_graph: &RankingRuleGraph<ProximityGraph>, query_graph: &RankingRuleGraph<ProximityGraph>,
paths: &[Vec<u16>], paths: &[Vec<Interned<ProximityCondition>>],
empty_paths_cache: &DeadEndPathCache<ProximityGraph>, empty_paths_cache: &DeadEndPathCache<ProximityGraph>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,
@ -76,7 +76,7 @@ pub trait SearchLogger<Q: RankingRuleQueryTrait> {
fn log_typo_state( fn log_typo_state(
&mut self, &mut self,
query_graph: &RankingRuleGraph<TypoGraph>, query_graph: &RankingRuleGraph<TypoGraph>,
paths: &[Vec<u16>], paths: &[Vec<Interned<TypoEdge>>],
empty_paths_cache: &DeadEndPathCache<TypoGraph>, empty_paths_cache: &DeadEndPathCache<TypoGraph>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>,
@ -136,7 +136,7 @@ impl<Q: RankingRuleQueryTrait> SearchLogger<Q> for DefaultSearchLogger {
fn log_proximity_state( fn log_proximity_state(
&mut self, &mut self,
_query_graph: &RankingRuleGraph<ProximityGraph>, _query_graph: &RankingRuleGraph<ProximityGraph>,
_paths_map: &[Vec<u16>], _paths_map: &[Vec<Interned<ProximityCondition>>],
_empty_paths_cache: &DeadEndPathCache<ProximityGraph>, _empty_paths_cache: &DeadEndPathCache<ProximityGraph>,
_universe: &RoaringBitmap, _universe: &RoaringBitmap,
_distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>, _distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,
@ -147,7 +147,7 @@ impl<Q: RankingRuleQueryTrait> SearchLogger<Q> for DefaultSearchLogger {
fn log_typo_state( fn log_typo_state(
&mut self, &mut self,
_query_graph: &RankingRuleGraph<TypoGraph>, _query_graph: &RankingRuleGraph<TypoGraph>,
_paths: &[Vec<u16>], _paths: &[Vec<Interned<TypoEdge>>],
_empty_paths_cache: &DeadEndPathCache<TypoGraph>, _empty_paths_cache: &DeadEndPathCache<TypoGraph>,
_universe: &RoaringBitmap, _universe: &RoaringBitmap,
_distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>, _distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>,

14
milli/src/search/new/query_graph.rs
View File

@ -181,8 +181,8 @@ impl QueryGraph {
(prev0, prev1, prev2) = (new_nodes, prev0, prev1); (prev0, prev1, prev2) = (new_nodes, prev0, prev1);
} }
let root_node = Interned::new(root_node); let root_node = Interned::from_raw(root_node);
let end_node = Interned::new(end_node); let end_node = Interned::from_raw(end_node);
let mut nodes = FixedSizeInterner::new( let mut nodes = FixedSizeInterner::new(
nodes_data.len() as u16, nodes_data.len() as u16,
QueryNode { QueryNode {
@ -197,22 +197,22 @@ impl QueryGraph {
.zip(successors.into_iter()) .zip(successors.into_iter())
.enumerate() .enumerate()
{ {
let node = nodes.get_mut(Interned::new(node_idx as u16)); let node = nodes.get_mut(Interned::from_raw(node_idx as u16));
node.data = node_data; node.data = node_data;
for x in predecessors { for x in predecessors {
node.predecessors.insert(Interned::new(x)); node.predecessors.insert(Interned::from_raw(x));
} }
for x in successors { for x in successors {
node.successors.insert(Interned::new(x)); node.successors.insert(Interned::from_raw(x));
} }
} }
let mut graph = QueryGraph { root_node, end_node, nodes }; let mut graph = QueryGraph { root_node, end_node, nodes };
graph.connect_to_node( graph.connect_to_node(
prev0.into_iter().map(Interned::new).collect::<Vec<_>>().as_slice(), prev0.into_iter().map(Interned::from_raw).collect::<Vec<_>>().as_slice(),
end_node, end_node,
); );
let empty_nodes = empty_nodes.into_iter().map(Interned::new).collect::<Vec<_>>(); let empty_nodes = empty_nodes.into_iter().map(Interned::from_raw).collect::<Vec<_>>();
graph.remove_nodes_keep_edges(&empty_nodes); graph.remove_nodes_keep_edges(&empty_nodes);
Ok(graph) Ok(graph)

26
milli/src/search/new/ranking_rule_graph/cheapest_paths.rs
View File

@ -24,7 +24,11 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
cost: u16, cost: u16,
all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::EdgeCondition>)>, QueryNode>, all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::EdgeCondition>)>, QueryNode>,
empty_paths_cache: &mut DeadEndPathCache<G>, empty_paths_cache: &mut DeadEndPathCache<G>,
mut visit: impl FnMut(&[u16], &mut Self, &mut DeadEndPathCache<G>) -> Result<ControlFlow<()>>, mut visit: impl FnMut(
&[Interned<G::EdgeCondition>],
&mut Self,
&mut DeadEndPathCache<G>,
) -> Result<ControlFlow<()>>,
) -> Result<()> { ) -> Result<()> {
let _ = self.visit_paths_of_cost_rec( let _ = self.visit_paths_of_cost_rec(
from, from,
@ -44,8 +48,12 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
cost: u16, cost: u16,
all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::EdgeCondition>)>, QueryNode>, all_distances: &MappedInterner<Vec<(u16, SmallBitmap<G::EdgeCondition>)>, QueryNode>,
empty_paths_cache: &mut DeadEndPathCache<G>, empty_paths_cache: &mut DeadEndPathCache<G>,
visit: &mut impl FnMut(&[u16], &mut Self, &mut DeadEndPathCache<G>) -> Result<ControlFlow<()>>, visit: &mut impl FnMut(
prev_conditions: &mut Vec<u16>, &[Interned<G::EdgeCondition>],
&mut Self,
&mut DeadEndPathCache<G>,
) -> Result<ControlFlow<()>>,
prev_conditions: &mut Vec<Interned<G::EdgeCondition>>,
cur_path: &mut SmallBitmap<G::EdgeCondition>, cur_path: &mut SmallBitmap<G::EdgeCondition>,
forbidden_conditions: &mut SmallBitmap<G::EdgeCondition>, forbidden_conditions: &mut SmallBitmap<G::EdgeCondition>,
) -> Result<bool> { ) -> Result<bool> {
@ -92,8 +100,7 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
continue; continue;
} }
cur_path.insert(condition); cur_path.insert(condition);
// TODO: typed path set prev_conditions.push(condition);
prev_conditions.push(condition.into_inner());
let mut new_forbidden_conditions = forbidden_conditions.clone(); let mut new_forbidden_conditions = forbidden_conditions.clone();
new_forbidden_conditions new_forbidden_conditions
@ -101,7 +108,7 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
empty_paths_cache.prefixes.final_edges_after_prefix( empty_paths_cache.prefixes.final_edges_after_prefix(
prev_conditions, prev_conditions,
&mut |x| { &mut |x| {
new_forbidden_conditions.insert(Interned::new(x)); new_forbidden_conditions.insert(x);
}, },
); );
let next_any_valid = if edge.dest_node == self.query_graph.end_node { let next_any_valid = if edge.dest_node == self.query_graph.end_node {
@ -137,12 +144,11 @@ impl<G: RankingRuleGraphTrait> RankingRuleGraph<G> {
} }
forbidden_conditions.union(&empty_paths_cache.conditions); forbidden_conditions.union(&empty_paths_cache.conditions);
for prev_condition in prev_conditions.iter() { for prev_condition in prev_conditions.iter() {
forbidden_conditions.union( forbidden_conditions
empty_paths_cache.condition_couples.get(Interned::new(*prev_condition)), .union(empty_paths_cache.condition_couples.get(*prev_condition));
);
} }
empty_paths_cache.prefixes.final_edges_after_prefix(prev_conditions, &mut |x| { empty_paths_cache.prefixes.final_edges_after_prefix(prev_conditions, &mut |x| {
forbidden_conditions.insert(Interned::new(x)); forbidden_conditions.insert(x);
}); });
} }
} }

12
milli/src/search/new/ranking_rule_graph/empty_paths_cache.rs
View File

@ -11,7 +11,7 @@ pub struct DeadEndPathCache<G: RankingRuleGraphTrait> {
/// The set of edge conditions that resolve to no documents. /// The set of edge conditions that resolve to no documents.
pub conditions: SmallBitmap<G::EdgeCondition>, pub conditions: SmallBitmap<G::EdgeCondition>,
/// A set of path prefixes that resolve to no documents. /// A set of path prefixes that resolve to no documents.
pub prefixes: PathSet, pub prefixes: PathSet<G::EdgeCondition>,
/// A set of empty couples of edge conditions that resolve to no documents. /// A set of empty couples of edge conditions that resolve to no documents.
pub condition_couples: MappedInterner<SmallBitmap<G::EdgeCondition>, G::EdgeCondition>, pub condition_couples: MappedInterner<SmallBitmap<G::EdgeCondition>, G::EdgeCondition>,
} }
@ -40,13 +40,13 @@ impl<G: RankingRuleGraphTrait> DeadEndPathCache<G> {
pub fn add_condition(&mut self, condition: Interned<G::EdgeCondition>) { pub fn add_condition(&mut self, condition: Interned<G::EdgeCondition>) {
self.conditions.insert(condition); self.conditions.insert(condition);
self.condition_couples.get_mut(condition).clear(); self.condition_couples.get_mut(condition).clear();
self.prefixes.remove_edge(condition.into_inner()); // TODO: typed PathSet self.prefixes.remove_edge(condition);
for (_, edges2) in self.condition_couples.iter_mut() { for (_, edges2) in self.condition_couples.iter_mut() {
edges2.remove(condition); edges2.remove(condition);
} }
} }
/// Store in the cache that every path containing the given prefix resolves to no documents. /// Store in the cache that every path containing the given prefix resolves to no documents.
pub fn add_prefix(&mut self, prefix: &[u16]) { pub fn add_prefix(&mut self, prefix: &[Interned<G::EdgeCondition>]) {
// TODO: typed PathSet // TODO: typed PathSet
self.prefixes.insert(prefix.iter().copied()); self.prefixes.insert(prefix.iter().copied());
} }
@ -63,15 +63,15 @@ impl<G: RankingRuleGraphTrait> DeadEndPathCache<G> {
/// Returns true if the cache can determine that the given path resolves to no documents. /// Returns true if the cache can determine that the given path resolves to no documents.
pub fn path_is_dead_end( pub fn path_is_dead_end(
&self, &self,
path: &[u16], path: &[Interned<G::EdgeCondition>],
path_bitmap: &SmallBitmap<G::EdgeCondition>, path_bitmap: &SmallBitmap<G::EdgeCondition>,
) -> bool { ) -> bool {
if path_bitmap.intersects(&self.conditions) { if path_bitmap.intersects(&self.conditions) {
return true; return true;
} }
for edge in path.iter() { for condition in path.iter() {
// TODO: typed path // TODO: typed path
let forbidden_other_edges = self.condition_couples.get(Interned::new(*edge)); let forbidden_other_edges = self.condition_couples.get(*condition);
if path_bitmap.intersects(forbidden_other_edges) { if path_bitmap.intersects(forbidden_other_edges) {
return true; return true;
} }

2
milli/src/search/new/ranking_rule_graph/mod.rs
View File

@ -117,7 +117,7 @@ pub trait RankingRuleGraphTrait: Sized {
fn log_state( fn log_state(
graph: &RankingRuleGraph<Self>, graph: &RankingRuleGraph<Self>,
paths: &[Vec<u16>], paths: &[Vec<Interned<Self::EdgeCondition>>],
empty_paths_cache: &DeadEndPathCache<Self>, empty_paths_cache: &DeadEndPathCache<Self>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<Self::EdgeCondition>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<Self::EdgeCondition>)>, QueryNode>,

40
milli/src/search/new/ranking_rule_graph/path_set.rs
View File

@ -2,14 +2,34 @@
// For the empty_prefixes field in the EmptyPathsCache only :/ // For the empty_prefixes field in the EmptyPathsCache only :/
// but it could be used for more, like efficient computing of a set of paths // but it could be used for more, like efficient computing of a set of paths
use crate::search::new::interner::Interned;
/// A set of [`Path`] /// A set of [`Path`]
#[derive(Default, Debug, Clone)] pub struct PathSet<T> {
pub struct PathSet { nodes: Vec<(Interned<T>, Self)>,
nodes: Vec<(u16, PathSet)>,
is_end: bool, is_end: bool,
} }
impl PathSet {
pub fn insert(&mut self, mut edges: impl Iterator<Item = u16>) { impl<T> Clone for PathSet<T> {
fn clone(&self) -> Self {
Self { nodes: self.nodes.clone(), is_end: self.is_end }
}
}
impl<T> std::fmt::Debug for PathSet<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PathSet").field("nodes", &self.nodes).field("is_end", &self.is_end).finish()
}
}
impl<T> Default for PathSet<T> {
fn default() -> Self {
Self { nodes: Default::default(), is_end: Default::default() }
}
}
impl<T> PathSet<T> {
pub fn insert(&mut self, mut edges: impl Iterator<Item = Interned<T>>) {
match edges.next() { match edges.next() {
None => { None => {
self.is_end = true; self.is_end = true;
@ -27,7 +47,7 @@ impl PathSet {
} }
} }
pub fn remove_edge(&mut self, forbidden_edge: u16) { pub fn remove_edge(&mut self, forbidden_edge: Interned<T>) {
let mut i = 0; let mut i = 0;
while i < self.nodes.len() { while i < self.nodes.len() {
let should_remove = if self.nodes[i].0 == forbidden_edge { let should_remove = if self.nodes[i].0 == forbidden_edge {
@ -46,7 +66,11 @@ impl PathSet {
} }
} }
pub fn final_edges_after_prefix(&self, prefix: &[u16], visit: &mut impl FnMut(u16)) { pub fn final_edges_after_prefix(
&self,
prefix: &[Interned<T>],
visit: &mut impl FnMut(Interned<T>),
) {
let [first_edge, remaining_prefix @ ..] = prefix else { let [first_edge, remaining_prefix @ ..] = prefix else {
for node in self.nodes.iter() { for node in self.nodes.iter() {
if node.1.is_end { if node.1.is_end {
@ -62,7 +86,7 @@ impl PathSet {
} }
} }
pub fn contains_prefix_of_path(&self, path: &[u16]) -> bool { pub fn contains_prefix_of_path(&self, path: &[Interned<T>]) -> bool {
if self.is_end { if self.is_end {
return true; return true;
} }

2
milli/src/search/new/ranking_rule_graph/proximity/mod.rs
View File

@ -66,7 +66,7 @@ impl RankingRuleGraphTrait for ProximityGraph {
fn log_state( fn log_state(
graph: &RankingRuleGraph<Self>, graph: &RankingRuleGraph<Self>,
paths: &[Vec<u16>], paths: &[Vec<Interned<ProximityCondition>>],
empty_paths_cache: &DeadEndPathCache<Self>, empty_paths_cache: &DeadEndPathCache<Self>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<ProximityCondition>)>, QueryNode>,

2
milli/src/search/new/ranking_rule_graph/typo/mod.rs
View File

@ -137,7 +137,7 @@ impl RankingRuleGraphTrait for TypoGraph {
fn log_state( fn log_state(
graph: &RankingRuleGraph<Self>, graph: &RankingRuleGraph<Self>,
paths: &[Vec<u16>], paths: &[Vec<Interned<TypoEdge>>],
empty_paths_cache: &DeadEndPathCache<Self>, empty_paths_cache: &DeadEndPathCache<Self>,
universe: &RoaringBitmap, universe: &RoaringBitmap,
distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>, distances: &MappedInterner<Vec<(u16, SmallBitmap<TypoEdge>)>, QueryNode>,

17
milli/src/search/new/small_bitmap.rs
View File

@ -32,10 +32,7 @@ impl<T> SmallBitmap<T> {
for_interner: &FixedSizeInterner<T>, for_interner: &FixedSizeInterner<T>,
) -> Self { ) -> Self {
Self { Self {
internal: SmallBitmapInternal::from_iter( internal: SmallBitmapInternal::from_iter(xs.map(|x| x.into_raw()), for_interner.len()),
xs.map(|x| x.into_inner()),
for_interner.len(),
),
_phantom: PhantomData, _phantom: PhantomData,
} }
} }
@ -46,13 +43,13 @@ impl<T> SmallBitmap<T> {
self.internal.clear() self.internal.clear()
} }
pub fn contains(&self, x: Interned<T>) -> bool { pub fn contains(&self, x: Interned<T>) -> bool {
self.internal.contains(x.into_inner()) self.internal.contains(x.into_raw())
} }
pub fn insert(&mut self, x: Interned<T>) { pub fn insert(&mut self, x: Interned<T>) {
self.internal.insert(x.into_inner()) self.internal.insert(x.into_raw())
} }
pub fn remove(&mut self, x: Interned<T>) { pub fn remove(&mut self, x: Interned<T>) {
self.internal.remove(x.into_inner()) self.internal.remove(x.into_raw())
} }
pub fn intersection(&mut self, other: &Self) { pub fn intersection(&mut self, other: &Self) {
@ -71,7 +68,7 @@ impl<T> SmallBitmap<T> {
self.internal.intersects(&other.internal) self.internal.intersects(&other.internal)
} }
pub fn iter(&self) -> impl Iterator<Item = Interned<T>> + '_ { pub fn iter(&self) -> impl Iterator<Item = Interned<T>> + '_ {
self.internal.iter().map(|x| Interned::new(x)) self.internal.iter().map(|x| Interned::from_raw(x))
} }
} }
#[derive(Clone)] #[derive(Clone)]
@ -80,14 +77,14 @@ pub enum SmallBitmapInternal {
Small(Box<[u64]>), Small(Box<[u64]>),
} }
impl SmallBitmapInternal { impl SmallBitmapInternal {
pub fn new(universe_length: u16) -> Self { fn new(universe_length: u16) -> Self {
if universe_length <= 64 { if universe_length <= 64 {
Self::Tiny(0) Self::Tiny(0)
} else { } else {
Self::Small(vec![0; 1 + universe_length as usize / 64].into_boxed_slice()) Self::Small(vec![0; 1 + universe_length as usize / 64].into_boxed_slice())
} }
} }
pub fn from_iter(xs: impl Iterator<Item = u16>, universe_length: u16) -> Self { fn from_iter(xs: impl Iterator<Item = u16>, universe_length: u16) -> Self {
let mut s = Self::new(universe_length); let mut s = Self::new(universe_length);
for x in xs { for x in xs {
s.insert(x); s.insert(x);