snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Allow fetching embedders from inside tests

Browse Source
This commit is contained in:
Louis Dureuil 2024-11-19 15:16:32 +01:00
parent 0019ec74fe
commit 4c59503c6c
No known key found for this signature in database
5 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
crates/milli/src/index.rs
View File

@ -1689,6 +1689,7 @@ pub(crate) mod tests {
use crate::error::{Error, InternalError};
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::new::indexer;
use crate::update::settings::InnerIndexSettings;
use crate::update::{
self, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Setting, Settings,
};
@ -1749,8 +1750,8 @@ pub(crate) mod tests {
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default();
/// TODO: fetch configs from the index
let embedders =
InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs;
let mut indexer =
indexer::DocumentOperation::new(self.index_documents_config.update_method);
indexer.add_documents(&documents).unwrap();
@ -1830,7 +1831,9 @@ pub(crate) mod tests {
let db_fields_ids_map = self.inner.fields_ids_map(&rtxn)?;
let mut new_fields_ids_map = db_fields_ids_map.clone();
let embedders = EmbeddingConfigs::default();
let embedders =
InnerIndexSettings::from_index(&self.inner, &rtxn, None)?.embedding_configs;
let mut indexer =
indexer::DocumentOperation::new(self.index_documents_config.update_method);
let external_document_ids: Vec<_> =

2
crates/milli/src/update/index_documents/transform.rs
View File

@ -620,7 +620,7 @@ impl<'a, 'i> Transform<'a, 'i> {
fst_new_external_documents_ids_builder.insert(key, value)
})?;
let old_inner_settings = InnerIndexSettings::from_index(self.index, wtxn)?;
let old_inner_settings = InnerIndexSettings::from_index(self.index, wtxn, None)?;
let fields_ids_map = self.fields_ids_map;
let primary_key_id = self.index.primary_key(wtxn)?.and_then(|name| fields_ids_map.id(name));
let mut new_inner_settings = old_inner_settings.clone();

2
crates/milli/src/update/mod.rs
View File

@ -19,7 +19,7 @@ pub(crate) mod facet;
mod index_documents;
mod indexer_config;
pub mod new;
mod settings;
pub(crate) mod settings;
mod update_step;
mod word_prefix_docids;
mod words_prefix_integer_docids;

2
crates/milli/src/update/new/indexer/mod.rs
View File

@ -475,7 +475,7 @@ where
}
// used to update the localized and weighted maps while sharing the update code with the settings pipeline.
let mut inner_index_settings = InnerIndexSettings::from_index(index, wtxn)?;
let mut inner_index_settings = InnerIndexSettings::from_index(index, wtxn, Some(embedders))?;
inner_index_settings.recompute_facets(wtxn, index)?;
inner_index_settings.recompute_searchables(wtxn, index)?;
index.put_field_distribution(wtxn, &field_distribution)?;

15
crates/milli/src/update/settings.rs
View File

@ -1169,7 +1169,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
{
self.index.set_updated_at(self.wtxn, &OffsetDateTime::now_utc())?;
let old_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn)?;
let old_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn, None)?;
// never trigger re-indexing
self.update_displayed()?;
@ -1199,7 +1199,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
let embedding_config_updates = self.update_embedding_configs()?;
let mut new_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn)?;
let mut new_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn, None)?;
new_inner_settings.recompute_facets(self.wtxn, self.index)?;
let primary_key_id = self
@ -1427,7 +1427,11 @@ pub(crate) struct InnerIndexSettings {
}
impl InnerIndexSettings {
pub fn from_index(index: &Index, rtxn: &heed::RoTxn<'_>) -> Result<Self> {
pub fn from_index(
index: &Index,
rtxn: &heed::RoTxn<'_>,
embedding_configs: Option<EmbeddingConfigs>,
) -> Result<Self> {
let stop_words = index.stop_words(rtxn)?;
let stop_words = stop_words.map(|sw| sw.map_data(Vec::from).unwrap());
let allowed_separators = index.allowed_separators(rtxn)?;
@ -1441,7 +1445,10 @@ impl InnerIndexSettings {
let mut faceted_fields_ids = index.faceted_fields_ids(rtxn)?;
let exact_attributes = index.exact_attributes_ids(rtxn)?;
let proximity_precision = index.proximity_precision(rtxn)?.unwrap_or_default();
let embedding_configs = embedders(index.embedding_configs(rtxn)?)?;
let embedding_configs = match embedding_configs {
Some(embedding_configs) => embedding_configs,
None => embedders(index.embedding_configs(rtxn)?)?,
};
let existing_fields: HashSet<_> = index
.field_distribution(rtxn)?
.into_iter()