snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 10:37:41 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Expose and use the WordsLevelPositions update

Browse Source
This commit is contained in:
Kerollmops 2021-03-17 13:55:24 +01:00 committed by many
parent c765f277a3
commit 3a25137ee4
No known key found for this signature in database
GPG Key ID: 2CEF23B75189EACA
2 changed files with 36 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
milli/src/update/index_documents/mod.rs
View File

@ -263,6 +263,8 @@ pub struct IndexDocuments<'t, 'u, 'i, 'a> {
facet_min_level_size: Option<NonZeroUsize>, facet_min_level_size: Option<NonZeroUsize>,
words_prefix_threshold: Option<f64>, words_prefix_threshold: Option<f64>,
max_prefix_length: Option<usize>, max_prefix_length: Option<usize>,
words_positions_level_group_size: Option<NonZeroUsize>,
words_positions_min_level_size: Option<NonZeroUsize>,
update_method: IndexDocumentsMethod, update_method: IndexDocumentsMethod,
update_format: UpdateFormat, update_format: UpdateFormat,
autogenerate_docids: bool, autogenerate_docids: bool,
@ -290,6 +292,8 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
facet_min_level_size: None, facet_min_level_size: None,
words_prefix_threshold: None, words_prefix_threshold: None,
max_prefix_length: None, max_prefix_length: None,
words_positions_level_group_size: None,
words_positions_min_level_size: None,
update_method: IndexDocumentsMethod::ReplaceDocuments, update_method: IndexDocumentsMethod::ReplaceDocuments,
update_format: UpdateFormat::Json, update_format: UpdateFormat::Json,
autogenerate_docids: true, autogenerate_docids: true,
@ -740,6 +744,19 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
} }
builder.execute()?; builder.execute()?;
// Run the words level positions update operation.
let mut builder = WordsLevelPositions::new(self.wtxn, self.index, self.update_id);
builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level;
builder.chunk_fusing_shrink_size = self.chunk_fusing_shrink_size;
if let Some(value) = self.words_positions_level_group_size {
builder.level_group_size(value);
}
if let Some(value) = self.words_positions_min_level_size {
builder.min_level_size(value);
}
builder.execute()?;
debug_assert_eq!(database_count, total_databases); debug_assert_eq!(database_count, total_databases);
info!("Transform output indexed in {:.02?}", before_indexing.elapsed()); info!("Transform output indexed in {:.02?}", before_indexing.elapsed());

20
milli/src/update/update_builder.rs
View File

@ -2,7 +2,10 @@ use grenad::CompressionType;
use rayon::ThreadPool; use rayon::ThreadPool;
use crate::Index; use crate::Index;
use super::{ClearDocuments, DeleteDocuments, IndexDocuments, Settings, Facets, WordsPrefixes}; use super::{
ClearDocuments, DeleteDocuments, IndexDocuments, Settings,
Facets, WordsPrefixes, WordsLevelPositions,
};
pub struct UpdateBuilder<'a> { pub struct UpdateBuilder<'a> {
pub(crate) log_every_n: Option<usize>, pub(crate) log_every_n: Option<usize>,
@ -150,4 +153,19 @@ impl<'a> UpdateBuilder<'a> {
builder builder
} }
pub fn words_level_positions<'t, 'u, 'i>(
self,
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> WordsLevelPositions<'t, 'u, 'i>
{
let mut builder = WordsLevelPositions::new(wtxn, index, self.update_id);
builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level;
builder.chunk_fusing_shrink_size = self.chunk_fusing_shrink_size;
builder
}
} }