diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 11682177f..315988e98 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -27,8 +27,8 @@ use crate::proximity::ProximityPrecision; use crate::update::index_documents::IndexDocumentsMethod; use crate::update::{IndexDocuments, UpdateIndexingStep}; use crate::vector::settings::{ - check_set, check_unset, EmbedderAction, EmbedderSource, EmbeddingSettings, ReindexAction, - WriteBackToDocuments, + EmbedderAction, EmbedderSource, EmbeddingSettings, NestingContext, ReindexAction, + SubEmbeddingSettings, WriteBackToDocuments, }; use crate::vector::{Embedder, EmbeddingConfig, EmbeddingConfigs}; use crate::{FieldId, FieldsIdsMap, Index, LocalizedAttributesRule, LocalizedFieldIds, Result}; @@ -1669,26 +1669,12 @@ fn embedders(embedding_configs: Vec) -> Result, -) -> Result> { - match new { - Setting::Set(EmbeddingSettings { - source, - model, - revision, - pooling, - api_key, - dimensions, - document_template: Setting::Set(template), - document_template_max_bytes, - url, - request, - response, - distribution, - headers, - binary_quantized: binary_quantize, - }) => { - let max_bytes = match document_template_max_bytes.set() { + new_prompt: Setting, + max_bytes: Setting, +) -> Result> { + match new_prompt { + Setting::Set(template) => { + let max_bytes = match max_bytes.set() { Some(max_bytes) => NonZeroUsize::new(max_bytes).ok_or_else(|| { crate::error::UserError::InvalidSettingsDocumentTemplateMaxBytes { embedder_name: name.to_owned(), @@ -1706,22 +1692,7 @@ fn validate_prompt( .map(|prompt| crate::prompt::PromptData::from(prompt).template) .map_err(|inner| UserError::InvalidPromptForEmbeddings(name.to_owned(), inner))?; - Ok(Setting::Set(EmbeddingSettings { - source, - model, - revision, - pooling, - api_key, - dimensions, - document_template: Setting::Set(template), - document_template_max_bytes, - url, - request, - response, - distribution, - headers, - binary_quantized: binary_quantize, - })) + Ok(Setting::Set(template)) } new => Ok(new), } @@ -1731,7 +1702,6 @@ pub fn validate_embedding_settings( settings: Setting, name: &str, ) -> Result> { - let settings = validate_prompt(name, settings)?; let Setting::Set(settings) = settings else { return Ok(settings) }; let EmbeddingSettings { source, @@ -1745,11 +1715,15 @@ pub fn validate_embedding_settings( url, request, response, + search_embedder, + mut indexing_embedder, distribution, headers, binary_quantized: binary_quantize, } = settings; + let document_template = validate_prompt(name, document_template, document_template_max_bytes)?; + if let Some(0) = dimensions.set() { return Err(crate::error::UserError::InvalidSettingsDimensions { embedder_name: name.to_owned(), @@ -1775,6 +1749,7 @@ pub fn validate_embedding_settings( } let Some(inferred_source) = source.set() else { + // we are validating the fused settings, so we always have a source return Ok(Setting::Set(EmbeddingSettings { source, model, @@ -1787,20 +1762,35 @@ pub fn validate_embedding_settings( url, request, response, + search_embedder, + indexing_embedder, distribution, headers, binary_quantized: binary_quantize, })); }; + EmbeddingSettings::check_settings( + name, + inferred_source, + NestingContext::NotNested, + &model, + &revision, + &pooling, + &dimensions, + &api_key, + &url, + &request, + &response, + &document_template, + &document_template_max_bytes, + &headers, + &search_embedder, + &indexing_embedder, + &binary_quantize, + &distribution, + )?; match inferred_source { EmbedderSource::OpenAi => { - check_unset(&revision, EmbeddingSettings::REVISION, inferred_source, name)?; - check_unset(&pooling, EmbeddingSettings::POOLING, inferred_source, name)?; - - check_unset(&request, EmbeddingSettings::REQUEST, inferred_source, name)?; - check_unset(&response, EmbeddingSettings::RESPONSE, inferred_source, name)?; - check_unset(&headers, EmbeddingSettings::HEADERS, inferred_source, name)?; - if let Setting::Set(model) = &model { let model = crate::vector::openai::EmbeddingModel::from_name(model.as_str()) .ok_or(crate::error::UserError::InvalidOpenAiModel { @@ -1831,55 +1821,117 @@ pub fn validate_embedding_settings( } } } - EmbedderSource::Ollama => { - check_set(&model, EmbeddingSettings::MODEL, inferred_source, name)?; - check_unset(&revision, EmbeddingSettings::REVISION, inferred_source, name)?; - check_unset(&pooling, EmbeddingSettings::POOLING, inferred_source, name)?; + EmbedderSource::Ollama + | EmbedderSource::HuggingFace + | EmbedderSource::UserProvided + | EmbedderSource::Rest => {} + EmbedderSource::Composite => { + if let Setting::Set(embedder) = &search_embedder { + if let Some(source) = embedder.source.set() { + let search_embedder = match embedder.search_embedder.clone() { + Setting::Set(search_embedder) => Setting::Set(deserialize_sub_embedder( + search_embedder, + name, + NestingContext::Search, + )?), + Setting::Reset => Setting::Reset, + Setting::NotSet => Setting::NotSet, + }; + let indexing_embedder = match embedder.indexing_embedder.clone() { + Setting::Set(indexing_embedder) => Setting::Set(deserialize_sub_embedder( + indexing_embedder, + name, + NestingContext::Search, + )?), + Setting::Reset => Setting::Reset, + Setting::NotSet => Setting::NotSet, + }; + EmbeddingSettings::check_nested_source(name, source, NestingContext::Search)?; + EmbeddingSettings::check_settings( + name, + source, + NestingContext::Search, + &embedder.model, + &embedder.revision, + &embedder.pooling, + &embedder.dimensions, + &embedder.api_key, + &embedder.url, + &embedder.request, + &embedder.response, + &embedder.document_template, + &embedder.document_template_max_bytes, + &embedder.headers, + &search_embedder, + &indexing_embedder, + &embedder.binary_quantized, + &embedder.distribution, + )?; + } else { + return Err(UserError::MissingSourceForNested { + embedder_name: NestingContext::Search.embedder_name_with_context(name), + } + .into()); + } + } - check_unset(&request, EmbeddingSettings::REQUEST, inferred_source, name)?; - check_unset(&response, EmbeddingSettings::RESPONSE, inferred_source, name)?; - check_unset(&headers, EmbeddingSettings::HEADERS, inferred_source, name)?; - } - EmbedderSource::HuggingFace => { - check_unset(&api_key, EmbeddingSettings::API_KEY, inferred_source, name)?; - check_unset(&dimensions, EmbeddingSettings::DIMENSIONS, inferred_source, name)?; + indexing_embedder = if let Setting::Set(mut embedder) = indexing_embedder { + embedder.document_template = validate_prompt( + name, + embedder.document_template, + embedder.document_template_max_bytes, + )?; - check_unset(&url, EmbeddingSettings::URL, inferred_source, name)?; - check_unset(&request, EmbeddingSettings::REQUEST, inferred_source, name)?; - check_unset(&response, EmbeddingSettings::RESPONSE, inferred_source, name)?; - check_unset(&headers, EmbeddingSettings::HEADERS, inferred_source, name)?; - } - EmbedderSource::UserProvided => { - check_unset(&model, EmbeddingSettings::MODEL, inferred_source, name)?; - check_unset(&revision, EmbeddingSettings::REVISION, inferred_source, name)?; - check_unset(&pooling, EmbeddingSettings::POOLING, inferred_source, name)?; - check_unset(&api_key, EmbeddingSettings::API_KEY, inferred_source, name)?; - check_unset( - &document_template, - EmbeddingSettings::DOCUMENT_TEMPLATE, - inferred_source, - name, - )?; - check_unset( - &document_template_max_bytes, - EmbeddingSettings::DOCUMENT_TEMPLATE_MAX_BYTES, - inferred_source, - name, - )?; - check_set(&dimensions, EmbeddingSettings::DIMENSIONS, inferred_source, name)?; - - check_unset(&url, EmbeddingSettings::URL, inferred_source, name)?; - check_unset(&request, EmbeddingSettings::REQUEST, inferred_source, name)?; - check_unset(&response, EmbeddingSettings::RESPONSE, inferred_source, name)?; - check_unset(&headers, EmbeddingSettings::HEADERS, inferred_source, name)?; - } - EmbedderSource::Rest => { - check_unset(&model, EmbeddingSettings::MODEL, inferred_source, name)?; - check_unset(&revision, EmbeddingSettings::REVISION, inferred_source, name)?; - check_unset(&pooling, EmbeddingSettings::POOLING, inferred_source, name)?; - check_set(&url, EmbeddingSettings::URL, inferred_source, name)?; - check_set(&request, EmbeddingSettings::REQUEST, inferred_source, name)?; - check_set(&response, EmbeddingSettings::RESPONSE, inferred_source, name)?; + if let Some(source) = embedder.source.set() { + let search_embedder = match embedder.search_embedder.clone() { + Setting::Set(search_embedder) => Setting::Set(deserialize_sub_embedder( + search_embedder, + name, + NestingContext::Indexing, + )?), + Setting::Reset => Setting::Reset, + Setting::NotSet => Setting::NotSet, + }; + let indexing_embedder = match embedder.indexing_embedder.clone() { + Setting::Set(indexing_embedder) => Setting::Set(deserialize_sub_embedder( + indexing_embedder, + name, + NestingContext::Indexing, + )?), + Setting::Reset => Setting::Reset, + Setting::NotSet => Setting::NotSet, + }; + EmbeddingSettings::check_nested_source(name, source, NestingContext::Indexing)?; + EmbeddingSettings::check_settings( + name, + source, + NestingContext::Indexing, + &embedder.model, + &embedder.revision, + &embedder.pooling, + &embedder.dimensions, + &embedder.api_key, + &embedder.url, + &embedder.request, + &embedder.response, + &embedder.document_template, + &embedder.document_template_max_bytes, + &embedder.headers, + &search_embedder, + &indexing_embedder, + &embedder.binary_quantized, + &embedder.distribution, + )?; + } else { + return Err(UserError::MissingSourceForNested { + embedder_name: NestingContext::Indexing.embedder_name_with_context(name), + } + .into()); + } + Setting::Set(embedder) + } else { + indexing_embedder + }; } } Ok(Setting::Set(EmbeddingSettings { diff --git a/crates/milli/src/vector/settings.rs b/crates/milli/src/vector/settings.rs index 4e9997028..610597dd5 100644 --- a/crates/milli/src/vector/settings.rs +++ b/crates/milli/src/vector/settings.rs @@ -6,8 +6,9 @@ use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +use super::composite::SubEmbedderOptions; use super::hf::OverridePooling; -use super::{ollama, openai, DistributionShift}; +use super::{ollama, openai, DistributionShift, EmbedderOptions}; use crate::prompt::{default_max_bytes, PromptData}; use crate::update::Setting; use crate::vector::EmbeddingConfig; @@ -265,6 +266,17 @@ pub struct EmbeddingSettings { /// /// - 🌱 Changing the value of this parameter never regenerates embeddings pub headers: Setting>, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + pub search_embedder: Setting, + + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + pub indexing_embedder: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] #[deserr(default)] #[schema(value_type = Option)] @@ -280,23 +292,254 @@ pub struct EmbeddingSettings { pub distribution: Setting, } -pub fn check_unset( - key: &Setting, - field: &'static str, - source: EmbedderSource, - embedder_name: &str, -) -> Result<(), UserError> { - if matches!(key, Setting::NotSet) { - Ok(()) - } else { - Err(UserError::InvalidFieldForSource { - embedder_name: embedder_name.to_owned(), - source_: source, - field, - allowed_fields_for_source: EmbeddingSettings::allowed_fields_for_source(source), - allowed_sources_for_field: EmbeddingSettings::allowed_sources_for_field(field), - }) - } +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[deserr(rename_all = camelCase, deny_unknown_fields)] +pub struct SubEmbeddingSettings { + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The source used to provide the embeddings. + /// + /// Which embedder parameters are available and mandatory is determined by the value of this setting. + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings. + /// + /// # Defaults + /// + /// - Defaults to `openAi` + pub source: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The name of the model to use. + /// + /// # Mandatory + /// + /// - This parameter is mandatory for source `ollama` + /// + /// # Availability + /// + /// - This parameter is available for sources `openAi`, `huggingFace`, `ollama` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings. + /// + /// # Defaults + /// + /// - For source `openAi`, defaults to `text-embedding-3-small` + /// - For source `huggingFace`, defaults to `BAAI/bge-base-en-v1.5` + pub model: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The revision (commit SHA1) of the model to use. + /// + /// If unspecified, Meilisearch picks the latest revision of the model. + /// + /// # Availability + /// + /// - This parameter is available for source `huggingFace` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings + /// + /// # Defaults + /// + /// - When `model` is set to default, defaults to `617ca489d9e86b49b8167676d8220688b99db36e` + /// - Otherwise, defaults to `null` + pub revision: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The pooling method to use. + /// + /// # Availability + /// + /// - This parameter is available for source `huggingFace` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings + /// + /// # Defaults + /// + /// - Defaults to `useModel` + /// + /// # Compatibility Note + /// + /// - Embedders created before this parameter was available default to `forceMean` to preserve the existing behavior. + pub pooling: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The API key to pass to the remote embedder while making requests. + /// + /// # Availability + /// + /// - This parameter is available for source `openAi`, `ollama`, `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🌱 Changing the value of this parameter never regenerates embeddings + /// + /// # Defaults + /// + /// - For source `openAi`, the key is read from `OPENAI_API_KEY`, then `MEILI_OPENAI_API_KEY`. + /// - For other sources, no bearer token is sent if this parameter is not set. + /// + /// # Note + /// + /// - This setting is partially hidden when returned by the settings + pub api_key: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// The expected dimensions of the embeddings produced by this embedder. + /// + /// # Mandatory + /// + /// - This parameter is mandatory for source `userProvided` + /// + /// # Availability + /// + /// - This parameter is available for source `openAi`, `ollama`, `rest`, `userProvided` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ When the source is `openAi`, changing the value of this parameter always regenerates embeddings + /// - 🌱 For other sources, changing the value of this parameter never regenerates embeddings + /// + /// # Defaults + /// + /// - For source `openAi`, the dimensions is the maximum allowed by the model. + /// - For sources `ollama` and `rest`, the dimensions are inferred by embedding a sample text. + pub dimensions: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// A liquid template used to render documents to a text that can be embedded. + /// + /// Meillisearch interpolates the template for each document and sends the resulting text to the embedder. + /// The embedder then generates document vectors based on this text. + /// + /// # Availability + /// + /// - This parameter is available for source `openAi`, `huggingFace`, `ollama` and `rest + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ When modified, embeddings are regenerated for documents whose rendering through the template produces a different text. + pub document_template: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// Rendered texts are truncated to this size. + /// + /// # Availability + /// + /// - This parameter is available for source `openAi`, `huggingFace`, `ollama` and `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ When increased, embeddings are regenerated for documents whose rendering through the template produces a different text. + /// - 🌱 When decreased, embeddings are never regenerated + /// + /// # Default + /// + /// - Defaults to 400 + pub document_template_max_bytes: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// URL to reach the remote embedder. + /// + /// # Mandatory + /// + /// - This parameter is mandatory for source `rest` + /// + /// # Availability + /// + /// - This parameter is available for source `openAi`, `ollama` and `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🌱 When modified for source `openAi`, embeddings are never regenerated + /// - 🏗️ When modified for sources `ollama` and `rest`, embeddings are always regenerated + pub url: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// Template request to send to the remote embedder. + /// + /// # Mandatory + /// + /// - This parameter is mandatory for source `rest` + /// + /// # Availability + /// + /// - This parameter is available for source `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings + pub request: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option)] + /// Template response indicating how to find the embeddings in the response from the remote embedder. + /// + /// # Mandatory + /// + /// - This parameter is mandatory for source `rest` + /// + /// # Availability + /// + /// - This parameter is available for source `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🏗️ Changing the value of this parameter always regenerates embeddings + pub response: Setting, + #[serde(default, skip_serializing_if = "Setting::is_not_set")] + #[deserr(default)] + #[schema(value_type = Option>)] + /// Additional headers to send to the remote embedder. + /// + /// # Availability + /// + /// - This parameter is available for source `rest` + /// + /// # 🔄 Reindexing + /// + /// - 🌱 Changing the value of this parameter never regenerates embeddings + pub headers: Setting>, + + // The following fields are provided for the sake of improving error handling + // They should always be set to `NotSet`, otherwise an error will be returned + #[serde(default, skip_serializing)] + #[deserr(default)] + #[schema(ignore)] + pub distribution: Setting, + + #[serde(default, skip_serializing)] + #[deserr(default)] + #[schema(ignore)] + pub binary_quantized: Setting, + + #[serde(default, skip_serializing)] + #[deserr(default)] + #[schema(ignore)] + pub search_embedder: Setting, + + #[serde(default, skip_serializing)] + #[deserr(default)] + #[schema(ignore)] + pub indexing_embedder: Setting, } /// Indicates what action should take place during a reindexing operation for an embedder @@ -381,6 +624,8 @@ impl SettingsDiff { mut url, mut request, mut response, + mut search_embedder, + mut indexing_embedder, mut distribution, mut headers, mut document_template_max_bytes, @@ -398,6 +643,8 @@ impl SettingsDiff { url: new_url, request: new_request, response: new_response, + search_embedder: new_search_embedder, + indexing_embedder: new_indexing_embedder, distribution: new_distribution, headers: new_headers, document_template_max_bytes: new_document_template_max_bytes, @@ -414,93 +661,45 @@ impl SettingsDiff { let mut reindex_action = None; - // **Warning**: do not use short-circuiting || here, we want all these operations applied - if source.apply(new_source) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - // when the source changes, we need to reapply the default settings for the new source - apply_default_for_source( - &source, - &mut model, - &mut revision, - &mut pooling, - &mut dimensions, - &mut url, - &mut request, - &mut response, - &mut document_template, - &mut document_template_max_bytes, - &mut headers, - ) - } - if model.apply(new_model) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - } - if revision.apply(new_revision) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - } - if pooling.apply(new_pooling) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - } - if dimensions.apply(new_dimensions) { - match source { - // regenerate on dimensions change in OpenAI since truncation is supported - Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => { - ReindexAction::push_action( - &mut reindex_action, - ReindexAction::FullReindex, - ); - } - // for all other embedders, the parameter is a hint that should not be able to change the result - // and so won't cause a reindex by itself. - _ => {} - } - } + Self::diff( + &mut reindex_action, + &mut source, + &mut model, + &mut revision, + &mut pooling, + &mut api_key, + &mut dimensions, + &mut document_template, + &mut document_template_max_bytes, + &mut url, + &mut request, + &mut response, + &mut headers, + new_source, + new_model, + new_revision, + new_pooling, + new_api_key, + new_dimensions, + new_document_template, + new_document_template_max_bytes, + new_url, + new_request, + new_response, + new_headers, + ); + let binary_quantize_changed = binary_quantize.apply(new_binary_quantize); - if url.apply(new_url) { - match source { - // do not regenerate on an url change in OpenAI - Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => {} - _ => { - ReindexAction::push_action( - &mut reindex_action, - ReindexAction::FullReindex, - ); - } - } - } - if request.apply(new_request) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - } - if response.apply(new_response) { - ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex); - } - if document_template.apply(new_document_template) { - ReindexAction::push_action( - &mut reindex_action, - ReindexAction::RegeneratePrompts, - ); - } - if document_template_max_bytes.apply(new_document_template_max_bytes) { - let previous_document_template_max_bytes = - document_template_max_bytes.set().unwrap_or(default_max_bytes().get()); - let new_document_template_max_bytes = - new_document_template_max_bytes.set().unwrap_or(default_max_bytes().get()); - - // only reindex if the size increased. Reasoning: - // - size decrease is a performance optimization, so we don't reindex and we keep the more accurate vectors - // - size increase is an accuracy optimization, so we want to reindex - if new_document_template_max_bytes > previous_document_template_max_bytes { - ReindexAction::push_action( - &mut reindex_action, - ReindexAction::RegeneratePrompts, - ) - } - } + // changes to the *search* embedder never triggers any reindexing + search_embedder.apply(new_search_embedder); + indexing_embedder = Self::from_sub_settings( + indexing_embedder, + new_indexing_embedder, + &mut reindex_action, + )?; distribution.apply(new_distribution); - api_key.apply(new_api_key); - headers.apply(new_headers); let updated_settings = EmbeddingSettings { source, @@ -513,6 +712,8 @@ impl SettingsDiff { url, request, response, + search_embedder, + indexing_embedder, distribution, headers, document_template_max_bytes, @@ -538,6 +739,223 @@ impl SettingsDiff { }; Ok(ret) } + + fn from_sub_settings( + sub_embedder: Setting, + new_sub_embedder: Setting, + reindex_action: &mut Option, + ) -> Result, UserError> { + let ret = match new_sub_embedder { + Setting::Set(new_sub_embedder) => { + let Setting::Set(SubEmbeddingSettings { + mut source, + mut model, + mut revision, + mut pooling, + mut api_key, + mut dimensions, + mut document_template, + mut document_template_max_bytes, + mut url, + mut request, + mut response, + mut headers, + // phony settings + mut distribution, + mut binary_quantized, + mut search_embedder, + mut indexing_embedder, + }) = sub_embedder + else { + // return the new_indexing_embedder if the indexing_embedder was not set + // this should happen only when changing the source, so the decision to reindex is already taken. + return Ok(Setting::Set(new_sub_embedder)); + }; + + let SubEmbeddingSettings { + source: new_source, + model: new_model, + revision: new_revision, + pooling: new_pooling, + api_key: new_api_key, + dimensions: new_dimensions, + document_template: new_document_template, + document_template_max_bytes: new_document_template_max_bytes, + url: new_url, + request: new_request, + response: new_response, + headers: new_headers, + distribution: new_distribution, + binary_quantized: new_binary_quantized, + search_embedder: new_search_embedder, + indexing_embedder: new_indexing_embedder, + } = new_sub_embedder; + + Self::diff( + reindex_action, + &mut source, + &mut model, + &mut revision, + &mut pooling, + &mut api_key, + &mut dimensions, + &mut document_template, + &mut document_template_max_bytes, + &mut url, + &mut request, + &mut response, + &mut headers, + new_source, + new_model, + new_revision, + new_pooling, + new_api_key, + new_dimensions, + new_document_template, + new_document_template_max_bytes, + new_url, + new_request, + new_response, + new_headers, + ); + + // update phony settings, it is always an error to have them set. + distribution.apply(new_distribution); + binary_quantized.apply(new_binary_quantized); + search_embedder.apply(new_search_embedder); + indexing_embedder.apply(new_indexing_embedder); + + let updated_settings = SubEmbeddingSettings { + source, + model, + revision, + pooling, + api_key, + dimensions, + document_template, + url, + request, + response, + headers, + document_template_max_bytes, + distribution, + binary_quantized, + search_embedder, + indexing_embedder, + }; + Setting::Set(updated_settings) + } + // handled during validation of the settings + Setting::Reset | Setting::NotSet => sub_embedder, + }; + Ok(ret) + } + + #[allow(clippy::too_many_arguments)] + fn diff( + reindex_action: &mut Option, + source: &mut Setting, + model: &mut Setting, + revision: &mut Setting, + pooling: &mut Setting, + api_key: &mut Setting, + dimensions: &mut Setting, + document_template: &mut Setting, + document_template_max_bytes: &mut Setting, + url: &mut Setting, + request: &mut Setting, + response: &mut Setting, + headers: &mut Setting>, + new_source: Setting, + new_model: Setting, + new_revision: Setting, + new_pooling: Setting, + new_api_key: Setting, + new_dimensions: Setting, + new_document_template: Setting, + new_document_template_max_bytes: Setting, + new_url: Setting, + new_request: Setting, + new_response: Setting, + new_headers: Setting>, + ) { + // **Warning**: do not use short-circuiting || here, we want all these operations applied + if source.apply(new_source) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + // when the source changes, we need to reapply the default settings for the new source + apply_default_for_source( + &*source, + model, + revision, + pooling, + dimensions, + url, + request, + response, + document_template, + document_template_max_bytes, + headers, + // send dummy values, the source cannot recursively be composite + &mut Setting::NotSet, + &mut Setting::NotSet, + ) + } + if model.apply(new_model) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + if revision.apply(new_revision) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + if pooling.apply(new_pooling) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + if dimensions.apply(new_dimensions) { + match *source { + // regenerate on dimensions change in OpenAI since truncation is supported + Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + // for all other embedders, the parameter is a hint that should not be able to change the result + // and so won't cause a reindex by itself. + _ => {} + } + } + if url.apply(new_url) { + match *source { + // do not regenerate on an url change in OpenAI + Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => {} + _ => { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + } + } + if request.apply(new_request) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + if response.apply(new_response) { + ReindexAction::push_action(reindex_action, ReindexAction::FullReindex); + } + if document_template.apply(new_document_template) { + ReindexAction::push_action(reindex_action, ReindexAction::RegeneratePrompts); + } + + if document_template_max_bytes.apply(new_document_template_max_bytes) { + let previous_document_template_max_bytes = + document_template_max_bytes.set().unwrap_or(default_max_bytes().get()); + let new_document_template_max_bytes = + new_document_template_max_bytes.set().unwrap_or(default_max_bytes().get()); + + // only reindex if the size increased. Reasoning: + // - size decrease is a performance optimization, so we don't reindex and we keep the more accurate vectors + // - size increase is an accuracy optimization, so we want to reindex + if new_document_template_max_bytes > previous_document_template_max_bytes { + ReindexAction::push_action(reindex_action, ReindexAction::RegeneratePrompts) + } + } + + api_key.apply(new_api_key); + headers.apply(new_headers); + } } impl ReindexAction { @@ -563,6 +981,8 @@ fn apply_default_for_source( document_template: &mut Setting, document_template_max_bytes: &mut Setting, headers: &mut Setting>, + search_embedder: &mut Setting, + indexing_embedder: &mut Setting, ) { match source { Setting::Set(EmbedderSource::HuggingFace) => { @@ -574,6 +994,8 @@ fn apply_default_for_source( *request = Setting::NotSet; *response = Setting::NotSet; *headers = Setting::NotSet; + *search_embedder = Setting::NotSet; + *indexing_embedder = Setting::NotSet; } Setting::Set(EmbedderSource::Ollama) => { *model = Setting::Reset; @@ -584,6 +1006,8 @@ fn apply_default_for_source( *request = Setting::NotSet; *response = Setting::NotSet; *headers = Setting::NotSet; + *search_embedder = Setting::NotSet; + *indexing_embedder = Setting::NotSet; } Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => { *model = Setting::Reset; @@ -594,6 +1018,8 @@ fn apply_default_for_source( *request = Setting::NotSet; *response = Setting::NotSet; *headers = Setting::NotSet; + *search_embedder = Setting::NotSet; + *indexing_embedder = Setting::NotSet; } Setting::Set(EmbedderSource::Rest) => { *model = Setting::NotSet; @@ -604,6 +1030,8 @@ fn apply_default_for_source( *request = Setting::Reset; *response = Setting::Reset; *headers = Setting::Reset; + *search_embedder = Setting::NotSet; + *indexing_embedder = Setting::NotSet; } Setting::Set(EmbedderSource::UserProvided) => { *model = Setting::NotSet; @@ -616,148 +1044,374 @@ fn apply_default_for_source( *document_template = Setting::NotSet; *document_template_max_bytes = Setting::NotSet; *headers = Setting::NotSet; + *search_embedder = Setting::NotSet; + *indexing_embedder = Setting::NotSet; + } + Setting::Set(EmbedderSource::Composite) => { + *model = Setting::NotSet; + *revision = Setting::NotSet; + *pooling = Setting::NotSet; + *dimensions = Setting::NotSet; + *url = Setting::NotSet; + *request = Setting::NotSet; + *response = Setting::NotSet; + *document_template = Setting::NotSet; + *document_template_max_bytes = Setting::NotSet; + *headers = Setting::NotSet; + *search_embedder = Setting::Reset; + *indexing_embedder = Setting::Reset; } Setting::NotSet => {} } } -pub fn check_set( - key: &Setting, - field: &'static str, - source: EmbedderSource, - embedder_name: &str, -) -> Result<(), UserError> { - if matches!(key, Setting::Set(_)) { - Ok(()) - } else { - Err(UserError::MissingFieldForSource { - field, - source_: source, - embedder_name: embedder_name.to_owned(), - }) +pub(crate) enum FieldStatus { + Mandatory, + Allowed, + Disallowed, +} + +#[derive(Debug, Clone, Copy)] +pub enum NestingContext { + NotNested, + Search, + Indexing, +} + +impl NestingContext { + pub fn embedder_name_with_context(&self, embedder_name: &str) -> String { + match self { + NestingContext::NotNested => embedder_name.to_string(), + NestingContext::Search => format!("{embedder_name}.searchEmbedder"), + NestingContext::Indexing => format!("{embedder_name}.indexingEmbedder",), + } + } + + pub fn in_context(&self) -> &'static str { + match self { + NestingContext::NotNested => "", + NestingContext::Search => " for the search embedder", + NestingContext::Indexing => " for the indexing embedder", + } + } + + pub fn nesting_embedders(&self) -> &'static str { + match self { + NestingContext::NotNested => "", + NestingContext::Search => { + "\n - note: nesting embedders in `searchEmbedder` is not allowed" + } + NestingContext::Indexing => { + "\n - note: nesting embedders in `indexingEmbedder` is not allowed" + } + } + } +} + +#[derive(Debug, Clone, Copy, enum_iterator::Sequence)] +pub enum MetaEmbeddingSetting { + Source, + Model, + Revision, + Pooling, + ApiKey, + Dimensions, + DocumentTemplate, + DocumentTemplateMaxBytes, + Url, + Request, + Response, + Headers, + SearchEmbedder, + IndexingEmbedder, + Distribution, + BinaryQuantized, +} + +impl MetaEmbeddingSetting { + pub(crate) fn name(&self) -> &'static str { + use MetaEmbeddingSetting::*; + match self { + Source => "source", + Model => "model", + Revision => "revision", + Pooling => "pooling", + ApiKey => "apiKey", + Dimensions => "dimensions", + DocumentTemplate => "documentTemplate", + DocumentTemplateMaxBytes => "documentTemplateMaxBytes", + Url => "url", + Request => "request", + Response => "response", + Headers => "headers", + SearchEmbedder => "searchEmbedder", + IndexingEmbedder => "indexingEmbedder", + Distribution => "distribution", + BinaryQuantized => "binaryQuantized", + } } } impl EmbeddingSettings { - pub const SOURCE: &'static str = "source"; - pub const MODEL: &'static str = "model"; - pub const REVISION: &'static str = "revision"; - pub const POOLING: &'static str = "pooling"; - pub const API_KEY: &'static str = "apiKey"; - pub const DIMENSIONS: &'static str = "dimensions"; - pub const DOCUMENT_TEMPLATE: &'static str = "documentTemplate"; - pub const DOCUMENT_TEMPLATE_MAX_BYTES: &'static str = "documentTemplateMaxBytes"; + #[allow(clippy::too_many_arguments)] + pub(crate) fn check_settings( + embedder_name: &str, + source: EmbedderSource, + context: NestingContext, + model: &Setting, + revision: &Setting, + pooling: &Setting, + dimensions: &Setting, + api_key: &Setting, + url: &Setting, + request: &Setting, + response: &Setting, + document_template: &Setting, + document_template_max_bytes: &Setting, + headers: &Setting>, + search_embedder: &Setting, + indexing_embedder: &Setting, + binary_quantized: &Setting, + distribution: &Setting, + ) -> Result<(), UserError> { + Self::check_setting(embedder_name, source, MetaEmbeddingSetting::Model, context, model)?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Revision, + context, + revision, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Pooling, + context, + pooling, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Dimensions, + context, + dimensions, + )?; + Self::check_setting(embedder_name, source, MetaEmbeddingSetting::ApiKey, context, api_key)?; + Self::check_setting(embedder_name, source, MetaEmbeddingSetting::Url, context, url)?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Request, + context, + request, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Response, + context, + response, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::DocumentTemplate, + context, + document_template, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::DocumentTemplateMaxBytes, + context, + document_template_max_bytes, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Headers, + context, + headers, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::SearchEmbedder, + context, + search_embedder, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::IndexingEmbedder, + context, + indexing_embedder, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::BinaryQuantized, + context, + binary_quantized, + )?; + Self::check_setting( + embedder_name, + source, + MetaEmbeddingSetting::Distribution, + context, + distribution, + ) + } - pub const URL: &'static str = "url"; - pub const REQUEST: &'static str = "request"; - pub const RESPONSE: &'static str = "response"; - pub const HEADERS: &'static str = "headers"; + pub(crate) fn allowed_sources_for_field( + field: MetaEmbeddingSetting, + context: NestingContext, + ) -> Vec { + enum_iterator::all() + .filter(|source| { + !matches!(Self::field_status(*source, field, context), FieldStatus::Disallowed) + }) + .collect() + } - pub const DISTRIBUTION: &'static str = "distribution"; + pub(crate) fn allowed_fields_for_source( + source: EmbedderSource, + context: NestingContext, + ) -> Vec<&'static str> { + enum_iterator::all() + .filter(|field| { + !matches!(Self::field_status(source, *field, context), FieldStatus::Disallowed) + }) + .map(|field| field.name()) + .collect() + } - pub const BINARY_QUANTIZED: &'static str = "binaryQuantized"; - - pub fn allowed_sources_for_field(field: &'static str) -> &'static [EmbedderSource] { - match field { - Self::SOURCE => &[ - EmbedderSource::HuggingFace, - EmbedderSource::OpenAi, - EmbedderSource::UserProvided, - EmbedderSource::Rest, - EmbedderSource::Ollama, - ], - Self::MODEL => { - &[EmbedderSource::HuggingFace, EmbedderSource::OpenAi, EmbedderSource::Ollama] - } - Self::REVISION => &[EmbedderSource::HuggingFace], - Self::POOLING => &[EmbedderSource::HuggingFace], - Self::API_KEY => { - &[EmbedderSource::OpenAi, EmbedderSource::Ollama, EmbedderSource::Rest] - } - Self::DIMENSIONS => &[ - EmbedderSource::OpenAi, - EmbedderSource::UserProvided, - EmbedderSource::Ollama, - EmbedderSource::Rest, - ], - Self::DOCUMENT_TEMPLATE | Self::DOCUMENT_TEMPLATE_MAX_BYTES => &[ - EmbedderSource::HuggingFace, - EmbedderSource::OpenAi, - EmbedderSource::Ollama, - EmbedderSource::Rest, - ], - Self::URL => &[EmbedderSource::Ollama, EmbedderSource::Rest, EmbedderSource::OpenAi], - Self::REQUEST => &[EmbedderSource::Rest], - Self::RESPONSE => &[EmbedderSource::Rest], - Self::HEADERS => &[EmbedderSource::Rest], - Self::DISTRIBUTION => &[ - EmbedderSource::HuggingFace, - EmbedderSource::Ollama, - EmbedderSource::OpenAi, - EmbedderSource::Rest, - EmbedderSource::UserProvided, - ], - Self::BINARY_QUANTIZED => &[ - EmbedderSource::HuggingFace, - EmbedderSource::Ollama, - EmbedderSource::OpenAi, - EmbedderSource::Rest, - EmbedderSource::UserProvided, - ], - _other => unreachable!("unknown field"), + fn check_setting( + embedder_name: &str, + source: EmbedderSource, + field: MetaEmbeddingSetting, + context: NestingContext, + setting: &Setting, + ) -> Result<(), UserError> { + match (Self::field_status(source, field, context), setting) { + (FieldStatus::Mandatory, Setting::Set(_)) + | (FieldStatus::Allowed, _) + | (FieldStatus::Disallowed, Setting::NotSet) => Ok(()), + (FieldStatus::Disallowed, _) => Err(UserError::InvalidFieldForSource { + embedder_name: context.embedder_name_with_context(embedder_name), + source_: source, + context, + field, + }), + (FieldStatus::Mandatory, _) => Err(UserError::MissingFieldForSource { + field: field.name(), + source_: source, + embedder_name: embedder_name.to_owned(), + }), } } - pub fn allowed_fields_for_source(source: EmbedderSource) -> &'static [&'static str] { - match source { - EmbedderSource::OpenAi => &[ - Self::SOURCE, - Self::MODEL, - Self::API_KEY, - Self::DOCUMENT_TEMPLATE, - Self::DOCUMENT_TEMPLATE_MAX_BYTES, - Self::DIMENSIONS, - Self::DISTRIBUTION, - Self::URL, - Self::BINARY_QUANTIZED, - ], - EmbedderSource::HuggingFace => &[ - Self::SOURCE, - Self::MODEL, - Self::REVISION, - Self::POOLING, - Self::DOCUMENT_TEMPLATE, - Self::DOCUMENT_TEMPLATE_MAX_BYTES, - Self::DISTRIBUTION, - Self::BINARY_QUANTIZED, - ], - EmbedderSource::Ollama => &[ - Self::SOURCE, - Self::MODEL, - Self::DOCUMENT_TEMPLATE, - Self::DOCUMENT_TEMPLATE_MAX_BYTES, - Self::URL, - Self::API_KEY, - Self::DIMENSIONS, - Self::DISTRIBUTION, - Self::BINARY_QUANTIZED, - ], - EmbedderSource::UserProvided => { - &[Self::SOURCE, Self::DIMENSIONS, Self::DISTRIBUTION, Self::BINARY_QUANTIZED] + pub(crate) fn field_status( + source: EmbedderSource, + field: MetaEmbeddingSetting, + context: NestingContext, + ) -> FieldStatus { + use EmbedderSource::*; + use MetaEmbeddingSetting::*; + use NestingContext::*; + match (source, field, context) { + (_, Distribution | BinaryQuantized, NotNested) => FieldStatus::Allowed, + (_, Distribution | BinaryQuantized, _) => FieldStatus::Disallowed, + (_, DocumentTemplate | DocumentTemplateMaxBytes, Search) => FieldStatus::Disallowed, + ( + OpenAi, + Source + | Model + | ApiKey + | DocumentTemplate + | DocumentTemplateMaxBytes + | Dimensions + | Url, + _, + ) => FieldStatus::Allowed, + ( + OpenAi, + Revision | Pooling | Request | Response | Headers | SearchEmbedder + | IndexingEmbedder, + _, + ) => FieldStatus::Disallowed, + ( + HuggingFace, + Source | Model | Revision | Pooling | DocumentTemplate | DocumentTemplateMaxBytes, + _, + ) => FieldStatus::Allowed, + ( + HuggingFace, + ApiKey | Dimensions | Url | Request | Response | Headers | SearchEmbedder + | IndexingEmbedder, + _, + ) => FieldStatus::Disallowed, + (Ollama, Model, _) => FieldStatus::Mandatory, + ( + Ollama, + Source | DocumentTemplate | DocumentTemplateMaxBytes | Url | ApiKey | Dimensions, + _, + ) => FieldStatus::Allowed, + ( + Ollama, + Revision | Pooling | Request | Response | Headers | SearchEmbedder + | IndexingEmbedder, + _, + ) => FieldStatus::Disallowed, + (UserProvided, Dimensions, _) => FieldStatus::Mandatory, + (UserProvided, Source, _) => FieldStatus::Allowed, + ( + UserProvided, + Model + | Revision + | Pooling + | ApiKey + | DocumentTemplate + | DocumentTemplateMaxBytes + | Url + | Request + | Response + | Headers + | SearchEmbedder + | IndexingEmbedder, + _, + ) => FieldStatus::Disallowed, + (Rest, Url | Request | Response, _) => FieldStatus::Mandatory, + ( + Rest, + Source + | ApiKey + | Dimensions + | DocumentTemplate + | DocumentTemplateMaxBytes + | Headers, + _, + ) => FieldStatus::Allowed, + (Rest, Model | Revision | Pooling | SearchEmbedder | IndexingEmbedder, _) => { + FieldStatus::Disallowed } - EmbedderSource::Rest => &[ - Self::SOURCE, - Self::API_KEY, - Self::DIMENSIONS, - Self::DOCUMENT_TEMPLATE, - Self::DOCUMENT_TEMPLATE_MAX_BYTES, - Self::URL, - Self::REQUEST, - Self::RESPONSE, - Self::HEADERS, - Self::DISTRIBUTION, - Self::BINARY_QUANTIZED, - ], + (Composite, SearchEmbedder | IndexingEmbedder, _) => FieldStatus::Mandatory, + (Composite, Source, _) => FieldStatus::Allowed, + ( + Composite, + Model + | Revision + | Pooling + | ApiKey + | Dimensions + | DocumentTemplate + | DocumentTemplateMaxBytes + | Url + | Request + | Response + | Headers, + _, + ) => FieldStatus::Disallowed, } } @@ -781,9 +1435,45 @@ impl EmbeddingSettings { *model = Setting::Set(openai::EmbeddingModel::default().name().to_owned()) } } + + pub(crate) fn check_nested_source( + embedder_name: &str, + source: EmbedderSource, + context: NestingContext, + ) -> Result<(), UserError> { + match (context, source) { + (NestingContext::NotNested, _) => Ok(()), + ( + NestingContext::Search | NestingContext::Indexing, + EmbedderSource::Composite | EmbedderSource::UserProvided, + ) => Err(UserError::InvalidSourceForNested { + embedder_name: context.embedder_name_with_context(embedder_name), + source_: source, + }), + ( + NestingContext::Search | NestingContext::Indexing, + EmbedderSource::OpenAi + | EmbedderSource::HuggingFace + | EmbedderSource::Ollama + | EmbedderSource::Rest, + ) => Ok(()), + } + } } -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)] +#[derive( + Debug, + Clone, + Copy, + Default, + Serialize, + Deserialize, + PartialEq, + Eq, + Deserr, + ToSchema, + enum_iterator::Sequence, +)] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[deserr(rename_all = camelCase, deny_unknown_fields)] pub enum EmbedderSource { @@ -793,6 +1483,7 @@ pub enum EmbedderSource { Ollama, UserProvided, Rest, + Composite, } impl std::fmt::Display for EmbedderSource { @@ -803,125 +1494,311 @@ impl std::fmt::Display for EmbedderSource { EmbedderSource::UserProvided => "userProvided", EmbedderSource::Ollama => "ollama", EmbedderSource::Rest => "rest", + EmbedderSource::Composite => "composite", }; f.write_str(s) } } +impl EmbeddingSettings { + fn from_hugging_face( + super::hf::EmbedderOptions { + model, + revision, + distribution, + pooling, + }: super::hf::EmbedderOptions, + document_template: Setting, + document_template_max_bytes: Setting, + quantized: Option, + ) -> Self { + Self { + source: Setting::Set(EmbedderSource::HuggingFace), + model: Setting::Set(model), + revision: Setting::some_or_not_set(revision), + pooling: Setting::Set(pooling), + api_key: Setting::NotSet, + dimensions: Setting::NotSet, + document_template, + document_template_max_bytes, + url: Setting::NotSet, + request: Setting::NotSet, + response: Setting::NotSet, + headers: Setting::NotSet, + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, + distribution: Setting::some_or_not_set(distribution), + binary_quantized: Setting::some_or_not_set(quantized), + } + } + + fn from_openai( + super::openai::EmbedderOptions { + url, + api_key, + embedding_model, + dimensions, + distribution, + }: super::openai::EmbedderOptions, + document_template: Setting, + document_template_max_bytes: Setting, + quantized: Option, + ) -> Self { + Self { + source: Setting::Set(EmbedderSource::OpenAi), + model: Setting::Set(embedding_model.name().to_owned()), + revision: Setting::NotSet, + pooling: Setting::NotSet, + api_key: Setting::some_or_not_set(api_key), + dimensions: Setting::some_or_not_set(dimensions), + document_template, + document_template_max_bytes, + url: Setting::some_or_not_set(url), + request: Setting::NotSet, + response: Setting::NotSet, + headers: Setting::NotSet, + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, + distribution: Setting::some_or_not_set(distribution), + binary_quantized: Setting::some_or_not_set(quantized), + } + } + + fn from_ollama( + super::ollama::EmbedderOptions { + embedding_model, + url, + api_key, + distribution, + dimensions, + }: super::ollama::EmbedderOptions, + document_template: Setting, + document_template_max_bytes: Setting, + quantized: Option, + ) -> Self { + Self { + source: Setting::Set(EmbedderSource::Ollama), + model: Setting::Set(embedding_model), + revision: Setting::NotSet, + pooling: Setting::NotSet, + api_key: Setting::some_or_not_set(api_key), + dimensions: Setting::some_or_not_set(dimensions), + document_template, + document_template_max_bytes, + url: Setting::some_or_not_set(url), + request: Setting::NotSet, + response: Setting::NotSet, + headers: Setting::NotSet, + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, + distribution: Setting::some_or_not_set(distribution), + binary_quantized: Setting::some_or_not_set(quantized), + } + } + + fn from_user_provided( + super::manual::EmbedderOptions { dimensions, distribution }: super::manual::EmbedderOptions, + quantized: Option, + ) -> Self { + Self { + source: Setting::Set(EmbedderSource::UserProvided), + model: Setting::NotSet, + revision: Setting::NotSet, + pooling: Setting::NotSet, + api_key: Setting::NotSet, + dimensions: Setting::Set(dimensions), + document_template: Setting::NotSet, + document_template_max_bytes: Setting::NotSet, + url: Setting::NotSet, + request: Setting::NotSet, + response: Setting::NotSet, + headers: Setting::NotSet, + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, + distribution: Setting::some_or_not_set(distribution), + binary_quantized: Setting::some_or_not_set(quantized), + } + } + + fn from_rest( + super::rest::EmbedderOptions { + api_key, + dimensions, + url, + request, + response, + distribution, + headers, + }: super::rest::EmbedderOptions, + document_template: Setting, + document_template_max_bytes: Setting, + quantized: Option, + ) -> Self { + Self { + source: Setting::Set(EmbedderSource::Rest), + model: Setting::NotSet, + revision: Setting::NotSet, + pooling: Setting::NotSet, + api_key: Setting::some_or_not_set(api_key), + dimensions: Setting::some_or_not_set(dimensions), + document_template, + document_template_max_bytes, + url: Setting::Set(url), + request: Setting::Set(request), + response: Setting::Set(response), + distribution: Setting::some_or_not_set(distribution), + headers: Setting::Set(headers), + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, + binary_quantized: Setting::some_or_not_set(quantized), + } + } +} + impl From for EmbeddingSettings { fn from(value: EmbeddingConfig) -> Self { let EmbeddingConfig { embedder_options, prompt, quantized } = value; let document_template_max_bytes = Setting::Set(prompt.max_bytes.unwrap_or(default_max_bytes()).get()); match embedder_options { - super::EmbedderOptions::HuggingFace(super::hf::EmbedderOptions { - model, - revision, - distribution, - pooling, - }) => Self { - source: Setting::Set(EmbedderSource::HuggingFace), - model: Setting::Set(model), - revision: Setting::some_or_not_set(revision), - pooling: Setting::Set(pooling), - api_key: Setting::NotSet, - dimensions: Setting::NotSet, - document_template: Setting::Set(prompt.template), + super::EmbedderOptions::HuggingFace(options) => Self::from_hugging_face( + options, + Setting::Set(prompt.template), document_template_max_bytes, - url: Setting::NotSet, - request: Setting::NotSet, - response: Setting::NotSet, - headers: Setting::NotSet, - distribution: Setting::some_or_not_set(distribution), - binary_quantized: Setting::some_or_not_set(quantized), - }, - super::EmbedderOptions::OpenAi(super::openai::EmbedderOptions { - url, - api_key, - embedding_model, - dimensions, - distribution, - }) => Self { - source: Setting::Set(EmbedderSource::OpenAi), - model: Setting::Set(embedding_model.name().to_owned()), - revision: Setting::NotSet, - pooling: Setting::NotSet, - api_key: Setting::some_or_not_set(api_key), - dimensions: Setting::some_or_not_set(dimensions), - document_template: Setting::Set(prompt.template), + quantized, + ), + super::EmbedderOptions::OpenAi(options) => Self::from_openai( + options, + Setting::Set(prompt.template), document_template_max_bytes, - url: Setting::some_or_not_set(url), - request: Setting::NotSet, - response: Setting::NotSet, - headers: Setting::NotSet, - distribution: Setting::some_or_not_set(distribution), - binary_quantized: Setting::some_or_not_set(quantized), - }, - super::EmbedderOptions::Ollama(super::ollama::EmbedderOptions { - embedding_model, - url, - api_key, - distribution, - dimensions, - }) => Self { - source: Setting::Set(EmbedderSource::Ollama), - model: Setting::Set(embedding_model), - revision: Setting::NotSet, - pooling: Setting::NotSet, - api_key: Setting::some_or_not_set(api_key), - dimensions: Setting::some_or_not_set(dimensions), - document_template: Setting::Set(prompt.template), + quantized, + ), + super::EmbedderOptions::Ollama(options) => Self::from_ollama( + options, + Setting::Set(prompt.template), document_template_max_bytes, - url: Setting::some_or_not_set(url), - request: Setting::NotSet, - response: Setting::NotSet, - headers: Setting::NotSet, - distribution: Setting::some_or_not_set(distribution), - binary_quantized: Setting::some_or_not_set(quantized), - }, - super::EmbedderOptions::UserProvided(super::manual::EmbedderOptions { - dimensions, - distribution, + quantized, + ), + super::EmbedderOptions::UserProvided(options) => { + Self::from_user_provided(options, quantized) + } + super::EmbedderOptions::Rest(options) => Self::from_rest( + options, + Setting::Set(prompt.template), + document_template_max_bytes, + quantized, + ), + super::EmbedderOptions::Composite(super::composite::EmbedderOptions { + search, + index, }) => Self { - source: Setting::Set(EmbedderSource::UserProvided), + source: Setting::Set(EmbedderSource::Composite), model: Setting::NotSet, revision: Setting::NotSet, pooling: Setting::NotSet, api_key: Setting::NotSet, - dimensions: Setting::Set(dimensions), + dimensions: Setting::NotSet, + binary_quantized: Setting::some_or_not_set(quantized), document_template: Setting::NotSet, document_template_max_bytes: Setting::NotSet, url: Setting::NotSet, request: Setting::NotSet, response: Setting::NotSet, headers: Setting::NotSet, - distribution: Setting::some_or_not_set(distribution), - binary_quantized: Setting::some_or_not_set(quantized), + distribution: Setting::some_or_not_set(search.distribution()), + search_embedder: Setting::Set(SubEmbeddingSettings::from_options( + search, + Setting::NotSet, + Setting::NotSet, + )), + indexing_embedder: Setting::Set(SubEmbeddingSettings::from_options( + index, + Setting::Set(prompt.template), + document_template_max_bytes, + )), }, - super::EmbedderOptions::Rest(super::rest::EmbedderOptions { - api_key, - dimensions, - url, - request, - response, - distribution, - headers, - }) => Self { - source: Setting::Set(EmbedderSource::Rest), - model: Setting::NotSet, - revision: Setting::NotSet, - pooling: Setting::NotSet, - api_key: Setting::some_or_not_set(api_key), - dimensions: Setting::some_or_not_set(dimensions), - document_template: Setting::Set(prompt.template), + } + } +} + +impl SubEmbeddingSettings { + fn from_options( + options: SubEmbedderOptions, + document_template: Setting, + document_template_max_bytes: Setting, + ) -> Self { + let settings = match options { + SubEmbedderOptions::HuggingFace(embedder_options) => { + EmbeddingSettings::from_hugging_face( + embedder_options, + document_template, + document_template_max_bytes, + None, + ) + } + SubEmbedderOptions::OpenAi(embedder_options) => EmbeddingSettings::from_openai( + embedder_options, + document_template, document_template_max_bytes, - url: Setting::Set(url), - request: Setting::Set(request), - response: Setting::Set(response), - distribution: Setting::some_or_not_set(distribution), - headers: Setting::Set(headers), - binary_quantized: Setting::some_or_not_set(quantized), - }, + None, + ), + SubEmbedderOptions::Ollama(embedder_options) => EmbeddingSettings::from_ollama( + embedder_options, + document_template, + document_template_max_bytes, + None, + ), + SubEmbedderOptions::UserProvided(embedder_options) => { + EmbeddingSettings::from_user_provided(embedder_options, None) + } + SubEmbedderOptions::Rest(embedder_options) => EmbeddingSettings::from_rest( + embedder_options, + document_template, + document_template_max_bytes, + None, + ), + }; + settings.into() + } +} + +impl From for SubEmbeddingSettings { + fn from(value: EmbeddingSettings) -> Self { + let EmbeddingSettings { + source, + model, + revision, + pooling, + api_key, + dimensions, + document_template, + document_template_max_bytes, + url, + request, + response, + headers, + binary_quantized: _, + search_embedder: _, + indexing_embedder: _, + distribution: _, + } = value; + Self { + source, + model, + revision, + pooling, + api_key, + dimensions, + document_template, + document_template_max_bytes, + url, + request, + response, + headers, + distribution: Setting::NotSet, + binary_quantized: Setting::NotSet, + search_embedder: Setting::NotSet, + indexing_embedder: Setting::NotSet, } } } @@ -944,88 +1821,26 @@ impl From for EmbeddingConfig { distribution, headers, binary_quantized, + search_embedder, + mut indexing_embedder, } = value; this.quantized = binary_quantized.set(); - - if let Some(source) = source.set() { - match source { - EmbedderSource::OpenAi => { - let mut options = super::openai::EmbedderOptions::with_default_model(None); - if let Some(model) = model.set() { - if let Some(model) = super::openai::EmbeddingModel::from_name(&model) { - options.embedding_model = model; - } - } - if let Some(url) = url.set() { - options.url = Some(url); - } - if let Some(api_key) = api_key.set() { - options.api_key = Some(api_key); - } - if let Some(dimensions) = dimensions.set() { - options.dimensions = Some(dimensions); - } - options.distribution = distribution.set(); - this.embedder_options = super::EmbedderOptions::OpenAi(options); - } - EmbedderSource::Ollama => { - let mut options: ollama::EmbedderOptions = - super::ollama::EmbedderOptions::with_default_model( - api_key.set(), - url.set(), - dimensions.set(), - ); - if let Some(model) = model.set() { - options.embedding_model = model; - } - - options.distribution = distribution.set(); - this.embedder_options = super::EmbedderOptions::Ollama(options); - } - EmbedderSource::HuggingFace => { - let mut options = super::hf::EmbedderOptions::default(); - if let Some(model) = model.set() { - options.model = model; - // Reset the revision if we are setting the model. - // This allows the following: - // "huggingFace": {} -> default model with default revision - // "huggingFace": { "model": "name-of-the-default-model" } -> default model without a revision - // "huggingFace": { "model": "some-other-model" } -> most importantly, other model without a revision - options.revision = None; - } - if let Some(revision) = revision.set() { - options.revision = Some(revision); - } - if let Some(pooling) = pooling.set() { - options.pooling = pooling; - } - options.distribution = distribution.set(); - this.embedder_options = super::EmbedderOptions::HuggingFace(options); - } - EmbedderSource::UserProvided => { - this.embedder_options = - super::EmbedderOptions::UserProvided(super::manual::EmbedderOptions { - dimensions: dimensions.set().unwrap(), - distribution: distribution.set(), - }); - } - EmbedderSource::Rest => { - this.embedder_options = - super::EmbedderOptions::Rest(super::rest::EmbedderOptions { - api_key: api_key.set(), - dimensions: dimensions.set(), - url: url.set().unwrap(), - request: request.set().unwrap(), - response: response.set().unwrap(), - distribution: distribution.set(), - headers: headers.set().unwrap_or_default(), - }) - } + if let Some((template, document_template_max_bytes)) = + match (document_template, &mut indexing_embedder) { + (Setting::Set(template), _) => Some((template, document_template_max_bytes)), + // retrieve the prompt from the indexing embedder in case of a composite embedder + ( + _, + Setting::Set(SubEmbeddingSettings { + document_template: Setting::Set(document_template), + document_template_max_bytes, + .. + }), + ) => Some((std::mem::take(document_template), *document_template_max_bytes)), + _ => None, } - } - - if let Setting::Set(template) = document_template { + { let max_bytes = document_template_max_bytes .set() .and_then(NonZeroUsize::new) @@ -1034,6 +1849,208 @@ impl From for EmbeddingConfig { this.prompt = PromptData { template, max_bytes: Some(max_bytes) } } + if let Some(source) = source.set() { + this.embedder_options = match source { + EmbedderSource::OpenAi => { + SubEmbedderOptions::openai(model, url, api_key, dimensions, distribution).into() + } + EmbedderSource::Ollama => { + SubEmbedderOptions::ollama(model, url, api_key, dimensions, distribution).into() + } + EmbedderSource::HuggingFace => { + SubEmbedderOptions::hugging_face(model, revision, pooling, distribution).into() + } + EmbedderSource::UserProvided => { + SubEmbedderOptions::user_provided(dimensions.set().unwrap(), distribution) + .into() + } + EmbedderSource::Rest => SubEmbedderOptions::rest( + url.set().unwrap(), + api_key, + request.set().unwrap(), + response.set().unwrap(), + headers, + dimensions, + distribution, + ) + .into(), + EmbedderSource::Composite => { + super::EmbedderOptions::Composite(super::composite::EmbedderOptions { + // it is important to give the distribution to the search here, as this is from where we'll retrieve it + search: SubEmbedderOptions::from_settings( + search_embedder.set().unwrap(), + distribution, + ), + index: SubEmbedderOptions::from_settings( + indexing_embedder.set().unwrap(), + Setting::NotSet, + ), + }) + } + }; + } + this } } + +impl SubEmbedderOptions { + fn from_settings( + settings: SubEmbeddingSettings, + distribution: Setting, + ) -> Self { + let SubEmbeddingSettings { + source, + model, + revision, + pooling, + api_key, + dimensions, + // retrieved by the EmbeddingConfig + document_template: _, + document_template_max_bytes: _, + url, + request, + response, + headers, + // phony parameters + distribution: _, + binary_quantized: _, + search_embedder: _, + indexing_embedder: _, + } = settings; + + match source.set().unwrap() { + EmbedderSource::OpenAi => Self::openai(model, url, api_key, dimensions, distribution), + EmbedderSource::HuggingFace => { + Self::hugging_face(model, revision, pooling, distribution) + } + EmbedderSource::Ollama => Self::ollama(model, url, api_key, dimensions, distribution), + EmbedderSource::UserProvided => { + Self::user_provided(dimensions.set().unwrap(), distribution) + } + EmbedderSource::Rest => Self::rest( + url.set().unwrap(), + api_key, + request.set().unwrap(), + response.set().unwrap(), + headers, + dimensions, + distribution, + ), + EmbedderSource::Composite => panic!("nested composite embedders"), + } + } + + fn openai( + model: Setting, + url: Setting, + api_key: Setting, + dimensions: Setting, + distribution: Setting, + ) -> Self { + let mut options = super::openai::EmbedderOptions::with_default_model(None); + if let Some(model) = model.set() { + if let Some(model) = super::openai::EmbeddingModel::from_name(&model) { + options.embedding_model = model; + } + } + if let Some(url) = url.set() { + options.url = Some(url); + } + if let Some(api_key) = api_key.set() { + options.api_key = Some(api_key); + } + if let Some(dimensions) = dimensions.set() { + options.dimensions = Some(dimensions); + } + options.distribution = distribution.set(); + SubEmbedderOptions::OpenAi(options) + } + fn hugging_face( + model: Setting, + revision: Setting, + pooling: Setting, + distribution: Setting, + ) -> Self { + let mut options = super::hf::EmbedderOptions::default(); + if let Some(model) = model.set() { + options.model = model; + // Reset the revision if we are setting the model. + // This allows the following: + // "huggingFace": {} -> default model with default revision + // "huggingFace": { "model": "name-of-the-default-model" } -> default model without a revision + // "huggingFace": { "model": "some-other-model" } -> most importantly, other model without a revision + options.revision = None; + } + if let Some(revision) = revision.set() { + options.revision = Some(revision); + } + if let Some(pooling) = pooling.set() { + options.pooling = pooling; + } + options.distribution = distribution.set(); + SubEmbedderOptions::HuggingFace(options) + } + fn user_provided(dimensions: usize, distribution: Setting) -> Self { + Self::UserProvided(super::manual::EmbedderOptions { + dimensions, + distribution: distribution.set(), + }) + } + fn rest( + url: String, + api_key: Setting, + request: serde_json::Value, + response: serde_json::Value, + headers: Setting>, + dimensions: Setting, + distribution: Setting, + ) -> Self { + Self::Rest(super::rest::EmbedderOptions { + api_key: api_key.set(), + dimensions: dimensions.set(), + url, + request, + response, + distribution: distribution.set(), + headers: headers.set().unwrap_or_default(), + }) + } + fn ollama( + model: Setting, + url: Setting, + api_key: Setting, + dimensions: Setting, + distribution: Setting, + ) -> Self { + let mut options: ollama::EmbedderOptions = + super::ollama::EmbedderOptions::with_default_model( + api_key.set(), + url.set(), + dimensions.set(), + ); + if let Some(model) = model.set() { + options.embedding_model = model; + } + + options.distribution = distribution.set(); + SubEmbedderOptions::Ollama(options) + } +} + +impl From for EmbedderOptions { + fn from(value: SubEmbedderOptions) -> Self { + match value { + SubEmbedderOptions::HuggingFace(embedder_options) => { + Self::HuggingFace(embedder_options) + } + SubEmbedderOptions::OpenAi(embedder_options) => Self::OpenAi(embedder_options), + SubEmbedderOptions::Ollama(embedder_options) => Self::Ollama(embedder_options), + SubEmbedderOptions::UserProvided(embedder_options) => { + Self::UserProvided(embedder_options) + } + SubEmbedderOptions::Rest(embedder_options) => Self::Rest(embedder_options), + } + } +}