mirror of
https://github.com/meilisearch/meilisearch.git
synced 2024-11-22 18:17:39 +08:00
remove the uneeded files after the rebase
This commit is contained in:
Tamo
parent
99a9fde37f
commit
a1f228f662
@ -1,73 +0,0 @@
|
|||||||
mod v1_10;
|
|
||||||
mod v1_11;
|
|
||||||
mod v1_9;
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
|
|
||||||
use anyhow::{bail, Context};
|
|
||||||
use meilisearch_types::versioning::create_version_file;
|
|
||||||
|
|
||||||
use v1_10::v1_9_to_v1_10;
|
|
||||||
|
|
||||||
use crate::upgrade::v1_11::v1_10_to_v1_11;
|
|
||||||
|
|
||||||
pub struct OfflineUpgrade {
|
|
||||||
pub db_path: PathBuf,
|
|
||||||
pub current_version: (String, String, String),
|
|
||||||
pub target_version: (String, String, String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl OfflineUpgrade {
|
|
||||||
pub fn upgrade(self) -> anyhow::Result<()> {
|
|
||||||
let upgrade_list = [
|
|
||||||
(v1_9_to_v1_10 as fn(&Path) -> Result<(), anyhow::Error>, "1", "10", "0"),
|
|
||||||
(v1_10_to_v1_11, "1", "11", "0"),
|
|
||||||
];
|
|
||||||
|
|
||||||
let (current_major, current_minor, current_patch) = &self.current_version;
|
|
||||||
|
|
||||||
let start_at = match (
|
|
||||||
current_major.as_str(),
|
|
||||||
current_minor.as_str(),
|
|
||||||
current_patch.as_str(),
|
|
||||||
) {
|
|
||||||
("1", "9", _) => 0,
|
|
||||||
("1", "10", _) => 1,
|
|
||||||
_ => {
|
|
||||||
bail!("Unsupported current version {current_major}.{current_minor}.{current_patch}. Can only upgrade from v1.9")
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let (target_major, target_minor, target_patch) = &self.target_version;
|
|
||||||
|
|
||||||
let ends_at = match (target_major.as_str(), target_minor.as_str(), target_patch.as_str()) {
|
|
||||||
("1", "10", _) => 0,
|
|
||||||
("1", "11", _) => 1,
|
|
||||||
(major, _, _) if major.starts_with('v') => {
|
|
||||||
bail!("Target version must not starts with a `v`. Instead of writing `v1.9.0` write `1.9.0` for example.")
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
bail!("Unsupported target version {target_major}.{target_minor}.{target_patch}. Can only upgrade to v1.11")
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
println!("Starting the upgrade from {current_major}.{current_minor}.{current_patch} to {target_major}.{target_minor}.{target_patch}");
|
|
||||||
|
|
||||||
#[allow(clippy::needless_range_loop)]
|
|
||||||
for index in start_at..=ends_at {
|
|
||||||
let (func, major, minor, patch) = upgrade_list[index];
|
|
||||||
(func)(&self.db_path)?;
|
|
||||||
println!("Done");
|
|
||||||
// We're writing the version file just in case an issue arise _while_ upgrading.
|
|
||||||
// We don't want the DB to fail in an unknown state.
|
|
||||||
println!("Writing VERSION file");
|
|
||||||
|
|
||||||
create_version_file(&self.db_path, major, minor, patch)
|
|
||||||
.context("while writing VERSION file after the upgrade")?;
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("Success");
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,289 +0,0 @@
|
|||||||
use anyhow::bail;
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use anyhow::Context;
|
|
||||||
use meilisearch_types::{
|
|
||||||
heed::{
|
|
||||||
types::{SerdeJson, Str},
|
|
||||||
Database, Env, EnvOpenOptions, RoTxn, RwTxn, Unspecified,
|
|
||||||
},
|
|
||||||
milli::index::{db_name, main_key},
|
|
||||||
};
|
|
||||||
|
|
||||||
use crate::{try_opening_database, try_opening_poly_database, uuid_codec::UuidCodec};
|
|
||||||
|
|
||||||
use super::v1_9;
|
|
||||||
|
|
||||||
pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
|
|
||||||
|
|
||||||
/// The statistics that can be computed from an `Index` object.
|
|
||||||
#[derive(serde::Serialize, serde::Deserialize, Debug)]
|
|
||||||
pub struct IndexStats {
|
|
||||||
/// Number of documents in the index.
|
|
||||||
pub number_of_documents: u64,
|
|
||||||
/// Size taken up by the index' DB, in bytes.
|
|
||||||
///
|
|
||||||
/// This includes the size taken by both the used and free pages of the DB, and as the free pages
|
|
||||||
/// are not returned to the disk after a deletion, this number is typically larger than
|
|
||||||
/// `used_database_size` that only includes the size of the used pages.
|
|
||||||
pub database_size: u64,
|
|
||||||
/// Size taken by the used pages of the index' DB, in bytes.
|
|
||||||
///
|
|
||||||
/// As the DB backend does not return to the disk the pages that are not currently used by the DB,
|
|
||||||
/// this value is typically smaller than `database_size`.
|
|
||||||
pub used_database_size: u64,
|
|
||||||
/// Association of every field name with the number of times it occurs in the documents.
|
|
||||||
pub field_distribution: FieldDistribution,
|
|
||||||
/// Creation date of the index.
|
|
||||||
#[serde(with = "time::serde::rfc3339")]
|
|
||||||
pub created_at: time::OffsetDateTime,
|
|
||||||
/// Date of the last update of the index.
|
|
||||||
#[serde(with = "time::serde::rfc3339")]
|
|
||||||
pub updated_at: time::OffsetDateTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<v1_9::IndexStats> for IndexStats {
|
|
||||||
fn from(
|
|
||||||
v1_9::IndexStats {
|
|
||||||
number_of_documents,
|
|
||||||
database_size,
|
|
||||||
used_database_size,
|
|
||||||
field_distribution,
|
|
||||||
created_at,
|
|
||||||
updated_at,
|
|
||||||
}: v1_9::IndexStats,
|
|
||||||
) -> Self {
|
|
||||||
IndexStats {
|
|
||||||
number_of_documents,
|
|
||||||
database_size,
|
|
||||||
used_database_size,
|
|
||||||
field_distribution,
|
|
||||||
created_at: created_at.0,
|
|
||||||
updated_at: updated_at.0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Serialize, serde::Deserialize)]
|
|
||||||
#[serde(transparent)]
|
|
||||||
pub struct OffsetDateTime(#[serde(with = "time::serde::rfc3339")] pub time::OffsetDateTime);
|
|
||||||
|
|
||||||
fn update_index_stats(
|
|
||||||
index_stats: Database<UuidCodec, Unspecified>,
|
|
||||||
index_uid: &str,
|
|
||||||
index_uuid: uuid::Uuid,
|
|
||||||
sched_wtxn: &mut RwTxn,
|
|
||||||
) -> anyhow::Result<()> {
|
|
||||||
let ctx = || format!("while updating index stats for index `{index_uid}`");
|
|
||||||
|
|
||||||
let stats: Option<&str> = index_stats
|
|
||||||
.remap_data_type::<Str>()
|
|
||||||
.get(sched_wtxn, &index_uuid)
|
|
||||||
.with_context(ctx)
|
|
||||||
.with_context(|| "While reading value")?;
|
|
||||||
dbg!(stats);
|
|
||||||
|
|
||||||
let stats: Option<v1_9::IndexStats> = index_stats
|
|
||||||
.remap_data_type::<SerdeJson<v1_9::IndexStats>>()
|
|
||||||
.get(sched_wtxn, &index_uuid)
|
|
||||||
.with_context(ctx)
|
|
||||||
.with_context(|| "While reading value")?;
|
|
||||||
|
|
||||||
if let Some(stats) = stats {
|
|
||||||
let stats: self::IndexStats = stats.into();
|
|
||||||
|
|
||||||
index_stats
|
|
||||||
.remap_data_type::<SerdeJson<self::IndexStats>>()
|
|
||||||
.put(sched_wtxn, &index_uuid, &stats)
|
|
||||||
.with_context(ctx)
|
|
||||||
.with_context(|| "While writing value")?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn update_date_format(
|
|
||||||
index_uid: &str,
|
|
||||||
index_env: &Env,
|
|
||||||
index_wtxn: &mut RwTxn,
|
|
||||||
) -> anyhow::Result<()> {
|
|
||||||
let main = try_opening_poly_database(index_env, index_wtxn, db_name::MAIN)
|
|
||||||
.with_context(|| format!("while updating date format for index `{index_uid}`"))?;
|
|
||||||
|
|
||||||
date_round_trip(index_wtxn, index_uid, main, main_key::CREATED_AT_KEY)?;
|
|
||||||
date_round_trip(index_wtxn, index_uid, main, main_key::UPDATED_AT_KEY)?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn find_rest_embedders(
|
|
||||||
index_uid: &str,
|
|
||||||
index_env: &Env,
|
|
||||||
index_txn: &RoTxn,
|
|
||||||
) -> anyhow::Result<Vec<String>> {
|
|
||||||
let main = try_opening_poly_database(index_env, index_txn, db_name::MAIN)
|
|
||||||
.with_context(|| format!("while checking REST embedders for index `{index_uid}`"))?;
|
|
||||||
|
|
||||||
let mut rest_embedders = vec![];
|
|
||||||
|
|
||||||
for config in main
|
|
||||||
.remap_types::<Str, SerdeJson<Vec<v1_9::IndexEmbeddingConfig>>>()
|
|
||||||
.get(index_txn, main_key::EMBEDDING_CONFIGS)?
|
|
||||||
.unwrap_or_default()
|
|
||||||
{
|
|
||||||
if let v1_9::EmbedderOptions::Rest(_) = config.config.embedder_options {
|
|
||||||
rest_embedders.push(config.name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(rest_embedders)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn date_round_trip(
|
|
||||||
wtxn: &mut RwTxn,
|
|
||||||
index_uid: &str,
|
|
||||||
db: Database<Unspecified, Unspecified>,
|
|
||||||
key: &str,
|
|
||||||
) -> anyhow::Result<()> {
|
|
||||||
let datetime =
|
|
||||||
db.remap_types::<Str, SerdeJson<v1_9::LegacyTime>>().get(wtxn, key).with_context(|| {
|
|
||||||
format!("could not read `{key}` while updating date format for index `{index_uid}`")
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if let Some(datetime) = datetime {
|
|
||||||
db.remap_types::<Str, SerdeJson<self::OffsetDateTime>>()
|
|
||||||
.put(wtxn, key, &self::OffsetDateTime(datetime.0))
|
|
||||||
.with_context(|| {
|
|
||||||
format!(
|
|
||||||
"could not write `{key}` while updating date format for index `{index_uid}`"
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn v1_9_to_v1_10(db_path: &Path) -> anyhow::Result<()> {
|
|
||||||
println!("Upgrading from v1.9.0 to v1.10.0");
|
|
||||||
// 2 changes here
|
|
||||||
|
|
||||||
// 1. date format. needs to be done before opening the Index
|
|
||||||
// 2. REST embedders. We don't support this case right now, so bail
|
|
||||||
|
|
||||||
let index_scheduler_path = db_path.join("tasks");
|
|
||||||
let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) }
|
|
||||||
.with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?;
|
|
||||||
|
|
||||||
let mut sched_wtxn = env.write_txn()?;
|
|
||||||
|
|
||||||
let index_mapping: Database<Str, UuidCodec> =
|
|
||||||
try_opening_database(&env, &sched_wtxn, "index-mapping")?;
|
|
||||||
|
|
||||||
let index_stats: Database<UuidCodec, Unspecified> =
|
|
||||||
try_opening_database(&env, &sched_wtxn, "index-stats").with_context(|| {
|
|
||||||
format!("While trying to open {:?}", index_scheduler_path.display())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let index_count =
|
|
||||||
index_mapping.len(&sched_wtxn).context("while reading the number of indexes")?;
|
|
||||||
|
|
||||||
// FIXME: not ideal, we have to pre-populate all indexes to prevent double borrow of sched_wtxn
|
|
||||||
// 1. immutably for the iteration
|
|
||||||
// 2. mutably for updating index stats
|
|
||||||
let indexes: Vec<_> = index_mapping
|
|
||||||
.iter(&sched_wtxn)?
|
|
||||||
.map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid)))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let mut rest_embedders = Vec::new();
|
|
||||||
|
|
||||||
let mut unwrapped_indexes = Vec::new();
|
|
||||||
|
|
||||||
// check that update can take place
|
|
||||||
for (index_index, result) in indexes.into_iter().enumerate() {
|
|
||||||
let (uid, uuid) = result?;
|
|
||||||
let index_path = db_path.join("indexes").join(uuid.to_string());
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"[{}/{index_count}]Checking that update can take place for `{uid}` at `{}`",
|
|
||||||
index_index + 1,
|
|
||||||
index_path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
let index_env = unsafe {
|
|
||||||
// FIXME: fetch the 25 magic number from the index file
|
|
||||||
EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
|
|
||||||
format!("while opening index {uid} at '{}'", index_path.display())
|
|
||||||
})?
|
|
||||||
};
|
|
||||||
|
|
||||||
let index_txn = index_env.read_txn().with_context(|| {
|
|
||||||
format!(
|
|
||||||
"while obtaining a write transaction for index {uid} at {}",
|
|
||||||
index_path.display()
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
println!("\t- Checking for incompatible embedders (REST embedders)");
|
|
||||||
let rest_embedders_for_index = find_rest_embedders(&uid, &index_env, &index_txn)?;
|
|
||||||
|
|
||||||
if rest_embedders_for_index.is_empty() {
|
|
||||||
unwrapped_indexes.push((uid, uuid));
|
|
||||||
} else {
|
|
||||||
// no need to add to unwrapped indexes because we'll exit early
|
|
||||||
rest_embedders.push((uid, rest_embedders_for_index));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !rest_embedders.is_empty() {
|
|
||||||
let rest_embedders = rest_embedders
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|(index, embedders)| std::iter::repeat(index.clone()).zip(embedders))
|
|
||||||
.map(|(index, embedder)| format!("\t- embedder `{embedder}` in index `{index}`"))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("\n");
|
|
||||||
bail!("The update cannot take place because there are REST embedder(s). Remove them before proceeding with the update:\n{rest_embedders}\n\n\
|
|
||||||
The database has not been modified and is still a valid v1.9 database.");
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("Update can take place, updating");
|
|
||||||
|
|
||||||
for (index_index, (uid, uuid)) in unwrapped_indexes.into_iter().enumerate() {
|
|
||||||
let index_path = db_path.join("indexes").join(uuid.to_string());
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"[{}/{index_count}]Updating index `{uid}` at `{}`",
|
|
||||||
index_index + 1,
|
|
||||||
index_path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
let index_env = unsafe {
|
|
||||||
// FIXME: fetch the 25 magic number from the index file
|
|
||||||
EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
|
|
||||||
format!("while opening index {uid} at '{}'", index_path.display())
|
|
||||||
})?
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut index_wtxn = index_env.write_txn().with_context(|| {
|
|
||||||
format!(
|
|
||||||
"while obtaining a write transaction for index `{uid}` at `{}`",
|
|
||||||
index_path.display()
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
println!("\t- Updating index stats");
|
|
||||||
update_index_stats(index_stats, &uid, uuid, &mut sched_wtxn)?;
|
|
||||||
println!("\t- Updating date format");
|
|
||||||
update_date_format(&uid, &index_env, &mut index_wtxn)?;
|
|
||||||
|
|
||||||
index_wtxn.commit().with_context(|| {
|
|
||||||
format!("while committing the write txn for index `{uid}` at {}", index_path.display())
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
|
|
||||||
sched_wtxn.commit().context("while committing the write txn for the index-scheduler")?;
|
|
||||||
|
|
||||||
println!("Upgrading database succeeded");
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@ -1,86 +0,0 @@
|
|||||||
//! The breaking changes that happened between the v1.10 and the v1.11 are:
|
|
||||||
//! - Arroy went from the v0.4.0 to the v0.5.0, see this release note to get the whole context: https://github.com/meilisearch/arroy/releases/tag/v0.5.0
|
|
||||||
//! - The `angular` distance has been renamed to `cosine` => We only need to update the string in the metadata.
|
|
||||||
//! - Reorganize the `NodeId` to make the appending of vectors work => We'll have to update the keys of almost all items in the DB.
|
|
||||||
//! - Store the list of updated IDs directly in LMDB instead of a roaring bitmap => This shouldn't be an issue since we are never supposed to commit this roaring bitmap, but it's not forbidden by arroy so ensuring it works is probably better than anything.
|
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
|
|
||||||
use anyhow::Context;
|
|
||||||
use meilisearch_types::{
|
|
||||||
heed::{types::Str, Database, EnvOpenOptions},
|
|
||||||
milli::index::db_name,
|
|
||||||
};
|
|
||||||
|
|
||||||
use crate::{try_opening_database, try_opening_poly_database, uuid_codec::UuidCodec};
|
|
||||||
|
|
||||||
pub fn v1_10_to_v1_11(db_path: &Path) -> anyhow::Result<()> {
|
|
||||||
println!("Upgrading from v1.10.0 to v1.11.0");
|
|
||||||
|
|
||||||
let index_scheduler_path = db_path.join("tasks");
|
|
||||||
let env = unsafe { EnvOpenOptions::new().max_dbs(100).open(&index_scheduler_path) }
|
|
||||||
.with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?;
|
|
||||||
|
|
||||||
let sched_rtxn = env.read_txn()?;
|
|
||||||
|
|
||||||
let index_mapping: Database<Str, UuidCodec> =
|
|
||||||
try_opening_database(&env, &sched_rtxn, "index-mapping")?;
|
|
||||||
|
|
||||||
let index_count =
|
|
||||||
index_mapping.len(&sched_rtxn).context("while reading the number of indexes")?;
|
|
||||||
|
|
||||||
let indexes: Vec<_> = index_mapping
|
|
||||||
.iter(&sched_rtxn)?
|
|
||||||
.map(|res| res.map(|(uid, uuid)| (uid.to_owned(), uuid)))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// check that update can take place
|
|
||||||
for (index_index, result) in indexes.into_iter().enumerate() {
|
|
||||||
let (uid, uuid) = result?;
|
|
||||||
let index_path = db_path.join("indexes").join(uuid.to_string());
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"[{}/{index_count}]Checking that update can take place for `{uid}` at `{}`",
|
|
||||||
index_index + 1,
|
|
||||||
index_path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
let index_env = unsafe {
|
|
||||||
EnvOpenOptions::new().max_dbs(25).open(&index_path).with_context(|| {
|
|
||||||
format!("while opening index {uid} at '{}'", index_path.display())
|
|
||||||
})?
|
|
||||||
};
|
|
||||||
|
|
||||||
let index_rtxn = index_env.read_txn().with_context(|| {
|
|
||||||
format!(
|
|
||||||
"while obtaining a read transaction for index {uid} at {}",
|
|
||||||
index_path.display()
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
let index_read_database =
|
|
||||||
try_opening_poly_database(&index_env, &index_rtxn, db_name::VECTOR_ARROY)
|
|
||||||
.with_context(|| format!("while updating date format for index `{uid}`"))?;
|
|
||||||
|
|
||||||
let mut index_wtxn = index_env.write_txn().with_context(|| {
|
|
||||||
format!(
|
|
||||||
"while obtaining a write transaction for index {uid} at {}",
|
|
||||||
index_path.display()
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let index_write_database =
|
|
||||||
try_opening_poly_database(&index_env, &index_wtxn, db_name::VECTOR_ARROY)
|
|
||||||
.with_context(|| format!("while updating date format for index `{uid}`"))?;
|
|
||||||
|
|
||||||
arroy_v04_to_v05::ugrade_from_prev_version(
|
|
||||||
&index_rtxn,
|
|
||||||
index_read_database,
|
|
||||||
&mut index_wtxn,
|
|
||||||
index_write_database,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
index_wtxn.commit()?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@ -1,106 +0,0 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use time::OffsetDateTime;
|
|
||||||
|
|
||||||
pub type FieldDistribution = std::collections::BTreeMap<String, u64>;
|
|
||||||
|
|
||||||
/// The statistics that can be computed from an `Index` object.
|
|
||||||
#[derive(serde::Serialize, serde::Deserialize, Debug)]
|
|
||||||
pub struct IndexStats {
|
|
||||||
/// Number of documents in the index.
|
|
||||||
pub number_of_documents: u64,
|
|
||||||
/// Size taken up by the index' DB, in bytes.
|
|
||||||
///
|
|
||||||
/// This includes the size taken by both the used and free pages of the DB, and as the free pages
|
|
||||||
/// are not returned to the disk after a deletion, this number is typically larger than
|
|
||||||
/// `used_database_size` that only includes the size of the used pages.
|
|
||||||
pub database_size: u64,
|
|
||||||
/// Size taken by the used pages of the index' DB, in bytes.
|
|
||||||
///
|
|
||||||
/// As the DB backend does not return to the disk the pages that are not currently used by the DB,
|
|
||||||
/// this value is typically smaller than `database_size`.
|
|
||||||
pub used_database_size: u64,
|
|
||||||
/// Association of every field name with the number of times it occurs in the documents.
|
|
||||||
pub field_distribution: FieldDistribution,
|
|
||||||
/// Creation date of the index.
|
|
||||||
pub created_at: LegacyTime,
|
|
||||||
/// Date of the last update of the index.
|
|
||||||
pub updated_at: LegacyTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize)]
|
|
||||||
pub struct IndexEmbeddingConfig {
|
|
||||||
pub name: String,
|
|
||||||
pub config: EmbeddingConfig,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub struct EmbeddingConfig {
|
|
||||||
/// Options of the embedder, specific to each kind of embedder
|
|
||||||
pub embedder_options: EmbedderOptions,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Options of an embedder, specific to each kind of embedder.
|
|
||||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub enum EmbedderOptions {
|
|
||||||
HuggingFace(hf::EmbedderOptions),
|
|
||||||
OpenAi(openai::EmbedderOptions),
|
|
||||||
Ollama(ollama::EmbedderOptions),
|
|
||||||
UserProvided(manual::EmbedderOptions),
|
|
||||||
Rest(rest::EmbedderOptions),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for EmbedderOptions {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::OpenAi(openai::EmbedderOptions { api_key: None, dimensions: None })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mod hf {
|
|
||||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub struct EmbedderOptions {
|
|
||||||
pub model: String,
|
|
||||||
pub revision: Option<String>,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod openai {
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub struct EmbedderOptions {
|
|
||||||
pub api_key: Option<String>,
|
|
||||||
pub dimensions: Option<usize>,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod ollama {
|
|
||||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub struct EmbedderOptions {
|
|
||||||
pub embedding_model: String,
|
|
||||||
pub url: Option<String>,
|
|
||||||
pub api_key: Option<String>,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod manual {
|
|
||||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
|
||||||
pub struct EmbedderOptions {
|
|
||||||
pub dimensions: usize,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mod rest {
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize, Hash)]
|
|
||||||
pub struct EmbedderOptions {
|
|
||||||
pub api_key: Option<String>,
|
|
||||||
pub dimensions: Option<usize>,
|
|
||||||
pub url: String,
|
|
||||||
pub input_field: Vec<String>,
|
|
||||||
// path to the array of embeddings
|
|
||||||
pub path_to_embeddings: Vec<String>,
|
|
||||||
// shape of a single embedding
|
|
||||||
pub embedding_object: Vec<String>,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2024-11-04 13:32:08.48368 +00:00:00
|
|
||||||
time::serde::format_description!(legacy_datetime, OffsetDateTime, "[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]");
|
|
||||||
|
|
||||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
|
||||||
#[serde(transparent)]
|
|
||||||
pub struct LegacyTime(#[serde(with = "legacy_datetime")] pub OffsetDateTime);
|
|
||||||
Loading…
Reference in New Issue
Block a user