clippy + fmt

This commit is contained in:
mpostma 2021-09-28 22:22:59 +02:00
parent 5fa9bc67d7
commit 102c46f88b
36 changed files with 596 additions and 473 deletions

View File

@ -2,9 +2,9 @@ use std::hash::{Hash, Hasher};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use log::debug; use log::debug;
use meilisearch_lib::MeiliSearch;
use serde::Serialize; use serde::Serialize;
use siphasher::sip::SipHasher; use siphasher::sip::SipHasher;
use meilisearch_lib::MeiliSearch;
use crate::Opt; use crate::Opt;

View File

@ -92,11 +92,7 @@ pub fn setup_temp_dir(db_path: impl AsRef<Path>) -> anyhow::Result<()> {
Ok(()) Ok(())
} }
pub fn configure_data( pub fn configure_data(config: &mut web::ServiceConfig, data: MeiliSearch, opt: &Opt) {
config: &mut web::ServiceConfig,
data: MeiliSearch,
opt: &Opt,
) {
let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize; let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
config config
.app_data(data) .app_data(data)
@ -139,8 +135,7 @@ pub fn configure_auth(config: &mut web::ServiceConfig, opts: &Opt) {
AuthConfig::NoAuth AuthConfig::NoAuth
}; };
config.app_data(auth_config) config.app_data(auth_config).app_data(keys);
.app_data(keys);
} }
#[cfg(feature = "mini-dashboard")] #[cfg(feature = "mini-dashboard")]

View File

@ -1,7 +1,7 @@
use std::env; use std::env;
use actix_web::HttpServer; use actix_web::HttpServer;
use meilisearch_http::{Opt, create_app, setup_meilisearch}; use meilisearch_http::{create_app, setup_meilisearch, Opt};
use meilisearch_lib::MeiliSearch; use meilisearch_lib::MeiliSearch;
use structopt::StructOpt; use structopt::StructOpt;
@ -23,7 +23,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
log_builder.init(); log_builder.init();
Ok(()) Ok(())
} }
@ -36,7 +35,9 @@ async fn main() -> anyhow::Result<()> {
match opt.env.as_ref() { match opt.env.as_ref() {
"production" => { "production" => {
if opt.master_key.is_none() { if opt.master_key.is_none() {
anyhow::bail!("In production mode, the environment variable MEILI_MASTER_KEY is mandatory") anyhow::bail!(
"In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
)
} }
} }
"development" => (), "development" => (),

View File

@ -1,16 +1,16 @@
use std::fs;
use std::io::{BufReader, Read}; use std::io::{BufReader, Read};
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use std::fs;
use byte_unit::Byte; use byte_unit::Byte;
use meilisearch_lib::options::IndexerOpts;
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
use rustls::{ use rustls::{
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
RootCertStore, RootCertStore,
}; };
use structopt::StructOpt; use structopt::StructOpt;
use meilisearch_lib::options::IndexerOpts;
const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
@ -173,24 +173,30 @@ impl Opt {
} }
fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> { fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
let certfile = fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?; let certfile =
fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
let mut reader = BufReader::new(certfile); let mut reader = BufReader::new(certfile);
Ok(certs(&mut reader).map_err(|_| anyhow::anyhow!("cannot read certificate file"))?) certs(&mut reader).map_err(|_| anyhow::anyhow!("cannot read certificate file"))
} }
fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> { fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
let rsa_keys = { let rsa_keys = {
let keyfile = let keyfile = fs::File::open(filename.clone())
fs::File::open(filename.clone()).map_err(|_| anyhow::anyhow!("cannot open private key file"))?; .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
let mut reader = BufReader::new(keyfile); let mut reader = BufReader::new(keyfile);
rsa_private_keys(&mut reader).map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))? rsa_private_keys(&mut reader)
.map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
}; };
let pkcs8_keys = { let pkcs8_keys = {
let keyfile = fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open private key file"))?; let keyfile = fs::File::open(filename)
.map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
let mut reader = BufReader::new(keyfile); let mut reader = BufReader::new(keyfile);
pkcs8_private_keys(&mut reader) pkcs8_private_keys(&mut reader).map_err(|_| {
.map_err(|_| anyhow::anyhow!("file contains invalid pkcs8 private key (encrypted keys not supported)"))? anyhow::anyhow!(
"file contains invalid pkcs8 private key (encrypted keys not supported)"
)
})?
}; };
// prefer to load pkcs8 keys // prefer to load pkcs8 keys

View File

@ -11,7 +11,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
.service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status))); .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
} }
pub async fn create_dump(meilisearch: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> { pub async fn create_dump(
meilisearch: GuardedData<Private, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> {
let res = meilisearch.create_dump().await?; let res = meilisearch.create_dump().await?;
debug!("returns: {:?}", res); debug!("returns: {:?}", res);

View File

@ -1,11 +1,11 @@
use actix_web::error::PayloadError; use actix_web::error::PayloadError;
use actix_web::{web, HttpResponse};
use actix_web::web::Bytes; use actix_web::web::Bytes;
use actix_web::{web, HttpResponse};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
use log::debug; use log::debug;
use meilisearch_lib::MeiliSearch;
use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update}; use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
use meilisearch_lib::milli::update::IndexDocumentsMethod; use meilisearch_lib::milli::update::IndexDocumentsMethod;
use meilisearch_lib::MeiliSearch;
use serde::Deserialize; use serde::Deserialize;
use serde_json::Value; use serde_json::Value;
use tokio::sync::mpsc; use tokio::sync::mpsc;
@ -104,9 +104,14 @@ pub async fn delete_document(
meilisearch: GuardedData<Private, MeiliSearch>, meilisearch: GuardedData<Private, MeiliSearch>,
path: web::Path<DocumentParam>, path: web::Path<DocumentParam>,
) -> Result<HttpResponse, ResponseError> { ) -> Result<HttpResponse, ResponseError> {
let DocumentParam { document_id, index_uid } = path.into_inner(); let DocumentParam {
document_id,
index_uid,
} = path.into_inner();
let update = Update::DeleteDocuments(vec![document_id]); let update = Update::DeleteDocuments(vec![document_id]);
let update_status = meilisearch.register_update(index_uid, update, false).await?; let update_status = meilisearch
.register_update(index_uid, update, false)
.await?;
debug!("returns: {:?}", update_status); debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
} }
@ -216,7 +221,9 @@ pub async fn delete_documents(
.collect(); .collect();
let update = Update::DeleteDocuments(ids); let update = Update::DeleteDocuments(ids);
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?; let update_status = meilisearch
.register_update(path.into_inner().index_uid, update, false)
.await?;
debug!("returns: {:?}", update_status); debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
} }
@ -226,7 +233,9 @@ pub async fn clear_all_documents(
path: web::Path<IndexParam>, path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> { ) -> Result<HttpResponse, ResponseError> {
let update = Update::ClearDocuments; let update = Update::ClearDocuments;
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?; let update_status = meilisearch
.register_update(path.into_inner().index_uid, update, false)
.await?;
debug!("returns: {:?}", update_status); debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
} }

View File

@ -1,8 +1,8 @@
use actix_web::{web, HttpResponse}; use actix_web::{web, HttpResponse};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use log::debug; use log::debug;
use meilisearch_lib::MeiliSearch;
use meilisearch_lib::index_controller::IndexSettings; use meilisearch_lib::index_controller::IndexSettings;
use meilisearch_lib::MeiliSearch;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::error::ResponseError; use crate::error::ResponseError;
@ -36,7 +36,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
); );
} }
pub async fn list_indexes(data: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> { pub async fn list_indexes(
data: GuardedData<Private, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> {
let indexes = data.list_indexes().await?; let indexes = data.list_indexes().await?;
debug!("returns: {:?}", indexes); debug!("returns: {:?}", indexes);
Ok(HttpResponse::Ok().json(indexes)) Ok(HttpResponse::Ok().json(indexes))

View File

@ -1,12 +1,12 @@
use log::debug; use log::debug;
use actix_web::{HttpResponse, web}; use actix_web::{web, HttpResponse};
use meilisearch_lib::MeiliSearch;
use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::index::{Settings, Unchecked};
use meilisearch_lib::index_controller::Update; use meilisearch_lib::index_controller::Update;
use meilisearch_lib::MeiliSearch;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::error::ResponseError; use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
#[macro_export] #[macro_export]
macro_rules! make_setting_route { macro_rules! make_setting_route {

View File

@ -53,7 +53,9 @@ pub async fn get_all_updates_status(
meilisearch: GuardedData<Private, MeiliSearch>, meilisearch: GuardedData<Private, MeiliSearch>,
path: web::Path<IndexParam>, path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> { ) -> Result<HttpResponse, ResponseError> {
let metas = meilisearch.all_update_status(path.into_inner().index_uid).await?; let metas = meilisearch
.all_update_status(path.into_inner().index_uid)
.await?;
let metas = metas let metas = metas
.into_iter() .into_iter()
.map(UpdateStatusResponse::from) .map(UpdateStatusResponse::from)

View File

@ -6,8 +6,8 @@ use log::debug;
use meilisearch_lib::index_controller::updates::status::{UpdateResult, UpdateStatus}; use meilisearch_lib::index_controller::updates::status::{UpdateResult, UpdateStatus};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use meilisearch_lib::{MeiliSearch, Update};
use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::index::{Settings, Unchecked};
use meilisearch_lib::{MeiliSearch, Update};
use crate::error::ResponseError; use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::authentication::{policies::*, GuardedData};
@ -233,7 +233,9 @@ pub async fn running() -> HttpResponse {
HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" })) HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
} }
async fn get_stats(meilisearch: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> { async fn get_stats(
meilisearch: GuardedData<Private, MeiliSearch>,
) -> Result<HttpResponse, ResponseError> {
let response = meilisearch.get_all_stats().await?; let response = meilisearch.get_all_stats().await?;
debug!("returns: {:?}", response); debug!("returns: {:?}", response);

View File

@ -35,7 +35,10 @@ impl Server {
let options = default_settings(dir.path()); let options = default_settings(dir.path());
let meilisearch = setup_meilisearch(&options).unwrap(); let meilisearch = setup_meilisearch(&options).unwrap();
let service = Service { meilisearch, options }; let service = Service {
meilisearch,
options,
};
Server { Server {
service, service,
@ -45,7 +48,10 @@ impl Server {
pub async fn new_with_options(options: Opt) -> Self { pub async fn new_with_options(options: Opt) -> Self {
let meilisearch = setup_meilisearch(&options).unwrap(); let meilisearch = setup_meilisearch(&options).unwrap();
let service = Service { meilisearch, options }; let service = Service {
meilisearch,
options,
};
Server { Server {
service, service,

View File

@ -2,7 +2,7 @@ use actix_web::{http::StatusCode, test};
use meilisearch_lib::MeiliSearch; use meilisearch_lib::MeiliSearch;
use serde_json::Value; use serde_json::Value;
use meilisearch_http::{Opt, create_app}; use meilisearch_http::{create_app, Opt};
pub struct Service { pub struct Service {
pub meilisearch: MeiliSearch, pub meilisearch: MeiliSearch,

View File

@ -16,7 +16,12 @@ async fn add_documents_test_json_content_types() {
// this is a what is expected and should work // this is a what is expected and should work
let server = Server::new().await; let server = Server::new().await;
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await; let app = test::init_service(create_app!(
&server.service.meilisearch,
true,
&server.service.options
))
.await;
let req = test::TestRequest::post() let req = test::TestRequest::post()
.uri("/indexes/dog/documents") .uri("/indexes/dog/documents")
.set_payload(document.to_string()) .set_payload(document.to_string())
@ -41,7 +46,12 @@ async fn add_documents_test_no_content_types() {
]); ]);
let server = Server::new().await; let server = Server::new().await;
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await; let app = test::init_service(create_app!(
&server.service.meilisearch,
true,
&server.service.options
))
.await;
let req = test::TestRequest::post() let req = test::TestRequest::post()
.uri("/indexes/dog/documents") .uri("/indexes/dog/documents")
.set_payload(document.to_string()) .set_payload(document.to_string())
@ -67,7 +77,12 @@ async fn add_documents_test_bad_content_types() {
]); ]);
let server = Server::new().await; let server = Server::new().await;
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await; let app = test::init_service(create_app!(
&server.service.meilisearch,
true,
&server.service.options
))
.await;
let req = test::TestRequest::post() let req = test::TestRequest::post()
.uri("/indexes/dog/documents") .uri("/indexes/dog/documents")
.set_payload(document.to_string()) .set_payload(document.to_string())

View File

@ -1,4 +1,7 @@
use std::{fmt, io::{Read, Seek, Write}}; use std::{
fmt,
io::{Read, Seek, Write},
};
use milli::documents::DocumentBatchBuilder; use milli::documents::DocumentBatchBuilder;
use serde_json::{Deserializer, Map, Value}; use serde_json::{Deserializer, Map, Value};
@ -25,12 +28,13 @@ pub enum DocumentFormatError {
#[error("Internal error: {0}")] #[error("Internal error: {0}")]
Internal(Box<dyn std::error::Error + Send + Sync + 'static>), Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
#[error("{0}. The {1} payload provided is malformed.")] #[error("{0}. The {1} payload provided is malformed.")]
MalformedPayload(Box<dyn std::error::Error + Send + Sync + 'static>, PayloadType), MalformedPayload(
Box<dyn std::error::Error + Send + Sync + 'static>,
PayloadType,
),
} }
internal_error!( internal_error!(DocumentFormatError: milli::documents::Error);
DocumentFormatError: milli::documents::Error
);
macro_rules! malformed { macro_rules! malformed {
($type:path, $e:expr) => { ($type:path, $e:expr) => {
@ -57,7 +61,8 @@ pub fn read_jsonl(input: impl Read, writer: impl Write + Seek) -> Result<()> {
pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<()> { pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<()> {
let mut builder = DocumentBatchBuilder::new(writer).unwrap(); let mut builder = DocumentBatchBuilder::new(writer).unwrap();
let documents: Vec<Map<String, Value>> = malformed!(PayloadType::Json, serde_json::from_reader(input))?; let documents: Vec<Map<String, Value>> =
malformed!(PayloadType::Json, serde_json::from_reader(input))?;
builder.add_documents(documents).unwrap(); builder.add_documents(documents).unwrap();
builder.finish().unwrap(); builder.finish().unwrap();

View File

@ -151,7 +151,9 @@ impl Index {
//If the document file is empty, we don't perform the document addition, to prevent //If the document file is empty, we don't perform the document addition, to prevent
//a primary key error to be thrown. //a primary key error to be thrown.
if !documents_reader.is_empty() { if !documents_reader.is_empty() {
let builder = update_handler.update_builder(0).index_documents(&mut txn, &index); let builder = update_handler
.update_builder(0)
.index_documents(&mut txn, &index);
builder.execute(documents_reader, |_, _| ())?; builder.execute(documents_reader, |_, _| ())?;
} }

View File

@ -8,17 +8,17 @@ use std::sync::Arc;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use heed::{EnvOpenOptions, RoTxn}; use heed::{EnvOpenOptions, RoTxn};
use milli::update::Setting; use milli::update::Setting;
use milli::{FieldDistribution, FieldId, obkv_to_json}; use milli::{obkv_to_json, FieldDistribution, FieldId};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use serde::{Serialize, Deserialize};
use error::Result; use error::Result;
pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
pub use updates::{Checked, Facets, Settings, Unchecked, apply_settings_to_builder}; pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
use uuid::Uuid; use uuid::Uuid;
use crate::EnvSizer;
use crate::index_controller::update_file_store::UpdateFileStore; use crate::index_controller::update_file_store::UpdateFileStore;
use crate::EnvSizer;
use self::error::IndexError; use self::error::IndexError;
use self::update_handler::UpdateHandler; use self::update_handler::UpdateHandler;
@ -92,12 +92,23 @@ impl Deref for Index {
} }
impl Index { impl Index {
pub fn open(path: impl AsRef<Path>, size: usize, update_file_store: Arc<UpdateFileStore>, uuid: Uuid, update_handler: Arc<UpdateHandler>) -> Result<Self> { pub fn open(
path: impl AsRef<Path>,
size: usize,
update_file_store: Arc<UpdateFileStore>,
uuid: Uuid,
update_handler: Arc<UpdateHandler>,
) -> Result<Self> {
create_dir_all(&path)?; create_dir_all(&path)?;
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(size); options.map_size(size);
let inner = Arc::new(milli::Index::new(options, &path)?); let inner = Arc::new(milli::Index::new(options, &path)?);
Ok(Index { inner, update_file_store, uuid, update_handler }) Ok(Index {
inner,
update_file_store,
uuid,
update_handler,
})
} }
pub fn stats(&self) -> Result<IndexStats> { pub fn stats(&self) -> Result<IndexStats> {
@ -268,7 +279,9 @@ impl Index {
create_dir_all(&dst)?; create_dir_all(&dst)?;
dst.push("data.mdb"); dst.push("data.mdb");
let _txn = self.write_txn()?; let _txn = self.write_txn()?;
self.inner.env.copy_to_path(dst, heed::CompactionOption::Enabled)?; self.inner
.env
.copy_to_path(dst, heed::CompactionOption::Enabled)?;
Ok(()) Ok(())
} }
} }

View File

@ -6,9 +6,7 @@ use either::Either;
use heed::RoTxn; use heed::RoTxn;
use indexmap::IndexMap; use indexmap::IndexMap;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
use milli::{ use milli::{AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, SortError};
AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, SortError
};
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value}; use serde_json::{json, Value};
@ -685,7 +683,7 @@ fn parse_filter_array(
} }
} }
Ok(FilterCondition::from_array(txn, &index, ands)?) Ok(FilterCondition::from_array(txn, index, ands)?)
} }
#[cfg(test)] #[cfg(test)]

View File

@ -8,11 +8,11 @@ use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder};
use serde::{Deserialize, Serialize, Serializer}; use serde::{Deserialize, Serialize, Serializer};
use uuid::Uuid; use uuid::Uuid;
use crate::Update;
use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult}; use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult};
use crate::Update;
use super::{Index, IndexMeta};
use super::error::{IndexError, Result}; use super::error::{IndexError, Result};
use super::{Index, IndexMeta};
fn serialize_with_wildcard<S>( fn serialize_with_wildcard<S>(
field: &Setting<Vec<String>>, field: &Setting<Vec<String>>,
@ -170,18 +170,26 @@ impl Index {
let result = (|| { let result = (|| {
let mut txn = self.write_txn()?; let mut txn = self.write_txn()?;
let result = match update.meta() { let result = match update.meta() {
Update::DocumentAddition { primary_key, content_uuid, method } => { Update::DocumentAddition {
self.update_documents(&mut txn, *method, *content_uuid, update_builder, primary_key.as_deref()) primary_key,
} content_uuid,
method,
} => self.update_documents(
&mut txn,
*method,
*content_uuid,
update_builder,
primary_key.as_deref(),
),
Update::Settings(settings) => { Update::Settings(settings) => {
let settings = settings.clone().check(); let settings = settings.clone().check();
self.update_settings(&mut txn, &settings, update_builder) self.update_settings(&mut txn, &settings, update_builder)
}, }
Update::ClearDocuments => { Update::ClearDocuments => {
let builder = update_builder.clear_documents(&mut txn, self); let builder = update_builder.clear_documents(&mut txn, self);
let _count = builder.execute()?; let _count = builder.execute()?;
Ok(UpdateResult::Other) Ok(UpdateResult::Other)
}, }
Update::DeleteDocuments(ids) => { Update::DeleteDocuments(ids) => {
let mut builder = update_builder.delete_documents(&mut txn, self)?; let mut builder = update_builder.delete_documents(&mut txn, self)?;
@ -276,7 +284,10 @@ impl Index {
} }
} }
pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut milli::update::Settings) { pub fn apply_settings_to_builder(
settings: &Settings<Checked>,
builder: &mut milli::update::Settings,
) {
match settings.searchable_attributes { match settings.searchable_attributes {
Setting::Set(ref names) => builder.set_searchable_fields(names.clone()), Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
Setting::Reset => builder.reset_searchable_fields(), Setting::Reset => builder.reset_searchable_fields(),
@ -298,9 +309,7 @@ pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut mil
} }
match settings.sortable_attributes { match settings.sortable_attributes {
Setting::Set(ref fields) => { Setting::Set(ref fields) => builder.set_sortable_fields(fields.iter().cloned().collect()),
builder.set_sortable_fields(fields.iter().cloned().collect())
}
Setting::Reset => builder.reset_sortable_fields(), Setting::Reset => builder.reset_sortable_fields(),
Setting::NotSet => (), Setting::NotSet => (),
} }
@ -318,9 +327,7 @@ pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut mil
} }
match settings.synonyms { match settings.synonyms {
Setting::Set(ref synonyms) => { Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
builder.set_synonyms(synonyms.clone().into_iter().collect())
}
Setting::Reset => builder.reset_synonyms(), Setting::Reset => builder.reset_synonyms(),
Setting::NotSet => (), Setting::NotSet => (),
} }

View File

@ -1,5 +1,5 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use std::fs::{File, create_dir_all}; use std::fs::{create_dir_all, File};
use std::io::{BufReader, Seek, SeekFrom}; use std::io::{BufReader, Seek, SeekFrom};
use std::marker::PhantomData; use std::marker::PhantomData;
use std::path::Path; use std::path::Path;
@ -17,10 +17,7 @@ use crate::index::update_handler::UpdateHandler;
use crate::index_controller::index_resolver::uuid_store::HeedUuidStore; use crate::index_controller::index_resolver::uuid_store::HeedUuidStore;
use crate::index_controller::{self, IndexMetadata}; use crate::index_controller::{self, IndexMetadata};
use crate::index_controller::{asc_ranking_rule, desc_ranking_rule}; use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
use crate::{ use crate::{index::Unchecked, options::IndexerOpts};
index::Unchecked,
options::IndexerOpts,
};
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
@ -136,16 +133,16 @@ fn load_index(
//If the document file is empty, we don't perform the document addition, to prevent //If the document file is empty, we don't perform the document addition, to prevent
//a primary key error to be thrown. //a primary key error to be thrown.
if !documents_reader.is_empty() { if !documents_reader.is_empty() {
let builder = update_handler.update_builder(0).index_documents(&mut txn, &index); let builder = update_handler
.update_builder(0)
.index_documents(&mut txn, &index);
builder.execute(documents_reader, |_, _| ())?; builder.execute(documents_reader, |_, _| ())?;
} }
txn.commit()?; txn.commit()?;
// Finaly, we extract the original milli::Index and close it // Finaly, we extract the original milli::Index and close it
index index.prepare_for_closing().wait();
.prepare_for_closing()
.wait();
// Updates are ignored in dumps V1. // Updates are ignored in dumps V1.

View File

@ -158,8 +158,7 @@ impl DumpTask {
create_dir_all(&self.path).await?; create_dir_all(&self.path).await?;
let temp_dump_dir = let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??;
tokio::task::spawn_blocking(|| tempfile::TempDir::new()).await??;
let temp_dump_path = temp_dump_dir.path().to_owned(); let temp_dump_path = temp_dump_dir.path().to_owned();
let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);

View File

@ -27,7 +27,8 @@ pub enum IndexResolverError {
} }
impl<T> From<MpscSendError<T>> for IndexResolverError impl<T> From<MpscSendError<T>> for IndexResolverError
where T: Send + Sync + 'static + fmt::Debug where
T: Send + Sync + 'static + fmt::Debug,
{ {
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self { fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
Self::Internal(Box::new(other)) Self::Internal(Box::new(other))

View File

@ -9,8 +9,8 @@ use tokio::task::spawn_blocking;
use uuid::Uuid; use uuid::Uuid;
use super::error::{IndexResolverError, Result}; use super::error::{IndexResolverError, Result};
use crate::index::Index;
use crate::index::update_handler::UpdateHandler; use crate::index::update_handler::UpdateHandler;
use crate::index::Index;
use crate::index_controller::update_file_store::UpdateFileStore; use crate::index_controller::update_file_store::UpdateFileStore;
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
@ -32,7 +32,11 @@ pub struct MapIndexStore {
} }
impl MapIndexStore { impl MapIndexStore {
pub fn new(path: impl AsRef<Path>, index_size: usize, indexer_opts: &IndexerOpts) -> anyhow::Result<Self> { pub fn new(
path: impl AsRef<Path>,
index_size: usize,
indexer_opts: &IndexerOpts,
) -> anyhow::Result<Self> {
let update_handler = Arc::new(UpdateHandler::new(indexer_opts)?); let update_handler = Arc::new(UpdateHandler::new(indexer_opts)?);
let update_file_store = Arc::new(UpdateFileStore::new(path.as_ref()).unwrap()); let update_file_store = Arc::new(UpdateFileStore::new(path.as_ref()).unwrap());
let path = path.as_ref().join("indexes/"); let path = path.as_ref().join("indexes/");
@ -100,7 +104,10 @@ impl IndexStore for MapIndexStore {
let index_size = self.index_size; let index_size = self.index_size;
let file_store = self.update_file_store.clone(); let file_store = self.update_file_store.clone();
let update_handler = self.update_handler.clone(); let update_handler = self.update_handler.clone();
let index = spawn_blocking(move || Index::open(path, index_size, file_store, uuid, update_handler)).await??; let index = spawn_blocking(move || {
Index::open(path, index_size, file_store, uuid, update_handler)
})
.await??;
self.index_store.write().await.insert(uuid, index.clone()); self.index_store.write().await.insert(uuid, index.clone());
Ok(Some(index)) Ok(Some(index))
} }

View File

@ -1,19 +1,26 @@
pub mod uuid_store;
mod index_store;
pub mod error; pub mod error;
mod index_store;
pub mod uuid_store;
use std::path::Path; use std::path::Path;
use uuid::Uuid; use error::{IndexResolverError, Result};
use uuid_store::{UuidStore, HeedUuidStore};
use index_store::{IndexStore, MapIndexStore}; use index_store::{IndexStore, MapIndexStore};
use error::{Result, IndexResolverError}; use uuid::Uuid;
use uuid_store::{HeedUuidStore, UuidStore};
use crate::{index::{Index, update_handler::UpdateHandler}, options::IndexerOpts}; use crate::{
index::{update_handler::UpdateHandler, Index},
options::IndexerOpts,
};
pub type HardStateIndexResolver = IndexResolver<HeedUuidStore, MapIndexStore>; pub type HardStateIndexResolver = IndexResolver<HeedUuidStore, MapIndexStore>;
pub fn create_index_resolver(path: impl AsRef<Path>, index_size: usize, indexer_opts: &IndexerOpts) -> anyhow::Result<HardStateIndexResolver> { pub fn create_index_resolver(
path: impl AsRef<Path>,
index_size: usize,
indexer_opts: &IndexerOpts,
) -> anyhow::Result<HardStateIndexResolver> {
let uuid_store = HeedUuidStore::new(&path)?; let uuid_store = HeedUuidStore::new(&path)?;
let index_store = MapIndexStore::new(&path, index_size, indexer_opts)?; let index_store = MapIndexStore::new(&path, index_size, indexer_opts)?;
Ok(IndexResolver::new(uuid_store, index_store)) Ok(IndexResolver::new(uuid_store, index_store))
@ -47,13 +54,11 @@ impl IndexResolver<HeedUuidStore, MapIndexStore> {
} }
impl<U, I> IndexResolver<U, I> impl<U, I> IndexResolver<U, I>
where U: UuidStore, where
U: UuidStore,
I: IndexStore, I: IndexStore,
{ {
pub fn new( pub fn new(index_uuid_store: U, index_store: I) -> Self {
index_uuid_store: U,
index_store: I,
) -> Self {
Self { Self {
index_uuid_store, index_uuid_store,
index_store, index_store,
@ -75,7 +80,10 @@ where U: UuidStore,
} }
pub async fn snapshot(&self, path: impl AsRef<Path>) -> Result<Vec<Index>> { pub async fn snapshot(&self, path: impl AsRef<Path>) -> Result<Vec<Index>> {
let uuids = self.index_uuid_store.snapshot(path.as_ref().to_owned()).await?; let uuids = self
.index_uuid_store
.snapshot(path.as_ref().to_owned())
.await?;
let mut indexes = Vec::new(); let mut indexes = Vec::new();
for uuid in uuids { for uuid in uuids {
indexes.push(self.get_index_by_uuid(uuid).await?); indexes.push(self.get_index_by_uuid(uuid).await?);
@ -99,13 +107,11 @@ where U: UuidStore,
let mut indexes = Vec::new(); let mut indexes = Vec::new();
for (name, uuid) in uuids { for (name, uuid) in uuids {
match self.index_store.get(uuid).await? { match self.index_store.get(uuid).await? {
Some(index) => { Some(index) => indexes.push((name, index)),
indexes.push((name, index))
},
None => { None => {
// we found an unexisting index, we remove it from the uuid store // we found an unexisting index, we remove it from the uuid store
let _ = self.index_uuid_store.delete(name).await; let _ = self.index_uuid_store.delete(name).await;
}, }
} }
} }
@ -124,7 +130,10 @@ where U: UuidStore,
pub async fn get_index_by_uuid(&self, uuid: Uuid) -> Result<Index> { pub async fn get_index_by_uuid(&self, uuid: Uuid) -> Result<Index> {
// TODO: Handle this error better. // TODO: Handle this error better.
self.index_store.get(uuid).await?.ok_or(IndexResolverError::UnexistingIndex(String::new())) self.index_store
.get(uuid)
.await?
.ok_or_else(|| IndexResolverError::UnexistingIndex(String::new()))
} }
pub async fn get_index(&self, uid: String) -> Result<Index> { pub async fn get_index(&self, uid: String) -> Result<Index> {
@ -137,17 +146,17 @@ where U: UuidStore,
// and remove the uuid from th uuid store. // and remove the uuid from th uuid store.
let _ = self.index_uuid_store.delete(name.clone()).await; let _ = self.index_uuid_store.delete(name.clone()).await;
Err(IndexResolverError::UnexistingIndex(name)) Err(IndexResolverError::UnexistingIndex(name))
},
} }
} }
(name, _) => Err(IndexResolverError::UnexistingIndex(name)) }
(name, _) => Err(IndexResolverError::UnexistingIndex(name)),
} }
} }
pub async fn get_uuid(&self, uid: String) -> Result<Uuid> { pub async fn get_uuid(&self, uid: String) -> Result<Uuid> {
match self.index_uuid_store.get_uuid(uid).await? { match self.index_uuid_store.get_uuid(uid).await? {
(_, Some(uuid)) => Ok(uuid), (_, Some(uuid)) => Ok(uuid),
(name, _) => Err(IndexResolverError::UnexistingIndex(name)) (name, _) => Err(IndexResolverError::UnexistingIndex(name)),
} }
} }
} }

View File

@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use uuid::Uuid; use uuid::Uuid;
use super::error::{Result, IndexResolverError}; use super::error::{IndexResolverError, Result};
use crate::EnvSizer; use crate::EnvSizer;
const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB

View File

@ -18,25 +18,27 @@ use dump_actor::DumpActorHandle;
pub use dump_actor::{DumpInfo, DumpStatus}; pub use dump_actor::{DumpInfo, DumpStatus};
use snapshot::load_snapshot; use snapshot::load_snapshot;
use crate::index::{Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked}; use crate::index::error::Result as IndexResult;
use crate::index::{
Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked,
};
use crate::index_controller::index_resolver::create_index_resolver; use crate::index_controller::index_resolver::create_index_resolver;
use crate::index_controller::snapshot::SnapshotService; use crate::index_controller::snapshot::SnapshotService;
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
use error::Result; use error::Result;
use crate::index::error::{Result as IndexResult};
use self::dump_actor::load_dump; use self::dump_actor::load_dump;
use self::index_resolver::HardStateIndexResolver;
use self::index_resolver::error::IndexResolverError; use self::index_resolver::error::IndexResolverError;
use self::index_resolver::HardStateIndexResolver;
use self::updates::status::UpdateStatus; use self::updates::status::UpdateStatus;
use self::updates::UpdateMsg; use self::updates::UpdateMsg;
mod dump_actor; mod dump_actor;
pub mod error; pub mod error;
mod index_resolver;
mod snapshot; mod snapshot;
pub mod update_file_store; pub mod update_file_store;
pub mod updates; pub mod updates;
mod index_resolver;
pub type Payload = Box< pub type Payload = Box<
dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin, dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
@ -79,6 +81,7 @@ pub struct Stats {
pub indexes: BTreeMap<String, IndexStats>, pub indexes: BTreeMap<String, IndexStats>,
} }
#[allow(clippy::large_enum_variant)]
#[derive(derivative::Derivative)] #[derive(derivative::Derivative)]
#[derivative(Debug)] #[derivative(Debug)]
pub enum Update { pub enum Update {
@ -141,12 +144,19 @@ impl IndexControllerBuilder {
std::fs::create_dir_all(db_path.as_ref())?; std::fs::create_dir_all(db_path.as_ref())?;
let index_resolver = Arc::new(create_index_resolver(&db_path, index_size, &indexer_options)?); let index_resolver = Arc::new(create_index_resolver(
&db_path,
index_size,
&indexer_options,
)?);
#[allow(unreachable_code)] #[allow(unreachable_code)]
let update_sender = updates::create_update_handler(index_resolver.clone(), &db_path, update_store_size)?; let update_sender =
updates::create_update_handler(index_resolver.clone(), &db_path, update_store_size)?;
let dump_path = self.dump_dst.ok_or_else(|| anyhow::anyhow!("Missing dump directory path"))?; let dump_path = self
.dump_dst
.ok_or_else(|| anyhow::anyhow!("Missing dump directory path"))?;
let dump_handle = dump_actor::DumpActorHandleImpl::new( let dump_handle = dump_actor::DumpActorHandleImpl::new(
dump_path, dump_path,
index_resolver.clone(), index_resolver.clone(),
@ -159,8 +169,10 @@ impl IndexControllerBuilder {
let snapshot_service = SnapshotService::new( let snapshot_service = SnapshotService::new(
index_resolver.clone(), index_resolver.clone(),
update_sender.clone(), update_sender.clone(),
self.snapshot_interval.ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?, self.snapshot_interval
self.snapshot_dir.ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?, .ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?,
self.snapshot_dir
.ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?,
db_path db_path
.as_ref() .as_ref()
.file_name() .file_name()
@ -246,7 +258,12 @@ impl IndexController {
IndexControllerBuilder::default() IndexControllerBuilder::default()
} }
pub async fn register_update(&self, uid: String, update: Update, create_index: bool) -> Result<UpdateStatus> { pub async fn register_update(
&self,
uid: String,
update: Update,
create_index: bool,
) -> Result<UpdateStatus> {
match self.index_resolver.get_uuid(uid).await { match self.index_resolver.get_uuid(uid).await {
Ok(uuid) => { Ok(uuid) => {
let update_result = UpdateMsg::update(&self.update_sender, uuid, update).await?; let update_result = UpdateMsg::update(&self.update_sender, uuid, update).await?;
@ -255,7 +272,8 @@ impl IndexController {
Err(IndexResolverError::UnexistingIndex(name)) => { Err(IndexResolverError::UnexistingIndex(name)) => {
if create_index { if create_index {
let index = self.index_resolver.create_index(name, None).await?; let index = self.index_resolver.create_index(name, None).await?;
let update_result = UpdateMsg::update(&self.update_sender, index.uuid, update).await?; let update_result =
UpdateMsg::update(&self.update_sender, index.uuid, update).await?;
// ignore if index creation fails now, since it may already have been created // ignore if index creation fails now, since it may already have been created
Ok(update_result) Ok(update_result)
@ -310,7 +328,9 @@ impl IndexController {
attributes_to_retrieve: Option<Vec<String>>, attributes_to_retrieve: Option<Vec<String>>,
) -> Result<Vec<Document>> { ) -> Result<Vec<Document>> {
let index = self.index_resolver.get_index(uid).await?; let index = self.index_resolver.get_index(uid).await?;
let documents = spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)).await??; let documents =
spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
.await??;
Ok(documents) Ok(documents)
} }
@ -321,7 +341,9 @@ impl IndexController {
attributes_to_retrieve: Option<Vec<String>>, attributes_to_retrieve: Option<Vec<String>>,
) -> Result<Document> { ) -> Result<Document> {
let index = self.index_resolver.get_index(uid).await?; let index = self.index_resolver.get_index(uid).await?;
let document = spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve)).await??; let document =
spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve))
.await??;
Ok(document) Ok(document)
} }
@ -330,12 +352,12 @@ impl IndexController {
uid: String, uid: String,
mut index_settings: IndexSettings, mut index_settings: IndexSettings,
) -> Result<IndexMetadata> { ) -> Result<IndexMetadata> {
index_settings.uid.take(); index_settings.uid.take();
let index = self.index_resolver.get_index(uid.clone()).await?; let index = self.index_resolver.get_index(uid.clone()).await?;
let uuid = index.uuid; let uuid = index.uuid;
let meta = spawn_blocking(move || index.update_primary_key(index_settings.primary_key)).await??; let meta =
spawn_blocking(move || index.update_primary_key(index_settings.primary_key)).await??;
let meta = IndexMetadata { let meta = IndexMetadata {
uuid, uuid,
name: uid.clone(), name: uid.clone(),
@ -386,7 +408,8 @@ impl IndexController {
let stats = index.stats()?; let stats = index.stats()?;
let meta = index.meta()?; let meta = index.meta()?;
Ok((stats, meta)) Ok((stats, meta))
}).await??; })
.await??;
database_size += stats.size; database_size += stats.size;
@ -415,8 +438,15 @@ impl IndexController {
Ok(self.dump_handle.dump_info(uid).await?) Ok(self.dump_handle.dump_info(uid).await?)
} }
pub async fn create_index(&self, uid: String, primary_key: Option<String>) -> Result<IndexMetadata> { pub async fn create_index(
let index = self.index_resolver.create_index(uid.clone(), primary_key).await?; &self,
uid: String,
primary_key: Option<String>,
) -> Result<IndexMetadata> {
let index = self
.index_resolver
.create_index(uid.clone(), primary_key)
.await?;
let meta = spawn_blocking(move || -> IndexResult<_> { let meta = spawn_blocking(move || -> IndexResult<_> {
let meta = index.meta()?; let meta = index.meta()?;
let meta = IndexMetadata { let meta = IndexMetadata {
@ -426,7 +456,8 @@ impl IndexController {
meta, meta,
}; };
Ok(meta) Ok(meta)
}).await??; })
.await??;
Ok(meta) Ok(meta)
} }

View File

@ -4,14 +4,14 @@ use std::time::Duration;
use anyhow::bail; use anyhow::bail;
use log::{error, info, trace}; use log::{error, info, trace};
use tokio::fs;
use tokio::task::spawn_blocking; use tokio::task::spawn_blocking;
use tokio::time::sleep; use tokio::time::sleep;
use tokio::fs;
use crate::index_controller::updates::UpdateMsg; use crate::index_controller::updates::UpdateMsg;
use super::updates::UpdateSender;
use super::index_resolver::HardStateIndexResolver; use super::index_resolver::HardStateIndexResolver;
use super::updates::UpdateSender;
pub struct SnapshotService { pub struct SnapshotService {
index_resolver: Arc<HardStateIndexResolver>, index_resolver: Arc<HardStateIndexResolver>,
@ -56,8 +56,7 @@ impl SnapshotService {
let snapshot_dir = self.snapshot_path.clone(); let snapshot_dir = self.snapshot_path.clone();
fs::create_dir_all(&snapshot_dir).await?; fs::create_dir_all(&snapshot_dir).await?;
let temp_snapshot_dir = let temp_snapshot_dir = spawn_blocking(tempfile::tempdir).await??;
spawn_blocking(move || tempfile::tempdir()).await??;
let temp_snapshot_path = temp_snapshot_dir.path().to_owned(); let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
let indexes = self let indexes = self

View File

@ -151,7 +151,7 @@ impl UpdateFileStore {
} }
serde_json::to_writer(&mut dst_file, &document_buffer)?; serde_json::to_writer(&mut dst_file, &document_buffer)?;
dst_file.write(b"\n")?; dst_file.write_all(b"\n")?;
document_buffer.clear(); document_buffer.clear();
} }

View File

@ -1,9 +1,12 @@
use std::fmt;
use std::error::Error; use std::error::Error;
use std::fmt;
use meilisearch_error::{Code, ErrorCode}; use meilisearch_error::{Code, ErrorCode};
use crate::{document_formats::DocumentFormatError, index_controller::update_file_store::UpdateFileStoreError}; use crate::{
document_formats::DocumentFormatError,
index_controller::update_file_store::UpdateFileStoreError,
};
pub type Result<T> = std::result::Result<T, UpdateLoopError>; pub type Result<T> = std::result::Result<T, UpdateLoopError>;
@ -28,7 +31,8 @@ pub enum UpdateLoopError {
} }
impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateLoopError impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateLoopError
where T: Sync + Send + 'static + fmt::Debug where
T: Sync + Send + 'static + fmt::Debug,
{ {
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self { fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
Self::Internal(Box::new(other)) Self::Internal(Box::new(other))

View File

@ -44,7 +44,11 @@ pub enum UpdateMsg {
} }
impl UpdateMsg { impl UpdateMsg {
pub async fn snapshot(sender: &mpsc::Sender<Self>, path: PathBuf, indexes: Vec<Index>) -> Result<()> { pub async fn snapshot(
sender: &mpsc::Sender<Self>,
path: PathBuf,
indexes: Vec<Index>,
) -> Result<()> {
let (ret, rcv) = oneshot::channel(); let (ret, rcv) = oneshot::channel();
let msg = Self::Snapshot { path, indexes, ret }; let msg = Self::Snapshot { path, indexes, ret };
sender.send(msg).await?; sender.send(msg).await?;

View File

@ -80,7 +80,7 @@ impl<S: Stream<Item = std::result::Result<Bytes, PayloadError>> + Unpin> io::Rea
self.read(buf) self.read(buf)
} }
Some(Err(e)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, e)), Some(Err(e)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, e)),
None => return Ok(0), None => Ok(0),
}, },
} }
} }
@ -109,7 +109,13 @@ impl UpdateLoop {
let must_exit = Arc::new(AtomicBool::new(false)); let must_exit = Arc::new(AtomicBool::new(false));
let update_file_store = UpdateFileStore::new(&path).unwrap(); let update_file_store = UpdateFileStore::new(&path).unwrap();
let store = UpdateStore::open(options, &path, index_resolver.clone(), must_exit.clone(), update_file_store.clone())?; let store = UpdateStore::open(
options,
&path,
index_resolver,
must_exit.clone(),
update_file_store.clone(),
)?;
let inbox = Some(inbox); let inbox = Some(inbox);
@ -194,8 +200,8 @@ impl UpdateLoop {
update_file.persist()?; update_file.persist()?;
Ok(()) Ok(())
}).await??; })
.await??;
store::Update::DocumentAddition { store::Update::DocumentAddition {
primary_key, primary_key,
@ -216,7 +222,6 @@ impl UpdateLoop {
Ok(status.into()) Ok(status.into())
} }
async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> { async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
let update_store = self.store.clone(); let update_store = self.store.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
@ -248,8 +253,7 @@ impl UpdateLoop {
async fn handle_snapshot(&self, indexes: Vec<Index>, path: PathBuf) -> Result<()> { async fn handle_snapshot(&self, indexes: Vec<Index>, path: PathBuf) -> Result<()> {
let update_store = self.store.clone(); let update_store = self.store.clone();
tokio::task::spawn_blocking(move || update_store.snapshot(indexes, path)) tokio::task::spawn_blocking(move || update_store.snapshot(indexes, path)).await??;
.await??;
Ok(()) Ok(())
} }

View File

@ -6,7 +6,10 @@ use meilisearch_error::{Code, ErrorCode};
use milli::update::{DocumentAdditionResult, IndexDocumentsMethod}; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::{Update, index::{Settings, Unchecked}}; use crate::{
index::{Settings, Unchecked},
Update,
};
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub enum UpdateResult { pub enum UpdateResult {

View File

@ -1,7 +1,7 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::path::{Path, PathBuf}; use std::fs::{create_dir_all, File};
use std::io::{BufReader, Write}; use std::io::{BufReader, Write};
use std::fs::{File, create_dir_all}; use std::path::{Path, PathBuf};
use heed::{EnvOpenOptions, RoTxn}; use heed::{EnvOpenOptions, RoTxn};
use rayon::prelude::*; use rayon::prelude::*;
@ -11,7 +11,14 @@ use tempfile::{NamedTempFile, TempDir};
use uuid::Uuid; use uuid::Uuid;
use super::{Result, State, UpdateStore}; use super::{Result, State, UpdateStore};
use crate::{Update, index::Index, index_controller::{update_file_store::UpdateFileStore, updates::status::{Enqueued, UpdateStatus}}}; use crate::{
index::Index,
index_controller::{
update_file_store::UpdateFileStore,
updates::status::{Enqueued, UpdateStatus},
},
Update,
};
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
struct UpdateEntry { struct UpdateEntry {
@ -20,11 +27,7 @@ struct UpdateEntry {
} }
impl UpdateStore { impl UpdateStore {
pub fn dump( pub fn dump(&self, indexes: &[Index], path: PathBuf) -> Result<()> {
&self,
indexes: &[Index],
path: PathBuf,
) -> Result<()> {
let state_lock = self.state.write(); let state_lock = self.state.write();
state_lock.swap(State::Dumping); state_lock.swap(State::Dumping);
@ -35,7 +38,10 @@ impl UpdateStore {
self.dump_updates(&txn, &uuids, &path)?; self.dump_updates(&txn, &uuids, &path)?;
indexes.par_iter().try_for_each(|index| index.dump(&path)).unwrap(); indexes
.par_iter()
.try_for_each(|index| index.dump(&path))
.unwrap();
Ok(()) Ok(())
} }
@ -74,11 +80,13 @@ impl UpdateStore {
let update = data.decode()?; let update = data.decode()?;
if let Enqueued { if let Enqueued {
meta: Update::DocumentAddition { meta: Update::DocumentAddition { content_uuid, .. },
content_uuid, .. ..
}, .. } = update
} = update { {
self.update_file_store.dump(content_uuid, &dst_path).unwrap(); self.update_file_store
.dump(content_uuid, &dst_path)
.unwrap();
} }
let update_json = UpdateEntry { let update_json = UpdateEntry {
@ -122,7 +130,6 @@ impl UpdateStore {
dst: impl AsRef<Path>, dst: impl AsRef<Path>,
db_size: usize, db_size: usize,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
println!("target path: {}", dst.as_ref().display()); println!("target path: {}", dst.as_ref().display());
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();

View File

@ -17,25 +17,26 @@ use heed::zerocopy::U64;
use heed::{CompactionOption, Database, Env, EnvOpenOptions}; use heed::{CompactionOption, Database, Env, EnvOpenOptions};
use log::error; use log::error;
use parking_lot::{Mutex, MutexGuard}; use parking_lot::{Mutex, MutexGuard};
use rayon::prelude::*;
use tokio::runtime::Handle; use tokio::runtime::Handle;
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio::sync::mpsc::error::TrySendError; use tokio::sync::mpsc::error::TrySendError;
use tokio::time::timeout; use tokio::time::timeout;
use uuid::Uuid; use uuid::Uuid;
use rayon::prelude::*;
use codec::*; use codec::*;
use super::error::Result; use super::error::Result;
use super::status::{Enqueued, Processing}; use super::status::{Enqueued, Processing};
use crate::EnvSizer; use crate::index::Index;
use crate::index_controller::update_files_path; use crate::index_controller::update_files_path;
use crate::index_controller::updates::*; use crate::index_controller::updates::*;
use crate::index::Index; use crate::EnvSizer;
#[allow(clippy::upper_case_acronyms)] #[allow(clippy::upper_case_acronyms)]
type BEU64 = U64<heed::byteorder::BE>; type BEU64 = U64<heed::byteorder::BE>;
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Update { pub enum Update {
DeleteDocuments(Vec<String>), DeleteDocuments(Vec<String>),
@ -164,7 +165,8 @@ impl UpdateStore {
must_exit: Arc<AtomicBool>, must_exit: Arc<AtomicBool>,
update_file_store: UpdateFileStore, update_file_store: UpdateFileStore,
) -> anyhow::Result<Arc<Self>> { ) -> anyhow::Result<Arc<Self>> {
let (update_store, mut notification_receiver) = Self::new(options, path, update_file_store)?; let (update_store, mut notification_receiver) =
Self::new(options, path, update_file_store)?;
let update_store = Arc::new(update_store); let update_store = Arc::new(update_store);
// Send a first notification to trigger the process. // Send a first notification to trigger the process.
@ -250,11 +252,7 @@ impl UpdateStore {
/// Registers the update content in the pending store and the meta /// Registers the update content in the pending store and the meta
/// into the pending-meta store. Returns the new unique update id. /// into the pending-meta store. Returns the new unique update id.
pub fn register_update( pub fn register_update(&self, index_uuid: Uuid, update: Update) -> heed::Result<Enqueued> {
&self,
index_uuid: Uuid,
update: Update,
) -> heed::Result<Enqueued> {
let mut txn = self.env.write_txn()?; let mut txn = self.env.write_txn()?;
let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?; let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?;
let meta = Enqueued::new(update, update_id); let meta = Enqueued::new(update, update_id);
@ -299,7 +297,10 @@ impl UpdateStore {
/// Executes the user provided function on the next pending update (the one with the lowest id). /// Executes the user provided function on the next pending update (the one with the lowest id).
/// This is asynchronous as it let the user process the update with a read-only txn and /// This is asynchronous as it let the user process the update with a read-only txn and
/// only writing the result meta to the processed-meta store *after* it has been processed. /// only writing the result meta to the processed-meta store *after* it has been processed.
fn process_pending_update(&self, index_resolver: Arc<HardStateIndexResolver>) -> Result<Option<()>> { fn process_pending_update(
&self,
index_resolver: Arc<HardStateIndexResolver>,
) -> Result<Option<()>> {
// Create a read transaction to be able to retrieve the pending update in order. // Create a read transaction to be able to retrieve the pending update in order.
let rtxn = self.env.read_txn()?; let rtxn = self.env.read_txn()?;
let first_meta = self.pending_queue.first(&rtxn)?; let first_meta = self.pending_queue.first(&rtxn)?;
@ -315,8 +316,7 @@ impl UpdateStore {
let state = self.state.write(); let state = self.state.write();
state.swap(State::Processing(index_uuid, processing.clone())); state.swap(State::Processing(index_uuid, processing.clone()));
let result = let result = self.perform_update(processing, index_resolver, index_uuid, global_id);
self.perform_update(processing, index_resolver, index_uuid, global_id);
state.swap(State::Idle); state.swap(State::Idle);
@ -495,15 +495,10 @@ impl UpdateStore {
Ok(()) Ok(())
} }
pub fn snapshot( pub fn snapshot(&self, indexes: Vec<Index>, path: impl AsRef<Path>) -> Result<()> {
&self,
indexes: Vec<Index>,
path: impl AsRef<Path>,
) -> Result<()> {
let state_lock = self.state.write(); let state_lock = self.state.write();
state_lock.swap(State::Snapshoting); state_lock.swap(State::Snapshoting);
let txn = self.env.write_txn()?; let txn = self.env.write_txn()?;
let update_path = path.as_ref().join("updates"); let update_path = path.as_ref().join("updates");
@ -523,19 +518,22 @@ impl UpdateStore {
let ((_, uuid, _), pending) = entry?; let ((_, uuid, _), pending) = entry?;
if uuids.contains(&uuid) { if uuids.contains(&uuid) {
if let Enqueued { if let Enqueued {
meta: Update::DocumentAddition { meta: Update::DocumentAddition { content_uuid, .. },
content_uuid, ..
},
.. ..
} = pending.decode()? } = pending.decode()?
{ {
self.update_file_store.snapshot(content_uuid, &path).unwrap(); self.update_file_store
.snapshot(content_uuid, &path)
.unwrap();
} }
} }
} }
let path = path.as_ref().to_owned(); let path = path.as_ref().to_owned();
indexes.par_iter().try_for_each(|index| index.snapshot(path.clone())).unwrap(); indexes
.par_iter()
.try_for_each(|index| index.snapshot(path.clone()))
.unwrap();
Ok(()) Ok(())
} }
@ -546,10 +544,7 @@ impl UpdateStore {
for entry in self.pending_queue.iter(&txn)? { for entry in self.pending_queue.iter(&txn)? {
let (_, pending) = entry?; let (_, pending) = entry?;
if let Enqueued { if let Enqueued {
meta: store::Update::DocumentAddition { meta: store::Update::DocumentAddition { content_uuid, .. },
content_uuid,
..
},
.. ..
} = pending } = pending
{ {

View File

@ -5,7 +5,7 @@ pub mod options;
pub mod index; pub mod index;
pub mod index_controller; pub mod index_controller;
pub use index_controller::{IndexController as MeiliSearch, updates::store::Update}; pub use index_controller::{updates::store::Update, IndexController as MeiliSearch};
pub use milli; pub use milli;
@ -55,4 +55,3 @@ pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Res
ar.unpack(&dest)?; ar.unpack(&dest)?;
Ok(()) Ok(())
} }

View File

@ -112,4 +112,3 @@ fn total_memory_bytes() -> Option<u64> {
None None
} }
} }