mirror of
https://github.com/meilisearch/meilisearch.git
synced 2024-11-23 02:27:40 +08:00
clippy + fmt
This commit is contained in:
parent
5fa9bc67d7
commit
102c46f88b
@ -2,9 +2,9 @@ use std::hash::{Hash, Hasher};
|
|||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use log::debug;
|
use log::debug;
|
||||||
|
use meilisearch_lib::MeiliSearch;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use siphasher::sip::SipHasher;
|
use siphasher::sip::SipHasher;
|
||||||
use meilisearch_lib::MeiliSearch;
|
|
||||||
|
|
||||||
use crate::Opt;
|
use crate::Opt;
|
||||||
|
|
||||||
|
@ -92,11 +92,7 @@ pub fn setup_temp_dir(db_path: impl AsRef<Path>) -> anyhow::Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn configure_data(
|
pub fn configure_data(config: &mut web::ServiceConfig, data: MeiliSearch, opt: &Opt) {
|
||||||
config: &mut web::ServiceConfig,
|
|
||||||
data: MeiliSearch,
|
|
||||||
opt: &Opt,
|
|
||||||
) {
|
|
||||||
let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
|
let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
|
||||||
config
|
config
|
||||||
.app_data(data)
|
.app_data(data)
|
||||||
@ -139,8 +135,7 @@ pub fn configure_auth(config: &mut web::ServiceConfig, opts: &Opt) {
|
|||||||
AuthConfig::NoAuth
|
AuthConfig::NoAuth
|
||||||
};
|
};
|
||||||
|
|
||||||
config.app_data(auth_config)
|
config.app_data(auth_config).app_data(keys);
|
||||||
.app_data(keys);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "mini-dashboard")]
|
#[cfg(feature = "mini-dashboard")]
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
use std::env;
|
use std::env;
|
||||||
|
|
||||||
use actix_web::HttpServer;
|
use actix_web::HttpServer;
|
||||||
use meilisearch_http::{Opt, create_app, setup_meilisearch};
|
use meilisearch_http::{create_app, setup_meilisearch, Opt};
|
||||||
use meilisearch_lib::MeiliSearch;
|
use meilisearch_lib::MeiliSearch;
|
||||||
use structopt::StructOpt;
|
use structopt::StructOpt;
|
||||||
|
|
||||||
@ -23,7 +23,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
log_builder.init();
|
log_builder.init();
|
||||||
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -36,7 +35,9 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
match opt.env.as_ref() {
|
match opt.env.as_ref() {
|
||||||
"production" => {
|
"production" => {
|
||||||
if opt.master_key.is_none() {
|
if opt.master_key.is_none() {
|
||||||
anyhow::bail!("In production mode, the environment variable MEILI_MASTER_KEY is mandatory")
|
anyhow::bail!(
|
||||||
|
"In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"development" => (),
|
"development" => (),
|
||||||
|
@ -1,16 +1,16 @@
|
|||||||
|
use std::fs;
|
||||||
use std::io::{BufReader, Read};
|
use std::io::{BufReader, Read};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::fs;
|
|
||||||
|
|
||||||
use byte_unit::Byte;
|
use byte_unit::Byte;
|
||||||
|
use meilisearch_lib::options::IndexerOpts;
|
||||||
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
|
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
|
||||||
use rustls::{
|
use rustls::{
|
||||||
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
|
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
|
||||||
RootCertStore,
|
RootCertStore,
|
||||||
};
|
};
|
||||||
use structopt::StructOpt;
|
use structopt::StructOpt;
|
||||||
use meilisearch_lib::options::IndexerOpts;
|
|
||||||
|
|
||||||
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
|
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
|
||||||
|
|
||||||
@ -173,24 +173,30 @@ impl Opt {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
|
fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
|
||||||
let certfile = fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
|
let certfile =
|
||||||
|
fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
|
||||||
let mut reader = BufReader::new(certfile);
|
let mut reader = BufReader::new(certfile);
|
||||||
Ok(certs(&mut reader).map_err(|_| anyhow::anyhow!("cannot read certificate file"))?)
|
certs(&mut reader).map_err(|_| anyhow::anyhow!("cannot read certificate file"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
|
fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
|
||||||
let rsa_keys = {
|
let rsa_keys = {
|
||||||
let keyfile =
|
let keyfile = fs::File::open(filename.clone())
|
||||||
fs::File::open(filename.clone()).map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
|
.map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
|
||||||
let mut reader = BufReader::new(keyfile);
|
let mut reader = BufReader::new(keyfile);
|
||||||
rsa_private_keys(&mut reader).map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
|
rsa_private_keys(&mut reader)
|
||||||
|
.map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
|
||||||
};
|
};
|
||||||
|
|
||||||
let pkcs8_keys = {
|
let pkcs8_keys = {
|
||||||
let keyfile = fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
|
let keyfile = fs::File::open(filename)
|
||||||
|
.map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
|
||||||
let mut reader = BufReader::new(keyfile);
|
let mut reader = BufReader::new(keyfile);
|
||||||
pkcs8_private_keys(&mut reader)
|
pkcs8_private_keys(&mut reader).map_err(|_| {
|
||||||
.map_err(|_| anyhow::anyhow!("file contains invalid pkcs8 private key (encrypted keys not supported)"))?
|
anyhow::anyhow!(
|
||||||
|
"file contains invalid pkcs8 private key (encrypted keys not supported)"
|
||||||
|
)
|
||||||
|
})?
|
||||||
};
|
};
|
||||||
|
|
||||||
// prefer to load pkcs8 keys
|
// prefer to load pkcs8 keys
|
||||||
|
@ -11,7 +11,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
|
|||||||
.service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
|
.service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn create_dump(meilisearch: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> {
|
pub async fn create_dump(
|
||||||
|
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||||
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let res = meilisearch.create_dump().await?;
|
let res = meilisearch.create_dump().await?;
|
||||||
|
|
||||||
debug!("returns: {:?}", res);
|
debug!("returns: {:?}", res);
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
use actix_web::error::PayloadError;
|
use actix_web::error::PayloadError;
|
||||||
use actix_web::{web, HttpResponse};
|
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
|
use actix_web::{web, HttpResponse};
|
||||||
use futures::{Stream, StreamExt};
|
use futures::{Stream, StreamExt};
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use meilisearch_lib::MeiliSearch;
|
|
||||||
use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
|
use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
|
||||||
use meilisearch_lib::milli::update::IndexDocumentsMethod;
|
use meilisearch_lib::milli::update::IndexDocumentsMethod;
|
||||||
|
use meilisearch_lib::MeiliSearch;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
@ -104,9 +104,14 @@ pub async fn delete_document(
|
|||||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||||
path: web::Path<DocumentParam>,
|
path: web::Path<DocumentParam>,
|
||||||
) -> Result<HttpResponse, ResponseError> {
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let DocumentParam { document_id, index_uid } = path.into_inner();
|
let DocumentParam {
|
||||||
|
document_id,
|
||||||
|
index_uid,
|
||||||
|
} = path.into_inner();
|
||||||
let update = Update::DeleteDocuments(vec![document_id]);
|
let update = Update::DeleteDocuments(vec![document_id]);
|
||||||
let update_status = meilisearch.register_update(index_uid, update, false).await?;
|
let update_status = meilisearch
|
||||||
|
.register_update(index_uid, update, false)
|
||||||
|
.await?;
|
||||||
debug!("returns: {:?}", update_status);
|
debug!("returns: {:?}", update_status);
|
||||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||||
}
|
}
|
||||||
@ -216,7 +221,9 @@ pub async fn delete_documents(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let update = Update::DeleteDocuments(ids);
|
let update = Update::DeleteDocuments(ids);
|
||||||
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?;
|
let update_status = meilisearch
|
||||||
|
.register_update(path.into_inner().index_uid, update, false)
|
||||||
|
.await?;
|
||||||
debug!("returns: {:?}", update_status);
|
debug!("returns: {:?}", update_status);
|
||||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||||
}
|
}
|
||||||
@ -226,7 +233,9 @@ pub async fn clear_all_documents(
|
|||||||
path: web::Path<IndexParam>,
|
path: web::Path<IndexParam>,
|
||||||
) -> Result<HttpResponse, ResponseError> {
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let update = Update::ClearDocuments;
|
let update = Update::ClearDocuments;
|
||||||
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?;
|
let update_status = meilisearch
|
||||||
|
.register_update(path.into_inner().index_uid, update, false)
|
||||||
|
.await?;
|
||||||
debug!("returns: {:?}", update_status);
|
debug!("returns: {:?}", update_status);
|
||||||
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
use actix_web::{web, HttpResponse};
|
use actix_web::{web, HttpResponse};
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use meilisearch_lib::MeiliSearch;
|
|
||||||
use meilisearch_lib::index_controller::IndexSettings;
|
use meilisearch_lib::index_controller::IndexSettings;
|
||||||
|
use meilisearch_lib::MeiliSearch;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::error::ResponseError;
|
use crate::error::ResponseError;
|
||||||
@ -36,7 +36,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_indexes(data: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> {
|
pub async fn list_indexes(
|
||||||
|
data: GuardedData<Private, MeiliSearch>,
|
||||||
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let indexes = data.list_indexes().await?;
|
let indexes = data.list_indexes().await?;
|
||||||
debug!("returns: {:?}", indexes);
|
debug!("returns: {:?}", indexes);
|
||||||
Ok(HttpResponse::Ok().json(indexes))
|
Ok(HttpResponse::Ok().json(indexes))
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
use log::debug;
|
use log::debug;
|
||||||
|
|
||||||
use actix_web::{HttpResponse, web};
|
use actix_web::{web, HttpResponse};
|
||||||
use meilisearch_lib::MeiliSearch;
|
|
||||||
use meilisearch_lib::index::{Settings, Unchecked};
|
use meilisearch_lib::index::{Settings, Unchecked};
|
||||||
use meilisearch_lib::index_controller::Update;
|
use meilisearch_lib::index_controller::Update;
|
||||||
|
use meilisearch_lib::MeiliSearch;
|
||||||
|
|
||||||
use crate::extractors::authentication::{policies::*, GuardedData};
|
|
||||||
use crate::error::ResponseError;
|
use crate::error::ResponseError;
|
||||||
|
use crate::extractors::authentication::{policies::*, GuardedData};
|
||||||
|
|
||||||
#[macro_export]
|
#[macro_export]
|
||||||
macro_rules! make_setting_route {
|
macro_rules! make_setting_route {
|
||||||
|
@ -53,7 +53,9 @@ pub async fn get_all_updates_status(
|
|||||||
meilisearch: GuardedData<Private, MeiliSearch>,
|
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||||
path: web::Path<IndexParam>,
|
path: web::Path<IndexParam>,
|
||||||
) -> Result<HttpResponse, ResponseError> {
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let metas = meilisearch.all_update_status(path.into_inner().index_uid).await?;
|
let metas = meilisearch
|
||||||
|
.all_update_status(path.into_inner().index_uid)
|
||||||
|
.await?;
|
||||||
let metas = metas
|
let metas = metas
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(UpdateStatusResponse::from)
|
.map(UpdateStatusResponse::from)
|
||||||
|
@ -6,8 +6,8 @@ use log::debug;
|
|||||||
use meilisearch_lib::index_controller::updates::status::{UpdateResult, UpdateStatus};
|
use meilisearch_lib::index_controller::updates::status::{UpdateResult, UpdateStatus};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use meilisearch_lib::{MeiliSearch, Update};
|
|
||||||
use meilisearch_lib::index::{Settings, Unchecked};
|
use meilisearch_lib::index::{Settings, Unchecked};
|
||||||
|
use meilisearch_lib::{MeiliSearch, Update};
|
||||||
|
|
||||||
use crate::error::ResponseError;
|
use crate::error::ResponseError;
|
||||||
use crate::extractors::authentication::{policies::*, GuardedData};
|
use crate::extractors::authentication::{policies::*, GuardedData};
|
||||||
@ -233,7 +233,9 @@ pub async fn running() -> HttpResponse {
|
|||||||
HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
|
HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_stats(meilisearch: GuardedData<Private, MeiliSearch>) -> Result<HttpResponse, ResponseError> {
|
async fn get_stats(
|
||||||
|
meilisearch: GuardedData<Private, MeiliSearch>,
|
||||||
|
) -> Result<HttpResponse, ResponseError> {
|
||||||
let response = meilisearch.get_all_stats().await?;
|
let response = meilisearch.get_all_stats().await?;
|
||||||
|
|
||||||
debug!("returns: {:?}", response);
|
debug!("returns: {:?}", response);
|
||||||
|
@ -35,7 +35,10 @@ impl Server {
|
|||||||
let options = default_settings(dir.path());
|
let options = default_settings(dir.path());
|
||||||
|
|
||||||
let meilisearch = setup_meilisearch(&options).unwrap();
|
let meilisearch = setup_meilisearch(&options).unwrap();
|
||||||
let service = Service { meilisearch, options };
|
let service = Service {
|
||||||
|
meilisearch,
|
||||||
|
options,
|
||||||
|
};
|
||||||
|
|
||||||
Server {
|
Server {
|
||||||
service,
|
service,
|
||||||
@ -45,7 +48,10 @@ impl Server {
|
|||||||
|
|
||||||
pub async fn new_with_options(options: Opt) -> Self {
|
pub async fn new_with_options(options: Opt) -> Self {
|
||||||
let meilisearch = setup_meilisearch(&options).unwrap();
|
let meilisearch = setup_meilisearch(&options).unwrap();
|
||||||
let service = Service { meilisearch, options };
|
let service = Service {
|
||||||
|
meilisearch,
|
||||||
|
options,
|
||||||
|
};
|
||||||
|
|
||||||
Server {
|
Server {
|
||||||
service,
|
service,
|
||||||
|
@ -2,7 +2,7 @@ use actix_web::{http::StatusCode, test};
|
|||||||
use meilisearch_lib::MeiliSearch;
|
use meilisearch_lib::MeiliSearch;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use meilisearch_http::{Opt, create_app};
|
use meilisearch_http::{create_app, Opt};
|
||||||
|
|
||||||
pub struct Service {
|
pub struct Service {
|
||||||
pub meilisearch: MeiliSearch,
|
pub meilisearch: MeiliSearch,
|
||||||
|
@ -16,7 +16,12 @@ async fn add_documents_test_json_content_types() {
|
|||||||
|
|
||||||
// this is a what is expected and should work
|
// this is a what is expected and should work
|
||||||
let server = Server::new().await;
|
let server = Server::new().await;
|
||||||
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await;
|
let app = test::init_service(create_app!(
|
||||||
|
&server.service.meilisearch,
|
||||||
|
true,
|
||||||
|
&server.service.options
|
||||||
|
))
|
||||||
|
.await;
|
||||||
let req = test::TestRequest::post()
|
let req = test::TestRequest::post()
|
||||||
.uri("/indexes/dog/documents")
|
.uri("/indexes/dog/documents")
|
||||||
.set_payload(document.to_string())
|
.set_payload(document.to_string())
|
||||||
@ -41,7 +46,12 @@ async fn add_documents_test_no_content_types() {
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
let server = Server::new().await;
|
let server = Server::new().await;
|
||||||
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await;
|
let app = test::init_service(create_app!(
|
||||||
|
&server.service.meilisearch,
|
||||||
|
true,
|
||||||
|
&server.service.options
|
||||||
|
))
|
||||||
|
.await;
|
||||||
let req = test::TestRequest::post()
|
let req = test::TestRequest::post()
|
||||||
.uri("/indexes/dog/documents")
|
.uri("/indexes/dog/documents")
|
||||||
.set_payload(document.to_string())
|
.set_payload(document.to_string())
|
||||||
@ -67,7 +77,12 @@ async fn add_documents_test_bad_content_types() {
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
let server = Server::new().await;
|
let server = Server::new().await;
|
||||||
let app = test::init_service(create_app!(&server.service.meilisearch, true, &server.service.options)).await;
|
let app = test::init_service(create_app!(
|
||||||
|
&server.service.meilisearch,
|
||||||
|
true,
|
||||||
|
&server.service.options
|
||||||
|
))
|
||||||
|
.await;
|
||||||
let req = test::TestRequest::post()
|
let req = test::TestRequest::post()
|
||||||
.uri("/indexes/dog/documents")
|
.uri("/indexes/dog/documents")
|
||||||
.set_payload(document.to_string())
|
.set_payload(document.to_string())
|
||||||
|
@ -1,4 +1,7 @@
|
|||||||
use std::{fmt, io::{Read, Seek, Write}};
|
use std::{
|
||||||
|
fmt,
|
||||||
|
io::{Read, Seek, Write},
|
||||||
|
};
|
||||||
|
|
||||||
use milli::documents::DocumentBatchBuilder;
|
use milli::documents::DocumentBatchBuilder;
|
||||||
use serde_json::{Deserializer, Map, Value};
|
use serde_json::{Deserializer, Map, Value};
|
||||||
@ -25,12 +28,13 @@ pub enum DocumentFormatError {
|
|||||||
#[error("Internal error: {0}")]
|
#[error("Internal error: {0}")]
|
||||||
Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
|
Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
|
||||||
#[error("{0}. The {1} payload provided is malformed.")]
|
#[error("{0}. The {1} payload provided is malformed.")]
|
||||||
MalformedPayload(Box<dyn std::error::Error + Send + Sync + 'static>, PayloadType),
|
MalformedPayload(
|
||||||
|
Box<dyn std::error::Error + Send + Sync + 'static>,
|
||||||
|
PayloadType,
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
internal_error!(
|
internal_error!(DocumentFormatError: milli::documents::Error);
|
||||||
DocumentFormatError: milli::documents::Error
|
|
||||||
);
|
|
||||||
|
|
||||||
macro_rules! malformed {
|
macro_rules! malformed {
|
||||||
($type:path, $e:expr) => {
|
($type:path, $e:expr) => {
|
||||||
@ -57,7 +61,8 @@ pub fn read_jsonl(input: impl Read, writer: impl Write + Seek) -> Result<()> {
|
|||||||
pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<()> {
|
pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<()> {
|
||||||
let mut builder = DocumentBatchBuilder::new(writer).unwrap();
|
let mut builder = DocumentBatchBuilder::new(writer).unwrap();
|
||||||
|
|
||||||
let documents: Vec<Map<String, Value>> = malformed!(PayloadType::Json, serde_json::from_reader(input))?;
|
let documents: Vec<Map<String, Value>> =
|
||||||
|
malformed!(PayloadType::Json, serde_json::from_reader(input))?;
|
||||||
builder.add_documents(documents).unwrap();
|
builder.add_documents(documents).unwrap();
|
||||||
builder.finish().unwrap();
|
builder.finish().unwrap();
|
||||||
|
|
||||||
|
@ -151,7 +151,9 @@ impl Index {
|
|||||||
//If the document file is empty, we don't perform the document addition, to prevent
|
//If the document file is empty, we don't perform the document addition, to prevent
|
||||||
//a primary key error to be thrown.
|
//a primary key error to be thrown.
|
||||||
if !documents_reader.is_empty() {
|
if !documents_reader.is_empty() {
|
||||||
let builder = update_handler.update_builder(0).index_documents(&mut txn, &index);
|
let builder = update_handler
|
||||||
|
.update_builder(0)
|
||||||
|
.index_documents(&mut txn, &index);
|
||||||
builder.execute(documents_reader, |_, _| ())?;
|
builder.execute(documents_reader, |_, _| ())?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8,17 +8,17 @@ use std::sync::Arc;
|
|||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
use heed::{EnvOpenOptions, RoTxn};
|
use heed::{EnvOpenOptions, RoTxn};
|
||||||
use milli::update::Setting;
|
use milli::update::Setting;
|
||||||
use milli::{FieldDistribution, FieldId, obkv_to_json};
|
use milli::{obkv_to_json, FieldDistribution, FieldId};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
use serde::{Serialize, Deserialize};
|
|
||||||
|
|
||||||
use error::Result;
|
use error::Result;
|
||||||
pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
|
pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
|
||||||
pub use updates::{Checked, Facets, Settings, Unchecked, apply_settings_to_builder};
|
pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::EnvSizer;
|
|
||||||
use crate::index_controller::update_file_store::UpdateFileStore;
|
use crate::index_controller::update_file_store::UpdateFileStore;
|
||||||
|
use crate::EnvSizer;
|
||||||
|
|
||||||
use self::error::IndexError;
|
use self::error::IndexError;
|
||||||
use self::update_handler::UpdateHandler;
|
use self::update_handler::UpdateHandler;
|
||||||
@ -92,12 +92,23 @@ impl Deref for Index {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Index {
|
impl Index {
|
||||||
pub fn open(path: impl AsRef<Path>, size: usize, update_file_store: Arc<UpdateFileStore>, uuid: Uuid, update_handler: Arc<UpdateHandler>) -> Result<Self> {
|
pub fn open(
|
||||||
|
path: impl AsRef<Path>,
|
||||||
|
size: usize,
|
||||||
|
update_file_store: Arc<UpdateFileStore>,
|
||||||
|
uuid: Uuid,
|
||||||
|
update_handler: Arc<UpdateHandler>,
|
||||||
|
) -> Result<Self> {
|
||||||
create_dir_all(&path)?;
|
create_dir_all(&path)?;
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(size);
|
options.map_size(size);
|
||||||
let inner = Arc::new(milli::Index::new(options, &path)?);
|
let inner = Arc::new(milli::Index::new(options, &path)?);
|
||||||
Ok(Index { inner, update_file_store, uuid, update_handler })
|
Ok(Index {
|
||||||
|
inner,
|
||||||
|
update_file_store,
|
||||||
|
uuid,
|
||||||
|
update_handler,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn stats(&self) -> Result<IndexStats> {
|
pub fn stats(&self) -> Result<IndexStats> {
|
||||||
@ -268,7 +279,9 @@ impl Index {
|
|||||||
create_dir_all(&dst)?;
|
create_dir_all(&dst)?;
|
||||||
dst.push("data.mdb");
|
dst.push("data.mdb");
|
||||||
let _txn = self.write_txn()?;
|
let _txn = self.write_txn()?;
|
||||||
self.inner.env.copy_to_path(dst, heed::CompactionOption::Enabled)?;
|
self.inner
|
||||||
|
.env
|
||||||
|
.copy_to_path(dst, heed::CompactionOption::Enabled)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -6,9 +6,7 @@ use either::Either;
|
|||||||
use heed::RoTxn;
|
use heed::RoTxn;
|
||||||
use indexmap::IndexMap;
|
use indexmap::IndexMap;
|
||||||
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
|
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
|
||||||
use milli::{
|
use milli::{AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, SortError};
|
||||||
AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, SortError
|
|
||||||
};
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
@ -685,7 +683,7 @@ fn parse_filter_array(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(FilterCondition::from_array(txn, &index, ands)?)
|
Ok(FilterCondition::from_array(txn, index, ands)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
@ -8,11 +8,11 @@ use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder};
|
|||||||
use serde::{Deserialize, Serialize, Serializer};
|
use serde::{Deserialize, Serialize, Serializer};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::Update;
|
|
||||||
use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult};
|
use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult};
|
||||||
|
use crate::Update;
|
||||||
|
|
||||||
use super::{Index, IndexMeta};
|
|
||||||
use super::error::{IndexError, Result};
|
use super::error::{IndexError, Result};
|
||||||
|
use super::{Index, IndexMeta};
|
||||||
|
|
||||||
fn serialize_with_wildcard<S>(
|
fn serialize_with_wildcard<S>(
|
||||||
field: &Setting<Vec<String>>,
|
field: &Setting<Vec<String>>,
|
||||||
@ -170,18 +170,26 @@ impl Index {
|
|||||||
let result = (|| {
|
let result = (|| {
|
||||||
let mut txn = self.write_txn()?;
|
let mut txn = self.write_txn()?;
|
||||||
let result = match update.meta() {
|
let result = match update.meta() {
|
||||||
Update::DocumentAddition { primary_key, content_uuid, method } => {
|
Update::DocumentAddition {
|
||||||
self.update_documents(&mut txn, *method, *content_uuid, update_builder, primary_key.as_deref())
|
primary_key,
|
||||||
}
|
content_uuid,
|
||||||
|
method,
|
||||||
|
} => self.update_documents(
|
||||||
|
&mut txn,
|
||||||
|
*method,
|
||||||
|
*content_uuid,
|
||||||
|
update_builder,
|
||||||
|
primary_key.as_deref(),
|
||||||
|
),
|
||||||
Update::Settings(settings) => {
|
Update::Settings(settings) => {
|
||||||
let settings = settings.clone().check();
|
let settings = settings.clone().check();
|
||||||
self.update_settings(&mut txn, &settings, update_builder)
|
self.update_settings(&mut txn, &settings, update_builder)
|
||||||
},
|
}
|
||||||
Update::ClearDocuments => {
|
Update::ClearDocuments => {
|
||||||
let builder = update_builder.clear_documents(&mut txn, self);
|
let builder = update_builder.clear_documents(&mut txn, self);
|
||||||
let _count = builder.execute()?;
|
let _count = builder.execute()?;
|
||||||
Ok(UpdateResult::Other)
|
Ok(UpdateResult::Other)
|
||||||
},
|
}
|
||||||
Update::DeleteDocuments(ids) => {
|
Update::DeleteDocuments(ids) => {
|
||||||
let mut builder = update_builder.delete_documents(&mut txn, self)?;
|
let mut builder = update_builder.delete_documents(&mut txn, self)?;
|
||||||
|
|
||||||
@ -276,7 +284,10 @@ impl Index {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut milli::update::Settings) {
|
pub fn apply_settings_to_builder(
|
||||||
|
settings: &Settings<Checked>,
|
||||||
|
builder: &mut milli::update::Settings,
|
||||||
|
) {
|
||||||
match settings.searchable_attributes {
|
match settings.searchable_attributes {
|
||||||
Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
|
Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
|
||||||
Setting::Reset => builder.reset_searchable_fields(),
|
Setting::Reset => builder.reset_searchable_fields(),
|
||||||
@ -298,9 +309,7 @@ pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut mil
|
|||||||
}
|
}
|
||||||
|
|
||||||
match settings.sortable_attributes {
|
match settings.sortable_attributes {
|
||||||
Setting::Set(ref fields) => {
|
Setting::Set(ref fields) => builder.set_sortable_fields(fields.iter().cloned().collect()),
|
||||||
builder.set_sortable_fields(fields.iter().cloned().collect())
|
|
||||||
}
|
|
||||||
Setting::Reset => builder.reset_sortable_fields(),
|
Setting::Reset => builder.reset_sortable_fields(),
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => (),
|
||||||
}
|
}
|
||||||
@ -318,9 +327,7 @@ pub fn apply_settings_to_builder(settings: &Settings<Checked>, builder: &mut mil
|
|||||||
}
|
}
|
||||||
|
|
||||||
match settings.synonyms {
|
match settings.synonyms {
|
||||||
Setting::Set(ref synonyms) => {
|
Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
|
||||||
builder.set_synonyms(synonyms.clone().into_iter().collect())
|
|
||||||
}
|
|
||||||
Setting::Reset => builder.reset_synonyms(),
|
Setting::Reset => builder.reset_synonyms(),
|
||||||
Setting::NotSet => (),
|
Setting::NotSet => (),
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::fs::{File, create_dir_all};
|
use std::fs::{create_dir_all, File};
|
||||||
use std::io::{BufReader, Seek, SeekFrom};
|
use std::io::{BufReader, Seek, SeekFrom};
|
||||||
use std::marker::PhantomData;
|
use std::marker::PhantomData;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
@ -17,10 +17,7 @@ use crate::index::update_handler::UpdateHandler;
|
|||||||
use crate::index_controller::index_resolver::uuid_store::HeedUuidStore;
|
use crate::index_controller::index_resolver::uuid_store::HeedUuidStore;
|
||||||
use crate::index_controller::{self, IndexMetadata};
|
use crate::index_controller::{self, IndexMetadata};
|
||||||
use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
|
use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
|
||||||
use crate::{
|
use crate::{index::Unchecked, options::IndexerOpts};
|
||||||
index::Unchecked,
|
|
||||||
options::IndexerOpts,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug)]
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
@ -136,16 +133,16 @@ fn load_index(
|
|||||||
//If the document file is empty, we don't perform the document addition, to prevent
|
//If the document file is empty, we don't perform the document addition, to prevent
|
||||||
//a primary key error to be thrown.
|
//a primary key error to be thrown.
|
||||||
if !documents_reader.is_empty() {
|
if !documents_reader.is_empty() {
|
||||||
let builder = update_handler.update_builder(0).index_documents(&mut txn, &index);
|
let builder = update_handler
|
||||||
|
.update_builder(0)
|
||||||
|
.index_documents(&mut txn, &index);
|
||||||
builder.execute(documents_reader, |_, _| ())?;
|
builder.execute(documents_reader, |_, _| ())?;
|
||||||
}
|
}
|
||||||
|
|
||||||
txn.commit()?;
|
txn.commit()?;
|
||||||
|
|
||||||
// Finaly, we extract the original milli::Index and close it
|
// Finaly, we extract the original milli::Index and close it
|
||||||
index
|
index.prepare_for_closing().wait();
|
||||||
.prepare_for_closing()
|
|
||||||
.wait();
|
|
||||||
|
|
||||||
// Updates are ignored in dumps V1.
|
// Updates are ignored in dumps V1.
|
||||||
|
|
||||||
|
@ -158,8 +158,7 @@ impl DumpTask {
|
|||||||
|
|
||||||
create_dir_all(&self.path).await?;
|
create_dir_all(&self.path).await?;
|
||||||
|
|
||||||
let temp_dump_dir =
|
let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??;
|
||||||
tokio::task::spawn_blocking(|| tempfile::TempDir::new()).await??;
|
|
||||||
let temp_dump_path = temp_dump_dir.path().to_owned();
|
let temp_dump_path = temp_dump_dir.path().to_owned();
|
||||||
|
|
||||||
let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);
|
let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);
|
||||||
|
@ -27,7 +27,8 @@ pub enum IndexResolverError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<T> From<MpscSendError<T>> for IndexResolverError
|
impl<T> From<MpscSendError<T>> for IndexResolverError
|
||||||
where T: Send + Sync + 'static + fmt::Debug
|
where
|
||||||
|
T: Send + Sync + 'static + fmt::Debug,
|
||||||
{
|
{
|
||||||
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
|
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
|
||||||
Self::Internal(Box::new(other))
|
Self::Internal(Box::new(other))
|
||||||
|
@ -9,8 +9,8 @@ use tokio::task::spawn_blocking;
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::error::{IndexResolverError, Result};
|
use super::error::{IndexResolverError, Result};
|
||||||
use crate::index::Index;
|
|
||||||
use crate::index::update_handler::UpdateHandler;
|
use crate::index::update_handler::UpdateHandler;
|
||||||
|
use crate::index::Index;
|
||||||
use crate::index_controller::update_file_store::UpdateFileStore;
|
use crate::index_controller::update_file_store::UpdateFileStore;
|
||||||
use crate::options::IndexerOpts;
|
use crate::options::IndexerOpts;
|
||||||
|
|
||||||
@ -32,7 +32,11 @@ pub struct MapIndexStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MapIndexStore {
|
impl MapIndexStore {
|
||||||
pub fn new(path: impl AsRef<Path>, index_size: usize, indexer_opts: &IndexerOpts) -> anyhow::Result<Self> {
|
pub fn new(
|
||||||
|
path: impl AsRef<Path>,
|
||||||
|
index_size: usize,
|
||||||
|
indexer_opts: &IndexerOpts,
|
||||||
|
) -> anyhow::Result<Self> {
|
||||||
let update_handler = Arc::new(UpdateHandler::new(indexer_opts)?);
|
let update_handler = Arc::new(UpdateHandler::new(indexer_opts)?);
|
||||||
let update_file_store = Arc::new(UpdateFileStore::new(path.as_ref()).unwrap());
|
let update_file_store = Arc::new(UpdateFileStore::new(path.as_ref()).unwrap());
|
||||||
let path = path.as_ref().join("indexes/");
|
let path = path.as_ref().join("indexes/");
|
||||||
@ -100,7 +104,10 @@ impl IndexStore for MapIndexStore {
|
|||||||
let index_size = self.index_size;
|
let index_size = self.index_size;
|
||||||
let file_store = self.update_file_store.clone();
|
let file_store = self.update_file_store.clone();
|
||||||
let update_handler = self.update_handler.clone();
|
let update_handler = self.update_handler.clone();
|
||||||
let index = spawn_blocking(move || Index::open(path, index_size, file_store, uuid, update_handler)).await??;
|
let index = spawn_blocking(move || {
|
||||||
|
Index::open(path, index_size, file_store, uuid, update_handler)
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
self.index_store.write().await.insert(uuid, index.clone());
|
self.index_store.write().await.insert(uuid, index.clone());
|
||||||
Ok(Some(index))
|
Ok(Some(index))
|
||||||
}
|
}
|
||||||
|
@ -1,19 +1,26 @@
|
|||||||
pub mod uuid_store;
|
|
||||||
mod index_store;
|
|
||||||
pub mod error;
|
pub mod error;
|
||||||
|
mod index_store;
|
||||||
|
pub mod uuid_store;
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use uuid::Uuid;
|
use error::{IndexResolverError, Result};
|
||||||
use uuid_store::{UuidStore, HeedUuidStore};
|
|
||||||
use index_store::{IndexStore, MapIndexStore};
|
use index_store::{IndexStore, MapIndexStore};
|
||||||
use error::{Result, IndexResolverError};
|
use uuid::Uuid;
|
||||||
|
use uuid_store::{HeedUuidStore, UuidStore};
|
||||||
|
|
||||||
use crate::{index::{Index, update_handler::UpdateHandler}, options::IndexerOpts};
|
use crate::{
|
||||||
|
index::{update_handler::UpdateHandler, Index},
|
||||||
|
options::IndexerOpts,
|
||||||
|
};
|
||||||
|
|
||||||
pub type HardStateIndexResolver = IndexResolver<HeedUuidStore, MapIndexStore>;
|
pub type HardStateIndexResolver = IndexResolver<HeedUuidStore, MapIndexStore>;
|
||||||
|
|
||||||
pub fn create_index_resolver(path: impl AsRef<Path>, index_size: usize, indexer_opts: &IndexerOpts) -> anyhow::Result<HardStateIndexResolver> {
|
pub fn create_index_resolver(
|
||||||
|
path: impl AsRef<Path>,
|
||||||
|
index_size: usize,
|
||||||
|
indexer_opts: &IndexerOpts,
|
||||||
|
) -> anyhow::Result<HardStateIndexResolver> {
|
||||||
let uuid_store = HeedUuidStore::new(&path)?;
|
let uuid_store = HeedUuidStore::new(&path)?;
|
||||||
let index_store = MapIndexStore::new(&path, index_size, indexer_opts)?;
|
let index_store = MapIndexStore::new(&path, index_size, indexer_opts)?;
|
||||||
Ok(IndexResolver::new(uuid_store, index_store))
|
Ok(IndexResolver::new(uuid_store, index_store))
|
||||||
@ -47,13 +54,11 @@ impl IndexResolver<HeedUuidStore, MapIndexStore> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<U, I> IndexResolver<U, I>
|
impl<U, I> IndexResolver<U, I>
|
||||||
where U: UuidStore,
|
where
|
||||||
|
U: UuidStore,
|
||||||
I: IndexStore,
|
I: IndexStore,
|
||||||
{
|
{
|
||||||
pub fn new(
|
pub fn new(index_uuid_store: U, index_store: I) -> Self {
|
||||||
index_uuid_store: U,
|
|
||||||
index_store: I,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
index_uuid_store,
|
index_uuid_store,
|
||||||
index_store,
|
index_store,
|
||||||
@ -75,7 +80,10 @@ where U: UuidStore,
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn snapshot(&self, path: impl AsRef<Path>) -> Result<Vec<Index>> {
|
pub async fn snapshot(&self, path: impl AsRef<Path>) -> Result<Vec<Index>> {
|
||||||
let uuids = self.index_uuid_store.snapshot(path.as_ref().to_owned()).await?;
|
let uuids = self
|
||||||
|
.index_uuid_store
|
||||||
|
.snapshot(path.as_ref().to_owned())
|
||||||
|
.await?;
|
||||||
let mut indexes = Vec::new();
|
let mut indexes = Vec::new();
|
||||||
for uuid in uuids {
|
for uuid in uuids {
|
||||||
indexes.push(self.get_index_by_uuid(uuid).await?);
|
indexes.push(self.get_index_by_uuid(uuid).await?);
|
||||||
@ -99,13 +107,11 @@ where U: UuidStore,
|
|||||||
let mut indexes = Vec::new();
|
let mut indexes = Vec::new();
|
||||||
for (name, uuid) in uuids {
|
for (name, uuid) in uuids {
|
||||||
match self.index_store.get(uuid).await? {
|
match self.index_store.get(uuid).await? {
|
||||||
Some(index) => {
|
Some(index) => indexes.push((name, index)),
|
||||||
indexes.push((name, index))
|
|
||||||
},
|
|
||||||
None => {
|
None => {
|
||||||
// we found an unexisting index, we remove it from the uuid store
|
// we found an unexisting index, we remove it from the uuid store
|
||||||
let _ = self.index_uuid_store.delete(name).await;
|
let _ = self.index_uuid_store.delete(name).await;
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -124,7 +130,10 @@ where U: UuidStore,
|
|||||||
|
|
||||||
pub async fn get_index_by_uuid(&self, uuid: Uuid) -> Result<Index> {
|
pub async fn get_index_by_uuid(&self, uuid: Uuid) -> Result<Index> {
|
||||||
// TODO: Handle this error better.
|
// TODO: Handle this error better.
|
||||||
self.index_store.get(uuid).await?.ok_or(IndexResolverError::UnexistingIndex(String::new()))
|
self.index_store
|
||||||
|
.get(uuid)
|
||||||
|
.await?
|
||||||
|
.ok_or_else(|| IndexResolverError::UnexistingIndex(String::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_index(&self, uid: String) -> Result<Index> {
|
pub async fn get_index(&self, uid: String) -> Result<Index> {
|
||||||
@ -137,17 +146,17 @@ where U: UuidStore,
|
|||||||
// and remove the uuid from th uuid store.
|
// and remove the uuid from th uuid store.
|
||||||
let _ = self.index_uuid_store.delete(name.clone()).await;
|
let _ = self.index_uuid_store.delete(name.clone()).await;
|
||||||
Err(IndexResolverError::UnexistingIndex(name))
|
Err(IndexResolverError::UnexistingIndex(name))
|
||||||
},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
(name, _) => Err(IndexResolverError::UnexistingIndex(name))
|
}
|
||||||
|
(name, _) => Err(IndexResolverError::UnexistingIndex(name)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_uuid(&self, uid: String) -> Result<Uuid> {
|
pub async fn get_uuid(&self, uid: String) -> Result<Uuid> {
|
||||||
match self.index_uuid_store.get_uuid(uid).await? {
|
match self.index_uuid_store.get_uuid(uid).await? {
|
||||||
(_, Some(uuid)) => Ok(uuid),
|
(_, Some(uuid)) => Ok(uuid),
|
||||||
(name, _) => Err(IndexResolverError::UnexistingIndex(name))
|
(name, _) => Err(IndexResolverError::UnexistingIndex(name)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -8,7 +8,7 @@ use heed::{CompactionOption, Database, Env, EnvOpenOptions};
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::error::{Result, IndexResolverError};
|
use super::error::{IndexResolverError, Result};
|
||||||
use crate::EnvSizer;
|
use crate::EnvSizer;
|
||||||
|
|
||||||
const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB
|
const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB
|
||||||
|
@ -18,25 +18,27 @@ use dump_actor::DumpActorHandle;
|
|||||||
pub use dump_actor::{DumpInfo, DumpStatus};
|
pub use dump_actor::{DumpInfo, DumpStatus};
|
||||||
use snapshot::load_snapshot;
|
use snapshot::load_snapshot;
|
||||||
|
|
||||||
use crate::index::{Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked};
|
use crate::index::error::Result as IndexResult;
|
||||||
|
use crate::index::{
|
||||||
|
Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked,
|
||||||
|
};
|
||||||
use crate::index_controller::index_resolver::create_index_resolver;
|
use crate::index_controller::index_resolver::create_index_resolver;
|
||||||
use crate::index_controller::snapshot::SnapshotService;
|
use crate::index_controller::snapshot::SnapshotService;
|
||||||
use crate::options::IndexerOpts;
|
use crate::options::IndexerOpts;
|
||||||
use error::Result;
|
use error::Result;
|
||||||
use crate::index::error::{Result as IndexResult};
|
|
||||||
|
|
||||||
use self::dump_actor::load_dump;
|
use self::dump_actor::load_dump;
|
||||||
use self::index_resolver::HardStateIndexResolver;
|
|
||||||
use self::index_resolver::error::IndexResolverError;
|
use self::index_resolver::error::IndexResolverError;
|
||||||
|
use self::index_resolver::HardStateIndexResolver;
|
||||||
use self::updates::status::UpdateStatus;
|
use self::updates::status::UpdateStatus;
|
||||||
use self::updates::UpdateMsg;
|
use self::updates::UpdateMsg;
|
||||||
|
|
||||||
mod dump_actor;
|
mod dump_actor;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
|
mod index_resolver;
|
||||||
mod snapshot;
|
mod snapshot;
|
||||||
pub mod update_file_store;
|
pub mod update_file_store;
|
||||||
pub mod updates;
|
pub mod updates;
|
||||||
mod index_resolver;
|
|
||||||
|
|
||||||
pub type Payload = Box<
|
pub type Payload = Box<
|
||||||
dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
|
dyn Stream<Item = std::result::Result<Bytes, PayloadError>> + Send + Sync + 'static + Unpin,
|
||||||
@ -79,6 +81,7 @@ pub struct Stats {
|
|||||||
pub indexes: BTreeMap<String, IndexStats>,
|
pub indexes: BTreeMap<String, IndexStats>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
#[derive(derivative::Derivative)]
|
#[derive(derivative::Derivative)]
|
||||||
#[derivative(Debug)]
|
#[derivative(Debug)]
|
||||||
pub enum Update {
|
pub enum Update {
|
||||||
@ -141,12 +144,19 @@ impl IndexControllerBuilder {
|
|||||||
|
|
||||||
std::fs::create_dir_all(db_path.as_ref())?;
|
std::fs::create_dir_all(db_path.as_ref())?;
|
||||||
|
|
||||||
let index_resolver = Arc::new(create_index_resolver(&db_path, index_size, &indexer_options)?);
|
let index_resolver = Arc::new(create_index_resolver(
|
||||||
|
&db_path,
|
||||||
|
index_size,
|
||||||
|
&indexer_options,
|
||||||
|
)?);
|
||||||
|
|
||||||
#[allow(unreachable_code)]
|
#[allow(unreachable_code)]
|
||||||
let update_sender = updates::create_update_handler(index_resolver.clone(), &db_path, update_store_size)?;
|
let update_sender =
|
||||||
|
updates::create_update_handler(index_resolver.clone(), &db_path, update_store_size)?;
|
||||||
|
|
||||||
let dump_path = self.dump_dst.ok_or_else(|| anyhow::anyhow!("Missing dump directory path"))?;
|
let dump_path = self
|
||||||
|
.dump_dst
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("Missing dump directory path"))?;
|
||||||
let dump_handle = dump_actor::DumpActorHandleImpl::new(
|
let dump_handle = dump_actor::DumpActorHandleImpl::new(
|
||||||
dump_path,
|
dump_path,
|
||||||
index_resolver.clone(),
|
index_resolver.clone(),
|
||||||
@ -159,8 +169,10 @@ impl IndexControllerBuilder {
|
|||||||
let snapshot_service = SnapshotService::new(
|
let snapshot_service = SnapshotService::new(
|
||||||
index_resolver.clone(),
|
index_resolver.clone(),
|
||||||
update_sender.clone(),
|
update_sender.clone(),
|
||||||
self.snapshot_interval.ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?,
|
self.snapshot_interval
|
||||||
self.snapshot_dir.ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?,
|
.ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?,
|
||||||
|
self.snapshot_dir
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?,
|
||||||
db_path
|
db_path
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.file_name()
|
.file_name()
|
||||||
@ -246,7 +258,12 @@ impl IndexController {
|
|||||||
IndexControllerBuilder::default()
|
IndexControllerBuilder::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn register_update(&self, uid: String, update: Update, create_index: bool) -> Result<UpdateStatus> {
|
pub async fn register_update(
|
||||||
|
&self,
|
||||||
|
uid: String,
|
||||||
|
update: Update,
|
||||||
|
create_index: bool,
|
||||||
|
) -> Result<UpdateStatus> {
|
||||||
match self.index_resolver.get_uuid(uid).await {
|
match self.index_resolver.get_uuid(uid).await {
|
||||||
Ok(uuid) => {
|
Ok(uuid) => {
|
||||||
let update_result = UpdateMsg::update(&self.update_sender, uuid, update).await?;
|
let update_result = UpdateMsg::update(&self.update_sender, uuid, update).await?;
|
||||||
@ -255,7 +272,8 @@ impl IndexController {
|
|||||||
Err(IndexResolverError::UnexistingIndex(name)) => {
|
Err(IndexResolverError::UnexistingIndex(name)) => {
|
||||||
if create_index {
|
if create_index {
|
||||||
let index = self.index_resolver.create_index(name, None).await?;
|
let index = self.index_resolver.create_index(name, None).await?;
|
||||||
let update_result = UpdateMsg::update(&self.update_sender, index.uuid, update).await?;
|
let update_result =
|
||||||
|
UpdateMsg::update(&self.update_sender, index.uuid, update).await?;
|
||||||
// ignore if index creation fails now, since it may already have been created
|
// ignore if index creation fails now, since it may already have been created
|
||||||
|
|
||||||
Ok(update_result)
|
Ok(update_result)
|
||||||
@ -310,7 +328,9 @@ impl IndexController {
|
|||||||
attributes_to_retrieve: Option<Vec<String>>,
|
attributes_to_retrieve: Option<Vec<String>>,
|
||||||
) -> Result<Vec<Document>> {
|
) -> Result<Vec<Document>> {
|
||||||
let index = self.index_resolver.get_index(uid).await?;
|
let index = self.index_resolver.get_index(uid).await?;
|
||||||
let documents = spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve)).await??;
|
let documents =
|
||||||
|
spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
|
||||||
|
.await??;
|
||||||
Ok(documents)
|
Ok(documents)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -321,7 +341,9 @@ impl IndexController {
|
|||||||
attributes_to_retrieve: Option<Vec<String>>,
|
attributes_to_retrieve: Option<Vec<String>>,
|
||||||
) -> Result<Document> {
|
) -> Result<Document> {
|
||||||
let index = self.index_resolver.get_index(uid).await?;
|
let index = self.index_resolver.get_index(uid).await?;
|
||||||
let document = spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve)).await??;
|
let document =
|
||||||
|
spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve))
|
||||||
|
.await??;
|
||||||
Ok(document)
|
Ok(document)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -330,12 +352,12 @@ impl IndexController {
|
|||||||
uid: String,
|
uid: String,
|
||||||
mut index_settings: IndexSettings,
|
mut index_settings: IndexSettings,
|
||||||
) -> Result<IndexMetadata> {
|
) -> Result<IndexMetadata> {
|
||||||
|
|
||||||
index_settings.uid.take();
|
index_settings.uid.take();
|
||||||
|
|
||||||
let index = self.index_resolver.get_index(uid.clone()).await?;
|
let index = self.index_resolver.get_index(uid.clone()).await?;
|
||||||
let uuid = index.uuid;
|
let uuid = index.uuid;
|
||||||
let meta = spawn_blocking(move || index.update_primary_key(index_settings.primary_key)).await??;
|
let meta =
|
||||||
|
spawn_blocking(move || index.update_primary_key(index_settings.primary_key)).await??;
|
||||||
let meta = IndexMetadata {
|
let meta = IndexMetadata {
|
||||||
uuid,
|
uuid,
|
||||||
name: uid.clone(),
|
name: uid.clone(),
|
||||||
@ -386,7 +408,8 @@ impl IndexController {
|
|||||||
let stats = index.stats()?;
|
let stats = index.stats()?;
|
||||||
let meta = index.meta()?;
|
let meta = index.meta()?;
|
||||||
Ok((stats, meta))
|
Ok((stats, meta))
|
||||||
}).await??;
|
})
|
||||||
|
.await??;
|
||||||
|
|
||||||
database_size += stats.size;
|
database_size += stats.size;
|
||||||
|
|
||||||
@ -415,8 +438,15 @@ impl IndexController {
|
|||||||
Ok(self.dump_handle.dump_info(uid).await?)
|
Ok(self.dump_handle.dump_info(uid).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn create_index(&self, uid: String, primary_key: Option<String>) -> Result<IndexMetadata> {
|
pub async fn create_index(
|
||||||
let index = self.index_resolver.create_index(uid.clone(), primary_key).await?;
|
&self,
|
||||||
|
uid: String,
|
||||||
|
primary_key: Option<String>,
|
||||||
|
) -> Result<IndexMetadata> {
|
||||||
|
let index = self
|
||||||
|
.index_resolver
|
||||||
|
.create_index(uid.clone(), primary_key)
|
||||||
|
.await?;
|
||||||
let meta = spawn_blocking(move || -> IndexResult<_> {
|
let meta = spawn_blocking(move || -> IndexResult<_> {
|
||||||
let meta = index.meta()?;
|
let meta = index.meta()?;
|
||||||
let meta = IndexMetadata {
|
let meta = IndexMetadata {
|
||||||
@ -426,7 +456,8 @@ impl IndexController {
|
|||||||
meta,
|
meta,
|
||||||
};
|
};
|
||||||
Ok(meta)
|
Ok(meta)
|
||||||
}).await??;
|
})
|
||||||
|
.await??;
|
||||||
|
|
||||||
Ok(meta)
|
Ok(meta)
|
||||||
}
|
}
|
||||||
|
@ -4,14 +4,14 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use anyhow::bail;
|
use anyhow::bail;
|
||||||
use log::{error, info, trace};
|
use log::{error, info, trace};
|
||||||
|
use tokio::fs;
|
||||||
use tokio::task::spawn_blocking;
|
use tokio::task::spawn_blocking;
|
||||||
use tokio::time::sleep;
|
use tokio::time::sleep;
|
||||||
use tokio::fs;
|
|
||||||
|
|
||||||
use crate::index_controller::updates::UpdateMsg;
|
use crate::index_controller::updates::UpdateMsg;
|
||||||
|
|
||||||
use super::updates::UpdateSender;
|
|
||||||
use super::index_resolver::HardStateIndexResolver;
|
use super::index_resolver::HardStateIndexResolver;
|
||||||
|
use super::updates::UpdateSender;
|
||||||
|
|
||||||
pub struct SnapshotService {
|
pub struct SnapshotService {
|
||||||
index_resolver: Arc<HardStateIndexResolver>,
|
index_resolver: Arc<HardStateIndexResolver>,
|
||||||
@ -56,8 +56,7 @@ impl SnapshotService {
|
|||||||
|
|
||||||
let snapshot_dir = self.snapshot_path.clone();
|
let snapshot_dir = self.snapshot_path.clone();
|
||||||
fs::create_dir_all(&snapshot_dir).await?;
|
fs::create_dir_all(&snapshot_dir).await?;
|
||||||
let temp_snapshot_dir =
|
let temp_snapshot_dir = spawn_blocking(tempfile::tempdir).await??;
|
||||||
spawn_blocking(move || tempfile::tempdir()).await??;
|
|
||||||
let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
|
let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
|
||||||
|
|
||||||
let indexes = self
|
let indexes = self
|
||||||
|
@ -151,7 +151,7 @@ impl UpdateFileStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
serde_json::to_writer(&mut dst_file, &document_buffer)?;
|
serde_json::to_writer(&mut dst_file, &document_buffer)?;
|
||||||
dst_file.write(b"\n")?;
|
dst_file.write_all(b"\n")?;
|
||||||
document_buffer.clear();
|
document_buffer.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
use std::fmt;
|
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
use meilisearch_error::{Code, ErrorCode};
|
use meilisearch_error::{Code, ErrorCode};
|
||||||
|
|
||||||
use crate::{document_formats::DocumentFormatError, index_controller::update_file_store::UpdateFileStoreError};
|
use crate::{
|
||||||
|
document_formats::DocumentFormatError,
|
||||||
|
index_controller::update_file_store::UpdateFileStoreError,
|
||||||
|
};
|
||||||
|
|
||||||
pub type Result<T> = std::result::Result<T, UpdateLoopError>;
|
pub type Result<T> = std::result::Result<T, UpdateLoopError>;
|
||||||
|
|
||||||
@ -28,7 +31,8 @@ pub enum UpdateLoopError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateLoopError
|
impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateLoopError
|
||||||
where T: Sync + Send + 'static + fmt::Debug
|
where
|
||||||
|
T: Sync + Send + 'static + fmt::Debug,
|
||||||
{
|
{
|
||||||
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
|
fn from(other: tokio::sync::mpsc::error::SendError<T>) -> Self {
|
||||||
Self::Internal(Box::new(other))
|
Self::Internal(Box::new(other))
|
||||||
|
@ -44,7 +44,11 @@ pub enum UpdateMsg {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl UpdateMsg {
|
impl UpdateMsg {
|
||||||
pub async fn snapshot(sender: &mpsc::Sender<Self>, path: PathBuf, indexes: Vec<Index>) -> Result<()> {
|
pub async fn snapshot(
|
||||||
|
sender: &mpsc::Sender<Self>,
|
||||||
|
path: PathBuf,
|
||||||
|
indexes: Vec<Index>,
|
||||||
|
) -> Result<()> {
|
||||||
let (ret, rcv) = oneshot::channel();
|
let (ret, rcv) = oneshot::channel();
|
||||||
let msg = Self::Snapshot { path, indexes, ret };
|
let msg = Self::Snapshot { path, indexes, ret };
|
||||||
sender.send(msg).await?;
|
sender.send(msg).await?;
|
||||||
|
@ -80,7 +80,7 @@ impl<S: Stream<Item = std::result::Result<Bytes, PayloadError>> + Unpin> io::Rea
|
|||||||
self.read(buf)
|
self.read(buf)
|
||||||
}
|
}
|
||||||
Some(Err(e)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, e)),
|
Some(Err(e)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, e)),
|
||||||
None => return Ok(0),
|
None => Ok(0),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -109,7 +109,13 @@ impl UpdateLoop {
|
|||||||
let must_exit = Arc::new(AtomicBool::new(false));
|
let must_exit = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
let update_file_store = UpdateFileStore::new(&path).unwrap();
|
let update_file_store = UpdateFileStore::new(&path).unwrap();
|
||||||
let store = UpdateStore::open(options, &path, index_resolver.clone(), must_exit.clone(), update_file_store.clone())?;
|
let store = UpdateStore::open(
|
||||||
|
options,
|
||||||
|
&path,
|
||||||
|
index_resolver,
|
||||||
|
must_exit.clone(),
|
||||||
|
update_file_store.clone(),
|
||||||
|
)?;
|
||||||
|
|
||||||
let inbox = Some(inbox);
|
let inbox = Some(inbox);
|
||||||
|
|
||||||
@ -194,8 +200,8 @@ impl UpdateLoop {
|
|||||||
update_file.persist()?;
|
update_file.persist()?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}).await??;
|
})
|
||||||
|
.await??;
|
||||||
|
|
||||||
store::Update::DocumentAddition {
|
store::Update::DocumentAddition {
|
||||||
primary_key,
|
primary_key,
|
||||||
@ -216,7 +222,6 @@ impl UpdateLoop {
|
|||||||
Ok(status.into())
|
Ok(status.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
|
async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
|
||||||
let update_store = self.store.clone();
|
let update_store = self.store.clone();
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
@ -248,8 +253,7 @@ impl UpdateLoop {
|
|||||||
async fn handle_snapshot(&self, indexes: Vec<Index>, path: PathBuf) -> Result<()> {
|
async fn handle_snapshot(&self, indexes: Vec<Index>, path: PathBuf) -> Result<()> {
|
||||||
let update_store = self.store.clone();
|
let update_store = self.store.clone();
|
||||||
|
|
||||||
tokio::task::spawn_blocking(move || update_store.snapshot(indexes, path))
|
tokio::task::spawn_blocking(move || update_store.snapshot(indexes, path)).await??;
|
||||||
.await??;
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,10 @@ use meilisearch_error::{Code, ErrorCode};
|
|||||||
use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
|
use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::{Update, index::{Settings, Unchecked}};
|
use crate::{
|
||||||
|
index::{Settings, Unchecked},
|
||||||
|
Update,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub enum UpdateResult {
|
pub enum UpdateResult {
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::path::{Path, PathBuf};
|
use std::fs::{create_dir_all, File};
|
||||||
use std::io::{BufReader, Write};
|
use std::io::{BufReader, Write};
|
||||||
use std::fs::{File, create_dir_all};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use heed::{EnvOpenOptions, RoTxn};
|
use heed::{EnvOpenOptions, RoTxn};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
@ -11,7 +11,14 @@ use tempfile::{NamedTempFile, TempDir};
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::{Result, State, UpdateStore};
|
use super::{Result, State, UpdateStore};
|
||||||
use crate::{Update, index::Index, index_controller::{update_file_store::UpdateFileStore, updates::status::{Enqueued, UpdateStatus}}};
|
use crate::{
|
||||||
|
index::Index,
|
||||||
|
index_controller::{
|
||||||
|
update_file_store::UpdateFileStore,
|
||||||
|
updates::status::{Enqueued, UpdateStatus},
|
||||||
|
},
|
||||||
|
Update,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize)]
|
#[derive(Serialize, Deserialize)]
|
||||||
struct UpdateEntry {
|
struct UpdateEntry {
|
||||||
@ -20,11 +27,7 @@ struct UpdateEntry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl UpdateStore {
|
impl UpdateStore {
|
||||||
pub fn dump(
|
pub fn dump(&self, indexes: &[Index], path: PathBuf) -> Result<()> {
|
||||||
&self,
|
|
||||||
indexes: &[Index],
|
|
||||||
path: PathBuf,
|
|
||||||
) -> Result<()> {
|
|
||||||
let state_lock = self.state.write();
|
let state_lock = self.state.write();
|
||||||
state_lock.swap(State::Dumping);
|
state_lock.swap(State::Dumping);
|
||||||
|
|
||||||
@ -35,7 +38,10 @@ impl UpdateStore {
|
|||||||
|
|
||||||
self.dump_updates(&txn, &uuids, &path)?;
|
self.dump_updates(&txn, &uuids, &path)?;
|
||||||
|
|
||||||
indexes.par_iter().try_for_each(|index| index.dump(&path)).unwrap();
|
indexes
|
||||||
|
.par_iter()
|
||||||
|
.try_for_each(|index| index.dump(&path))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -74,11 +80,13 @@ impl UpdateStore {
|
|||||||
let update = data.decode()?;
|
let update = data.decode()?;
|
||||||
|
|
||||||
if let Enqueued {
|
if let Enqueued {
|
||||||
meta: Update::DocumentAddition {
|
meta: Update::DocumentAddition { content_uuid, .. },
|
||||||
content_uuid, ..
|
..
|
||||||
}, ..
|
} = update
|
||||||
} = update {
|
{
|
||||||
self.update_file_store.dump(content_uuid, &dst_path).unwrap();
|
self.update_file_store
|
||||||
|
.dump(content_uuid, &dst_path)
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let update_json = UpdateEntry {
|
let update_json = UpdateEntry {
|
||||||
@ -122,7 +130,6 @@ impl UpdateStore {
|
|||||||
dst: impl AsRef<Path>,
|
dst: impl AsRef<Path>,
|
||||||
db_size: usize,
|
db_size: usize,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
|
|
||||||
println!("target path: {}", dst.as_ref().display());
|
println!("target path: {}", dst.as_ref().display());
|
||||||
|
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
|
@ -17,25 +17,26 @@ use heed::zerocopy::U64;
|
|||||||
use heed::{CompactionOption, Database, Env, EnvOpenOptions};
|
use heed::{CompactionOption, Database, Env, EnvOpenOptions};
|
||||||
use log::error;
|
use log::error;
|
||||||
use parking_lot::{Mutex, MutexGuard};
|
use parking_lot::{Mutex, MutexGuard};
|
||||||
|
use rayon::prelude::*;
|
||||||
use tokio::runtime::Handle;
|
use tokio::runtime::Handle;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
use tokio::sync::mpsc::error::TrySendError;
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
use rayon::prelude::*;
|
|
||||||
|
|
||||||
use codec::*;
|
use codec::*;
|
||||||
|
|
||||||
use super::error::Result;
|
use super::error::Result;
|
||||||
use super::status::{Enqueued, Processing};
|
use super::status::{Enqueued, Processing};
|
||||||
use crate::EnvSizer;
|
use crate::index::Index;
|
||||||
use crate::index_controller::update_files_path;
|
use crate::index_controller::update_files_path;
|
||||||
use crate::index_controller::updates::*;
|
use crate::index_controller::updates::*;
|
||||||
use crate::index::Index;
|
use crate::EnvSizer;
|
||||||
|
|
||||||
#[allow(clippy::upper_case_acronyms)]
|
#[allow(clippy::upper_case_acronyms)]
|
||||||
type BEU64 = U64<heed::byteorder::BE>;
|
type BEU64 = U64<heed::byteorder::BE>;
|
||||||
|
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub enum Update {
|
pub enum Update {
|
||||||
DeleteDocuments(Vec<String>),
|
DeleteDocuments(Vec<String>),
|
||||||
@ -164,7 +165,8 @@ impl UpdateStore {
|
|||||||
must_exit: Arc<AtomicBool>,
|
must_exit: Arc<AtomicBool>,
|
||||||
update_file_store: UpdateFileStore,
|
update_file_store: UpdateFileStore,
|
||||||
) -> anyhow::Result<Arc<Self>> {
|
) -> anyhow::Result<Arc<Self>> {
|
||||||
let (update_store, mut notification_receiver) = Self::new(options, path, update_file_store)?;
|
let (update_store, mut notification_receiver) =
|
||||||
|
Self::new(options, path, update_file_store)?;
|
||||||
let update_store = Arc::new(update_store);
|
let update_store = Arc::new(update_store);
|
||||||
|
|
||||||
// Send a first notification to trigger the process.
|
// Send a first notification to trigger the process.
|
||||||
@ -250,11 +252,7 @@ impl UpdateStore {
|
|||||||
|
|
||||||
/// Registers the update content in the pending store and the meta
|
/// Registers the update content in the pending store and the meta
|
||||||
/// into the pending-meta store. Returns the new unique update id.
|
/// into the pending-meta store. Returns the new unique update id.
|
||||||
pub fn register_update(
|
pub fn register_update(&self, index_uuid: Uuid, update: Update) -> heed::Result<Enqueued> {
|
||||||
&self,
|
|
||||||
index_uuid: Uuid,
|
|
||||||
update: Update,
|
|
||||||
) -> heed::Result<Enqueued> {
|
|
||||||
let mut txn = self.env.write_txn()?;
|
let mut txn = self.env.write_txn()?;
|
||||||
let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?;
|
let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?;
|
||||||
let meta = Enqueued::new(update, update_id);
|
let meta = Enqueued::new(update, update_id);
|
||||||
@ -299,7 +297,10 @@ impl UpdateStore {
|
|||||||
/// Executes the user provided function on the next pending update (the one with the lowest id).
|
/// Executes the user provided function on the next pending update (the one with the lowest id).
|
||||||
/// This is asynchronous as it let the user process the update with a read-only txn and
|
/// This is asynchronous as it let the user process the update with a read-only txn and
|
||||||
/// only writing the result meta to the processed-meta store *after* it has been processed.
|
/// only writing the result meta to the processed-meta store *after* it has been processed.
|
||||||
fn process_pending_update(&self, index_resolver: Arc<HardStateIndexResolver>) -> Result<Option<()>> {
|
fn process_pending_update(
|
||||||
|
&self,
|
||||||
|
index_resolver: Arc<HardStateIndexResolver>,
|
||||||
|
) -> Result<Option<()>> {
|
||||||
// Create a read transaction to be able to retrieve the pending update in order.
|
// Create a read transaction to be able to retrieve the pending update in order.
|
||||||
let rtxn = self.env.read_txn()?;
|
let rtxn = self.env.read_txn()?;
|
||||||
let first_meta = self.pending_queue.first(&rtxn)?;
|
let first_meta = self.pending_queue.first(&rtxn)?;
|
||||||
@ -315,8 +316,7 @@ impl UpdateStore {
|
|||||||
let state = self.state.write();
|
let state = self.state.write();
|
||||||
state.swap(State::Processing(index_uuid, processing.clone()));
|
state.swap(State::Processing(index_uuid, processing.clone()));
|
||||||
|
|
||||||
let result =
|
let result = self.perform_update(processing, index_resolver, index_uuid, global_id);
|
||||||
self.perform_update(processing, index_resolver, index_uuid, global_id);
|
|
||||||
|
|
||||||
state.swap(State::Idle);
|
state.swap(State::Idle);
|
||||||
|
|
||||||
@ -495,15 +495,10 @@ impl UpdateStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn snapshot(
|
pub fn snapshot(&self, indexes: Vec<Index>, path: impl AsRef<Path>) -> Result<()> {
|
||||||
&self,
|
|
||||||
indexes: Vec<Index>,
|
|
||||||
path: impl AsRef<Path>,
|
|
||||||
) -> Result<()> {
|
|
||||||
let state_lock = self.state.write();
|
let state_lock = self.state.write();
|
||||||
state_lock.swap(State::Snapshoting);
|
state_lock.swap(State::Snapshoting);
|
||||||
|
|
||||||
|
|
||||||
let txn = self.env.write_txn()?;
|
let txn = self.env.write_txn()?;
|
||||||
|
|
||||||
let update_path = path.as_ref().join("updates");
|
let update_path = path.as_ref().join("updates");
|
||||||
@ -523,19 +518,22 @@ impl UpdateStore {
|
|||||||
let ((_, uuid, _), pending) = entry?;
|
let ((_, uuid, _), pending) = entry?;
|
||||||
if uuids.contains(&uuid) {
|
if uuids.contains(&uuid) {
|
||||||
if let Enqueued {
|
if let Enqueued {
|
||||||
meta: Update::DocumentAddition {
|
meta: Update::DocumentAddition { content_uuid, .. },
|
||||||
content_uuid, ..
|
|
||||||
},
|
|
||||||
..
|
..
|
||||||
} = pending.decode()?
|
} = pending.decode()?
|
||||||
{
|
{
|
||||||
self.update_file_store.snapshot(content_uuid, &path).unwrap();
|
self.update_file_store
|
||||||
|
.snapshot(content_uuid, &path)
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let path = path.as_ref().to_owned();
|
let path = path.as_ref().to_owned();
|
||||||
indexes.par_iter().try_for_each(|index| index.snapshot(path.clone())).unwrap();
|
indexes
|
||||||
|
.par_iter()
|
||||||
|
.try_for_each(|index| index.snapshot(path.clone()))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -546,10 +544,7 @@ impl UpdateStore {
|
|||||||
for entry in self.pending_queue.iter(&txn)? {
|
for entry in self.pending_queue.iter(&txn)? {
|
||||||
let (_, pending) = entry?;
|
let (_, pending) = entry?;
|
||||||
if let Enqueued {
|
if let Enqueued {
|
||||||
meta: store::Update::DocumentAddition {
|
meta: store::Update::DocumentAddition { content_uuid, .. },
|
||||||
content_uuid,
|
|
||||||
..
|
|
||||||
},
|
|
||||||
..
|
..
|
||||||
} = pending
|
} = pending
|
||||||
{
|
{
|
||||||
|
@ -5,7 +5,7 @@ pub mod options;
|
|||||||
pub mod index;
|
pub mod index;
|
||||||
pub mod index_controller;
|
pub mod index_controller;
|
||||||
|
|
||||||
pub use index_controller::{IndexController as MeiliSearch, updates::store::Update};
|
pub use index_controller::{updates::store::Update, IndexController as MeiliSearch};
|
||||||
|
|
||||||
pub use milli;
|
pub use milli;
|
||||||
|
|
||||||
@ -55,4 +55,3 @@ pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Res
|
|||||||
ar.unpack(&dest)?;
|
ar.unpack(&dest)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -112,4 +112,3 @@ fn total_memory_bytes() -> Option<u64> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user