snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 12:05:05 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

refactor update actor

Browse Source
This commit is contained in:
mpostma 2021-09-22 11:52:29 +02:00
parent def737edee
commit 12542bf922
25 changed files with 253 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
meilisearch-lib/src/index/update_handler.rs
View File

@ -3,8 +3,8 @@ use milli::update::UpdateBuilder;
use milli::CompressionType;
use rayon::ThreadPool;
use crate::index_controller::update_actor::RegisterUpdate;
use crate::index_controller::{Failed, Processed, Processing};
use crate::index_controller::updates::RegisterUpdate;
use crate::index_controller::updates::status::{Failed, Processed, Processing};
use crate::options::IndexerOpts;
pub struct UpdateHandler {

2
meilisearch-lib/src/index/updates.rs
View File

@ -8,7 +8,7 @@ use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder};
use serde::{Deserialize, Serialize, Serializer};
use uuid::Uuid;
use crate::index_controller::UpdateResult;
use crate::index_controller::updates::status::UpdateResult;
use super::Index;
use super::error::Result;

14
meilisearch-lib/src/index_controller/dump_actor/actor.rs
View File

@ -7,19 +7,18 @@ use chrono::Utc;
use futures::{lock::Mutex, stream::StreamExt};
use log::{error, trace};
use tokio::sync::{mpsc, oneshot, RwLock};
use update_actor::UpdateActorHandle;
use super::error::{DumpActorError, Result};
use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask};
use crate::index_controller::uuid_resolver::UuidResolverSender;
use crate::index_controller::update_actor;
use crate::index_controller::updates::UpdateSender;
pub const CONCURRENT_DUMP_MSG: usize = 10;
pub struct DumpActor<Update> {
pub struct DumpActor {
inbox: Option<mpsc::Receiver<DumpMsg>>,
uuid_resolver: UuidResolverSender,
update: Update,
update: UpdateSender,
dump_path: PathBuf,
lock: Arc<Mutex<()>>,
dump_infos: Arc<RwLock<HashMap<String, DumpInfo>>>,
@ -32,14 +31,11 @@ fn generate_uid() -> String {
Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
}
impl<Update> DumpActor<Update>
where
Update: UpdateActorHandle + Send + Sync + Clone + 'static,
{
impl DumpActor {
pub fn new(
inbox: mpsc::Receiver<DumpMsg>,
uuid_resolver: UuidResolverSender,
update: Update,
update: UpdateSender,
dump_path: impl AsRef<Path>,
index_db_size: usize,
update_db_size: usize,

2
meilisearch-lib/src/index_controller/dump_actor/error.rs
View File

@ -1,6 +1,6 @@
use meilisearch_error::{Code, ErrorCode};
use crate::index_controller::update_actor::error::UpdateActorError;
use crate::index_controller::updates::error::UpdateActorError;
use crate::index_controller::uuid_resolver::error::UuidResolverError;
pub type Result<T> = std::result::Result<T, DumpActorError>;

2
meilisearch-lib/src/index_controller/dump_actor/handle_impl.rs
View File

@ -33,7 +33,7 @@ impl DumpActorHandleImpl {
pub fn new(
path: impl AsRef<Path>,
uuid_resolver: UuidResolverSender,
update: crate::index_controller::update_actor::UpdateActorHandleImpl,
update: crate::index_controller::updates::UpdateSender,
index_db_size: usize,
update_db_size: usize,
) -> anyhow::Result<Self> {

3
meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs
View File

@ -5,7 +5,8 @@ use log::info;
use serde::{Deserialize, Serialize};
use crate::index::Index;
use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::store::HeedUuidStore};
use crate::index_controller::updates::store::UpdateStore;
use crate::index_controller::{uuid_resolver::store::HeedUuidStore};
use crate::options::IndexerOpts;
#[derive(Serialize, Deserialize, Debug)]

17
meilisearch-lib/src/index_controller/dump_actor/mod.rs
View File

@ -16,9 +16,10 @@ pub use actor::DumpActor;
pub use handle_impl::*;
pub use message::DumpMsg;
use super::update_actor::UpdateActorHandle;
use super::updates::UpdateSender;
use super::uuid_resolver::UuidResolverSender;
use crate::index_controller::dump_actor::error::DumpActorError;
use crate::index_controller::updates::UpdateMsg;
use crate::index_controller::uuid_resolver::UuidResolverMsg;
use crate::options::IndexerOpts;
use error::Result;
@ -151,20 +152,16 @@ pub fn load_dump(
Ok(())
}
struct DumpTask<P> {
struct DumpTask {
path: PathBuf,
uuid_resolver: UuidResolverSender,
update_handle: P,
update_handle: UpdateSender,
uid: String,
update_db_size: usize,
index_db_size: usize,
}
impl<P> DumpTask<P>
where
P: UpdateActorHandle + Send + Sync + Clone + 'static,
{
impl DumpTask {
async fn run(self) -> Result<()> {
trace!("Performing dump.");
@ -182,9 +179,7 @@ where
let uuids = UuidResolverMsg::dump(&self.uuid_resolver, temp_dump_path.clone()).await?;
self.update_handle
.dump(uuids, temp_dump_path.clone())
.await?;
UpdateMsg::dump(&self.update_handle, uuids, temp_dump_path.clone()).await?;
let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;

2
meilisearch-lib/src/index_controller/error.rs
View File

@ -7,7 +7,7 @@ use crate::index::error::IndexError;
use super::dump_actor::error::DumpActorError;
use super::index_actor::error::IndexActorError;
use super::update_actor::error::UpdateActorError;
use super::updates::error::UpdateActorError;
use super::uuid_resolver::error::UuidResolverError;
pub type Result<T> = std::result::Result<T, IndexControllerError>;

3
meilisearch-lib/src/index_controller/index_actor/actor.rs
View File

@ -14,8 +14,9 @@ use crate::index::{
update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings,
};
use crate::index_controller::{
get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing,
get_arc_ownership_blocking, IndexStats,
};
use crate::index_controller::updates::status::{Failed, Processed, Processing};
use crate::options::IndexerOpts;
use super::error::{IndexActorError, Result};

5
meilisearch-lib/src/index_controller/index_actor/handle_impl.rs
View File

@ -1,4 +1,4 @@
use crate::options::IndexerOpts;
use crate::{index_controller::updates::status::{Failed, Processed, Processing}, options::IndexerOpts};
use std::path::{Path, PathBuf};
use tokio::sync::{mpsc, oneshot};
@ -6,11 +6,10 @@ use uuid::Uuid;
use crate::{
index::Checked,
index_controller::{IndexSettings, IndexStats, Processing},
index_controller::{IndexSettings, IndexStats},
};
use crate::{
index::{Document, SearchQuery, SearchResult, Settings},
index_controller::{Failed, Processed},
};
use super::error::Result;

3
meilisearch-lib/src/index_controller/index_actor/message.rs
View File

@ -5,7 +5,8 @@ use uuid::Uuid;
use super::error::Result as IndexResult;
use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
use crate::index_controller::{Failed, IndexStats, Processed, Processing};
use crate::index_controller::IndexStats;
use crate::index_controller::updates::status::{Failed, Processed, Processing};
use super::{IndexMeta, IndexSettings};

3
meilisearch-lib/src/index_controller/index_actor/mod.rs
View File

@ -13,10 +13,9 @@ use message::IndexMsg;
use store::{IndexStore, MapIndexStore};
use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings};
use crate::index_controller::{Failed, IndexStats, Processed, Processing};
use error::Result;
use super::IndexSettings;
use super::{IndexSettings, IndexStats, updates::status::{Failed, Processed, Processing}};
mod actor;
pub mod error;

27
meilisearch-lib/src/index_controller/mod.rs
View File

@ -18,8 +18,6 @@ use dump_actor::DumpActorHandle;
pub use dump_actor::{DumpInfo, DumpStatus};
use index_actor::IndexActorHandle;
use snapshot::load_snapshot;
use update_actor::UpdateActorHandle;
pub use updates::*;
use uuid_resolver::error::UuidResolverError;
use crate::options::IndexerOpts;
@ -27,14 +25,15 @@ use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
use error::Result;
use self::dump_actor::load_dump;
use self::updates::UpdateMsg;
use self::updates::status::UpdateStatus;
use self::uuid_resolver::UuidResolverMsg;
mod dump_actor;
pub mod error;
pub mod index_actor;
mod snapshot;
pub mod update_actor;
mod updates;
pub mod updates;
mod uuid_resolver;
pub mod update_file_store;
@ -74,7 +73,7 @@ pub struct IndexStats {
pub struct IndexController {
uuid_resolver: uuid_resolver::UuidResolverSender,
index_handle: index_actor::IndexActorHandleImpl,
update_handle: update_actor::UpdateActorHandleImpl,
update_handle: updates::UpdateSender,
dump_handle: dump_actor::DumpActorHandleImpl,
}
@ -140,8 +139,10 @@ impl IndexControllerBuilder {
let uuid_resolver = uuid_resolver::create_uuid_resolver(&db_path)?;
let index_handle =
index_actor::IndexActorHandleImpl::new(&db_path, index_size, &indexer_options)?;
let update_handle = update_actor::UpdateActorHandleImpl::new(
index_handle.clone(),
#[allow(unreachable_code)]
let update_handle = updates::create_update_handler(
todo!(),
&db_path,
update_store_size,
)?;
@ -235,12 +236,12 @@ impl IndexController {
let uuid = UuidResolverMsg::get(&self.uuid_resolver, uid.to_string()).await;
match uuid {
Ok(uuid) => {
let update_result = self.update_handle.update(uuid, update).await?;
let update_result = UpdateMsg::update(&self.update_handle, uuid, update).await?;
Ok(update_result)
},
Err(UuidResolverError::UnexistingIndex(name)) => {
let uuid = Uuid::new_v4();
let update_result = self.update_handle.update(uuid, update).await?;
let update_result = UpdateMsg::update(&self.update_handle, uuid, update).await?;
// ignore if index creation fails now, since it may already have been created
let _ = self.index_handle.create_index(uuid, None).await;
UuidResolverMsg::insert(&self.uuid_resolver, uuid, name).await?;
@ -378,13 +379,13 @@ impl IndexController {
pub async fn update_status(&self, uid: String, id: u64) -> Result<UpdateStatus> {
let uuid = UuidResolverMsg::get(&self.uuid_resolver, uid).await?;
let result = self.update_handle.update_status(uuid, id).await?;
let result = UpdateMsg::get_update(&self.update_handle, uuid, id).await?;
Ok(result)
}
pub async fn all_update_status(&self, uid: String) -> Result<Vec<UpdateStatus>> {
let uuid = UuidResolverMsg::get(&self.uuid_resolver, uid).await?;
let result = self.update_handle.get_all_updates_status(uuid).await?;
let result = UpdateMsg::list_updates(&self.update_handle, uuid).await?;
Ok(result)
}
@ -485,7 +486,7 @@ impl IndexController {
pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
let uuid = UuidResolverMsg::get(&self.uuid_resolver, uid).await?;
let update_infos = self.update_handle.get_info().await?;
let update_infos = UpdateMsg::get_info(&self.update_handle).await?;
let mut stats = self.index_handle.get_index_stats(uuid).await?;
// Check if the currently indexing update is from out index.
stats.is_indexing = Some(Some(uuid) == update_infos.processing);
@ -493,7 +494,7 @@ impl IndexController {
}
pub async fn get_all_stats(&self) -> Result<Stats> {
let update_infos = self.update_handle.get_info().await?;
let update_infos = UpdateMsg::get_info(&self.update_handle).await?;
let mut database_size = self.get_uuids_size().await? + update_infos.size;
let mut last_update: Option<DateTime<_>> = None;
let mut indexes = BTreeMap::new();

2
meilisearch-lib/src/index_controller/snapshot.rs
View File

@ -132,7 +132,7 @@ mod test {
use super::*;
use crate::index_controller::index_actor::MockIndexActorHandle;
use crate::index_controller::update_actor::{
use crate::index_controller::updates::{
error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl,
};
use crate::index_controller::uuid_resolver::{

94
meilisearch-lib/src/index_controller/update_actor/handle_impl.rs
View File

@ -1,94 +0,0 @@
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
use crate::index_controller::{IndexActorHandle, Update, UpdateStatus};
use super::error::Result;
use super::{UpdateActor, UpdateActorHandle, UpdateMsg, UpdateStoreInfo};
#[derive(Clone)]
pub struct UpdateActorHandleImpl {
sender: mpsc::Sender<UpdateMsg>,
}
impl UpdateActorHandleImpl {
pub fn new<I>(
index_handle: I,
path: impl AsRef<Path>,
update_store_size: usize,
) -> anyhow::Result<Self>
where
I: IndexActorHandle + Clone + Sync + Send +'static,
{
let path = path.as_ref().to_owned();
let (sender, receiver) = mpsc::channel(100);
let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?;
tokio::task::spawn_local(actor.run());
Ok(Self { sender })
}
}
#[async_trait::async_trait]
impl UpdateActorHandle for UpdateActorHandleImpl {
async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::ListUpdates { uuid, ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::GetUpdate { uuid, id, ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn delete(&self, uuid: Uuid) -> Result<()> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::Delete { uuid, ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::Snapshot { uuids, path, ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::Dump { uuids, path, ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn get_info(&self) -> Result<UpdateStoreInfo> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::GetInfo { ret };
self.sender.send(msg).await?;
receiver.await?
}
async fn update(
&self,
uuid: Uuid,
update: Update,
) -> Result<UpdateStatus> {
let (ret, receiver) = oneshot::channel();
let msg = UpdateMsg::Update {
uuid,
update,
ret,
};
self.sender.send(msg).await?;
receiver.await?
}
}

42
meilisearch-lib/src/index_controller/update_actor/message.rs
View File

@ -1,42 +0,0 @@
use std::collections::HashSet;
use std::path::PathBuf;
use tokio::sync::oneshot;
use uuid::Uuid;
use super::error::Result;
use super::{UpdateStatus, UpdateStoreInfo, Update};
pub enum UpdateMsg {
Update {
uuid: Uuid,
update: Update,
ret: oneshot::Sender<Result<UpdateStatus>>,
},
ListUpdates {
uuid: Uuid,
ret: oneshot::Sender<Result<Vec<UpdateStatus>>>,
},
GetUpdate {
uuid: Uuid,
ret: oneshot::Sender<Result<UpdateStatus>>,
id: u64,
},
Delete {
uuid: Uuid,
ret: oneshot::Sender<Result<()>>,
},
Snapshot {
uuids: HashSet<Uuid>,
path: PathBuf,
ret: oneshot::Sender<Result<()>>,
},
Dump {
uuids: HashSet<Uuid>,
path: PathBuf,
ret: oneshot::Sender<Result<()>>,
},
GetInfo {
ret: oneshot::Sender<Result<UpdateStoreInfo>>,
},
}

49
meilisearch-lib/src/index_controller/update_actor/mod.rs
View File

@ -1,49 +0,0 @@
use std::{collections::HashSet, path::PathBuf};
use milli::update::IndexDocumentsMethod;
use uuid::Uuid;
use serde::{Serialize, Deserialize};
use crate::index_controller::UpdateStatus;
use super::Update;
use actor::UpdateActor;
use error::Result;
use message::UpdateMsg;
pub use handle_impl::UpdateActorHandleImpl;
pub use store::{UpdateStore, UpdateStoreInfo};
mod actor;
pub mod error;
mod handle_impl;
mod message;
pub mod store;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RegisterUpdate {
DocumentAddition {
primary_key: Option<String>,
method: IndexDocumentsMethod,
content_uuid: Uuid,
}
}
#[cfg(test)]
use mockall::automock;
#[async_trait::async_trait]
pub trait UpdateActorHandle {
async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>>;
async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
async fn delete(&self, uuid: Uuid) -> Result<()>;
async fn snapshot(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()>;
async fn get_info(&self) -> Result<UpdateStoreInfo>;
async fn update(
&self,
uuid: Uuid,
update: Update,
) -> Result<UpdateStatus>;
}

0
meilisearch-lib/src/index_controller/update_actor/error.rs → meilisearch-lib/src/index_controller/updates/error.rs
View File

112
meilisearch-lib/src/index_controller/updates/message.rs Normal file
View File

@ -0,0 +1,112 @@
use std::collections::HashSet;
use std::path::PathBuf;
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
use super::error::Result;
use super::{Update, UpdateStatus, UpdateStoreInfo};
pub enum UpdateMsg {
Update {
uuid: Uuid,
update: Update,
ret: oneshot::Sender<Result<UpdateStatus>>,
},
ListUpdates {
uuid: Uuid,
ret: oneshot::Sender<Result<Vec<UpdateStatus>>>,
},
GetUpdate {
uuid: Uuid,
ret: oneshot::Sender<Result<UpdateStatus>>,
id: u64,
},
Delete {
uuid: Uuid,
ret: oneshot::Sender<Result<()>>,
},
Snapshot {
uuids: HashSet<Uuid>,
path: PathBuf,
ret: oneshot::Sender<Result<()>>,
},
Dump {
uuids: HashSet<Uuid>,
path: PathBuf,
ret: oneshot::Sender<Result<()>>,
},
GetInfo {
ret: oneshot::Sender<Result<UpdateStoreInfo>>,
},
}
impl UpdateMsg {
pub async fn dump(
sender: &mpsc::Sender<Self>,
uuids: HashSet<Uuid>,
path: PathBuf,
) -> Result<()> {
let (ret, rcv) = oneshot::channel();
let msg = Self::Dump {
path,
uuids,
ret,
};
sender.send(msg).await?;
rcv.await?
}
pub async fn update(
sender: &mpsc::Sender<Self>,
uuid: Uuid,
update: Update,
) -> Result<UpdateStatus> {
let (ret, rcv) = oneshot::channel();
let msg = Self::Update {
uuid,
update,
ret,
};
sender.send(msg).await?;
rcv.await?
}
pub async fn get_update(
sender: &mpsc::Sender<Self>,
uuid: Uuid,
id: u64,
) -> Result<UpdateStatus> {
let (ret, rcv) = oneshot::channel();
let msg = Self::GetUpdate {
uuid,
id,
ret,
};
sender.send(msg).await?;
rcv.await?
}
pub async fn list_updates(
sender: &mpsc::Sender<Self>,
uuid: Uuid,
) -> Result<Vec<UpdateStatus>> {
let (ret, rcv) = oneshot::channel();
let msg = Self::ListUpdates {
uuid,
ret,
};
sender.send(msg).await?;
rcv.await?
}
pub async fn get_info(
sender: &mpsc::Sender<Self>,
) -> Result<UpdateStoreInfo> {
let (ret, rcv) = oneshot::channel();
let msg = Self::GetInfo {
ret,
};
sender.send(msg).await?;
rcv.await?
}
}

121
meilisearch-lib/src/index_controller/update_actor/actor.rs → meilisearch-lib/src/index_controller/updates/mod.rs
View File

@ -1,3 +1,8 @@
pub mod error;
mod message;
pub mod status;
pub mod store;
use std::collections::HashSet;
use std::io;
use std::path::{Path, PathBuf};
@ -10,25 +15,47 @@ use bytes::Bytes;
use futures::{Stream, StreamExt};
use log::trace;
use milli::documents::DocumentBatchBuilder;
use milli::update::IndexDocumentsMethod;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use tokio::sync::mpsc;
use uuid::Uuid;
use super::error::{Result, UpdateActorError};
use super::RegisterUpdate;
use super::{UpdateMsg, UpdateStore, UpdateStoreInfo, Update};
use crate::index_controller::index_actor::IndexActorHandle;
use self::error::{Result, UpdateActorError};
pub use self::message::UpdateMsg;
use self::store::{UpdateStore, UpdateStoreInfo};
use crate::index_controller::update_file_store::UpdateFileStore;
use crate::index_controller::{DocumentAdditionFormat, Payload, UpdateStatus};
use status::UpdateStatus;
pub struct UpdateActor<I> {
store: Arc<UpdateStore>,
inbox: Option<mpsc::Receiver<UpdateMsg>>,
update_file_store: UpdateFileStore,
index_handle: I,
must_exit: Arc<AtomicBool>,
use super::{DocumentAdditionFormat, Payload, Update};
pub type UpdateSender = mpsc::Sender<UpdateMsg>;
type IndexSender = mpsc::Sender<()>;
pub fn create_update_handler(
index_sender: IndexSender,
db_path: impl AsRef<Path>,
update_store_size: usize,
) -> anyhow::Result<UpdateSender> {
let path = db_path.as_ref().to_owned();
let (sender, receiver) = mpsc::channel(100);
let actor = UpdateHandler::new(update_store_size, receiver, path, index_sender)?;
tokio::task::spawn_local(actor.run());
Ok(sender)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RegisterUpdate {
DocumentAddition {
primary_key: Option<String>,
method: IndexDocumentsMethod,
content_uuid: Uuid,
},
}
/// A wrapper type to implement read on a `Stream<Result<Bytes, Error>>`.
struct StreamReader<S> {
stream: S,
current: Option<Bytes>,
@ -36,13 +63,18 @@ struct StreamReader<S> {
impl<S> StreamReader<S> {
fn new(stream: S) -> Self {
Self { stream, current: None }
Self {
stream,
current: None,
}
}
}
impl<S: Stream<Item = std::result::Result<Bytes, PayloadError>> + Unpin> io::Read for StreamReader<S> {
impl<S: Stream<Item = std::result::Result<Bytes, PayloadError>> + Unpin> io::Read
for StreamReader<S>
{
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
// TODO: optimize buf filling
match self.current.take() {
Some(mut bytes) => {
let copied = bytes.split_to(buf.len());
@ -52,29 +84,32 @@ impl<S: Stream<Item = std::result::Result<Bytes, PayloadError>> + Unpin> io::Rea
}
Ok(copied.len())
}
None => {
match tokio::runtime::Handle::current().block_on(self.stream.next()) {
None => match tokio::runtime::Handle::current().block_on(self.stream.next()) {
Some(Ok(bytes)) => {
self.current.replace(bytes);
self.read(buf)
},
}
Some(Err(e)) => Err(io::Error::new(io::ErrorKind::BrokenPipe, e)),
None => return Ok(0),
}
}
},
}
}
}
impl<I> UpdateActor<I>
where
I: IndexActorHandle + Clone + Sync + Send + 'static,
{
pub struct UpdateHandler {
store: Arc<UpdateStore>,
inbox: Option<mpsc::Receiver<UpdateMsg>>,
update_file_store: UpdateFileStore,
index_handle: IndexSender,
must_exit: Arc<AtomicBool>,
}
impl UpdateHandler {
pub fn new(
update_db_size: usize,
inbox: mpsc::Receiver<UpdateMsg>,
path: impl AsRef<Path>,
index_handle: I,
index_handle: IndexSender,
) -> anyhow::Result<Self> {
let path = path.as_ref().to_owned();
std::fs::create_dir_all(&path)?;
@ -95,7 +130,7 @@ where
inbox,
index_handle,
must_exit,
update_file_store
update_file_store,
})
}
@ -128,11 +163,7 @@ where
stream
.for_each_concurrent(Some(10), |msg| async {
match msg {
Update {
uuid,
update,
ret,
} => {
Update { uuid, update, ret } => {
let _ = ret.send(self.handle_update(uuid, update).await);
}
ListUpdates { uuid, ret } => {
@ -158,23 +189,30 @@ where
.await;
}
async fn handle_update(
&self,
index_uuid: Uuid,
update: Update,
) -> Result<UpdateStatus> {
async fn handle_update(&self, index_uuid: Uuid, update: Update) -> Result<UpdateStatus> {
let registration = match update {
Update::DocumentAddition { payload, primary_key, method, format } => {
Update::DocumentAddition {
payload,
primary_key,
method,
format,
} => {
let content_uuid = match format {
DocumentAdditionFormat::Json => self.documents_from_json(payload).await?,
};
RegisterUpdate::DocumentAddition { primary_key, method, content_uuid }
RegisterUpdate::DocumentAddition {
primary_key,
method,
content_uuid,
}
}
};
let store = self.store.clone();
let status = tokio::task::spawn_blocking(move || store.register_update(index_uuid, registration)).await??;
let status =
tokio::task::spawn_blocking(move || store.register_update(index_uuid, registration))
.await??;
Ok(status.into())
}
@ -185,14 +223,16 @@ where
let (uuid, mut file) = file_store.new_update().unwrap();
let mut builder = DocumentBatchBuilder::new(&mut *file).unwrap();
let documents: Vec<Map<String, Value>> = serde_json::from_reader(StreamReader::new(payload))?;
let documents: Vec<Map<String, Value>> =
serde_json::from_reader(StreamReader::new(payload))?;
builder.add_documents(documents).unwrap();
builder.finish().unwrap();
file.persist();
Ok(uuid)
}).await?
})
.await?
}
async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
@ -256,5 +296,4 @@ where
Ok(info)
}
}

4
meilisearch-lib/src/index_controller/updates.rs → meilisearch-lib/src/index_controller/updates/status.rs
View File

@ -6,9 +6,7 @@ use meilisearch_error::{Code, ErrorCode};
use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
use serde::{Deserialize, Serialize};
use crate::index::{Settings, Unchecked};
use super::update_actor::RegisterUpdate;
use crate::{RegisterUpdate, index::{Settings, Unchecked}};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum UpdateResult {

0
meilisearch-lib/src/index_controller/update_actor/store/codec.rs → meilisearch-lib/src/index_controller/updates/store/codec.rs
View File

12
meilisearch-lib/src/index_controller/update_actor/store/dump.rs → meilisearch-lib/src/index_controller/updates/store/dump.rs
View File

@ -10,10 +10,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::{Result, State, UpdateStore};
use crate::index_controller::{
index_actor::IndexActorHandle,
UpdateStatus,
};
use crate::index_controller::{updates::{IndexSender, status::UpdateStatus}};
#[derive(Serialize, Deserialize)]
struct UpdateEntry {
@ -26,7 +23,7 @@ impl UpdateStore {
&self,
uuids: &HashSet<Uuid>,
path: PathBuf,
handle: impl IndexActorHandle,
handle: IndexSender,
) -> Result<()> {
let state_lock = self.state.write();
state_lock.swap(State::Dumping);
@ -175,11 +172,12 @@ impl UpdateStore {
async fn dump_indexes(
uuids: &HashSet<Uuid>,
handle: impl IndexActorHandle,
handle: IndexSender,
path: impl AsRef<Path>,
) -> Result<()> {
for uuid in uuids {
handle.dump(*uuid, path.as_ref().to_owned()).await?;
//handle.dump(*uuid, path.as_ref().to_owned()).await?;
todo!()
}
Ok(())

15
meilisearch-lib/src/index_controller/update_actor/store/mod.rs → meilisearch-lib/src/index_controller/updates/store/mod.rs
View File

@ -28,9 +28,10 @@ use codec::*;
use super::RegisterUpdate;
use super::error::Result;
use super::status::{Enqueued, Processing};
use crate::EnvSizer;
use crate::index_controller::update_files_path;
use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*};
#[allow(clippy::upper_case_acronyms)]
type BEU64 = U64<heed::byteorder::BE>;
@ -145,7 +146,7 @@ impl UpdateStore {
pub fn open(
options: EnvOpenOptions,
path: impl AsRef<Path>,
index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static,
index_handle: IndexSender,
must_exit: Arc<AtomicBool>,
) -> anyhow::Result<Arc<Self>> {
let (update_store, mut notification_receiver) = Self::new(options, path)?;
@ -283,7 +284,7 @@ impl UpdateStore {
/// Executes the user provided function on the next pending update (the one with the lowest id).
/// This is asynchronous as it let the user process the update with a read-only txn and
/// only writing the result meta to the processed-meta store *after* it has been processed.
fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result<Option<()>> {
fn process_pending_update(&self, index_handle: IndexSender) -> Result<Option<()>> {
// Create a read transaction to be able to retrieve the pending update in order.
let rtxn = self.env.read_txn()?;
let first_meta = self.pending_queue.first(&rtxn)?;
@ -313,7 +314,7 @@ impl UpdateStore {
fn perform_update(
&self,
processing: Processing,
index_handle: impl IndexActorHandle,
index_handle: IndexSender,
index_uuid: Uuid,
global_id: u64,
) -> Result<Option<()>> {
@ -321,7 +322,7 @@ impl UpdateStore {
let handle = Handle::current();
let update_id = processing.id();
let result =
match handle.block_on(index_handle.update(index_uuid, processing.clone())) {
match handle.block_on(/*index_handle.update(index_uuid, processing.clone())*/ todo!()) {
Ok(result) => result,
Err(e) => Err(processing.fail(e)),
};
@ -483,7 +484,7 @@ impl UpdateStore {
&self,
uuids: &HashSet<Uuid>,
path: impl AsRef<Path>,
handle: impl IndexActorHandle + Clone,
handle: IndexSender,
) -> Result<()> {
let state_lock = self.state.write();
state_lock.swap(State::Snapshoting);
@ -524,7 +525,7 @@ impl UpdateStore {
// Perform the snapshot of each index concurently. Only a third of the capabilities of
// the index actor at a time not to put too much pressure on the index actor
let mut stream = futures::stream::iter(uuids.iter())
.map(move |uuid| handle.snapshot(*uuid, path.clone()))
.map(move |uuid| todo!() /*handle.snapshot(*uuid, path.clone())*/)
.buffer_unordered(CONCURRENT_INDEX_MSG / 3);
Handle::current().block_on(async {

2
meilisearch-lib/src/lib.rs
View File

@ -5,7 +5,7 @@ pub mod options;
pub mod index;
pub mod index_controller;
pub use index_controller::{UpdateResult, UpdateStatus, IndexController as MeiliSearch, update_actor::RegisterUpdate};
pub use index_controller::{IndexController as MeiliSearch, updates::RegisterUpdate};
use walkdir::WalkDir;