Merge #2453

2453: test index resolver r=MarinPostma a=MarinPostma

add some tests to the `IndexResolver` implementation of `BatchHandler`


Co-authored-by: ad hoc <postma.marin@protonmail.com>

meilisearch-lib/src/dump/handler.rs

@@ -121,7 +121,6 @@ mod real {
 
 #[cfg(test)]
 mod test {
-    use std::marker::PhantomData;
     use std::path::PathBuf;
     use std::sync::Arc;
 
@@ -137,12 +136,12 @@ mod test {
 
     pub enum MockDumpHandler<U, I> {
         Real(super::real::DumpHandler<U, I>),
-        Mock(Mocker, PhantomData<(U, I)>),
+        Mock(Mocker),
     }
 
     impl<U, I> MockDumpHandler<U, I> {
         pub fn mock(mocker: Mocker) -> Self {
-            Self::Mock(mocker, PhantomData)
+            Self::Mock(mocker)
         }
     }
 
@@ -173,7 +172,7 @@ mod test {
         pub async fn run(&self, uid: String) -> Result<()> {
             match self {
                 DumpHandler::Real(real) => real.run(uid).await,
-                DumpHandler::Mock(mocker, _) => unsafe { mocker.get("run").call(uid) },
+                DumpHandler::Mock(mocker) => unsafe { mocker.get("run").call(uid) },
             }
         }
     }

meilisearch-lib/src/index_resolver/error.rs

@@ -5,7 +5,7 @@ use tokio::sync::mpsc::error::SendError as MpscSendError;
 use tokio::sync::oneshot::error::RecvError as OneshotRecvError;
 use uuid::Uuid;
 
-use crate::{error::MilliError, index::error::IndexError};
+use crate::{error::MilliError, index::error::IndexError, update_file_store::UpdateFileStoreError};
 
 pub type Result<T> = std::result::Result<T, IndexResolverError>;
 
@@ -49,7 +49,8 @@ internal_error!(
     uuid::Error,
     std::io::Error,
     tokio::task::JoinError,
-    serde_json::Error
+    serde_json::Error,
+    UpdateFileStoreError
 );
 
 impl ErrorCode for IndexResolverError {

meilisearch-lib/src/index_resolver/mod.rs

@@ -27,6 +27,12 @@ use self::meta_store::IndexMeta;
 
 pub type HardStateIndexResolver = IndexResolver<HeedMetaStore, MapIndexStore>;
 
+#[cfg(not(test))]
+pub use real::IndexResolver;
+
+#[cfg(test)]
+pub use test::MockIndexResolver as IndexResolver;
+
 /// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400
 /// bytes long
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
@@ -96,13 +102,16 @@ impl FromStr for IndexUid {
     }
 }
 
-pub struct IndexResolver<U, I> {
+mod real {
-    index_uuid_store: U,
+    use super::*;
-    index_store: I,
-    pub file_store: UpdateFileStore,
-}
 
-impl IndexResolver<HeedMetaStore, MapIndexStore> {
+    pub struct IndexResolver<U, I> {
+        pub(super) index_uuid_store: U,
+        pub(super) index_store: I,
+        pub(super) file_store: UpdateFileStore,
+    }
+
+    impl IndexResolver<HeedMetaStore, MapIndexStore> {
         pub fn load_dump(
             src: impl AsRef<Path>,
             dst: impl AsRef<Path>,
@@ -120,13 +129,13 @@ impl IndexResolver<HeedMetaStore, MapIndexStore> {
 
             Ok(())
         }
-}
+    }
 
-impl<U, I> IndexResolver<U, I>
+    impl<U, I> IndexResolver<U, I>
-where
+    where
         U: IndexMetaStore,
         I: IndexStore,
-{
+    {
         pub fn new(index_uuid_store: U, index_store: I, file_store: UpdateFileStore) -> Self {
             Self {
                 index_uuid_store,
@@ -225,9 +234,16 @@ where
             }
         }
 
+        pub async fn delete_content_file(&self, content_uuid: Uuid) -> Result<()> {
+            self.file_store.delete(content_uuid).await?;
+            Ok(())
+        }
+
         pub async fn process_task(&self, task: &Task) -> Result<TaskResult> {
             match &task.content {
-            TaskContent::DocumentAddition { .. } => panic!("updates should be handled by batch"),
+                TaskContent::DocumentAddition { .. } => {
+                    panic!("updates should be handled by batch")
+                }
                 TaskContent::DocumentDeletion {
                     deletion: DocumentDeletion::Ids(ids),
                     index_uid,
@@ -417,28 +433,117 @@ where
             )
             .ok_or(IndexResolverError::UnexistingIndex(uid))
         }
+    }
 }
 
 #[cfg(test)]
 mod test {
-    // use std::{collections::BTreeMap, vec::IntoIter};
+    use super::*;
-    //
+
-    // use super::*;
+    use nelson::Mocker;
-    //
+
-    // use futures::future::ok;
+    pub enum MockIndexResolver<U, I> {
-    // use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
+        Real(super::real::IndexResolver<U, I>),
-    // use nelson::Mocker;
+        Mock(Mocker),
-    // use proptest::prelude::*;
+    }
-    //
+
-    // use crate::{
+    impl MockIndexResolver<HeedMetaStore, MapIndexStore> {
-    //     index::{
+        pub fn load_dump(
-    //         error::{IndexError, Result as IndexResult},
+            src: impl AsRef<Path>,
-    //         Checked, IndexMeta, IndexStats, Settings,
+            dst: impl AsRef<Path>,
-    //     },
+            index_db_size: usize,
-    //     tasks::{batch::Batch, BatchHandler},
+            env: Arc<Env>,
-    // };
+            indexer_opts: &IndexerOpts,
-    // use index_store::MockIndexStore;
+        ) -> anyhow::Result<()> {
-    // use meta_store::MockIndexMetaStore;
+            super::real::IndexResolver::load_dump(src, dst, index_db_size, env, indexer_opts)
+        }
+    }
+
+    impl<U, I> MockIndexResolver<U, I>
+    where
+        U: IndexMetaStore,
+        I: IndexStore,
+    {
+        pub fn new(index_uuid_store: U, index_store: I, file_store: UpdateFileStore) -> Self {
+            Self::Real(super::real::IndexResolver {
+                index_uuid_store,
+                index_store,
+                file_store,
+            })
+        }
+
+        pub fn mock(mocker: Mocker) -> Self {
+            Self::Mock(mocker)
+        }
+
+        pub async fn process_document_addition_batch(&self, tasks: Vec<Task>) -> Vec<Task> {
+            match self {
+                IndexResolver::Real(r) => r.process_document_addition_batch(tasks).await,
+                IndexResolver::Mock(m) => unsafe {
+                    m.get("process_document_addition_batch").call(tasks)
+                },
+            }
+        }
+
+        pub async fn process_task(&self, task: &Task) -> Result<TaskResult> {
+            match self {
+                IndexResolver::Real(r) => r.process_task(task).await,
+                IndexResolver::Mock(m) => unsafe { m.get("process_task").call(task) },
+            }
+        }
+
+        pub async fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
+            match self {
+                IndexResolver::Real(r) => r.dump(path).await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        /// Get or create an index with name `uid`.
+        pub async fn get_or_create_index(&self, uid: IndexUid, task_id: TaskId) -> Result<Index> {
+            match self {
+                IndexResolver::Real(r) => r.get_or_create_index(uid, task_id).await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        pub async fn list(&self) -> Result<Vec<(String, Index)>> {
+            match self {
+                IndexResolver::Real(r) => r.list().await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        pub async fn delete_index(&self, uid: String) -> Result<Index> {
+            match self {
+                IndexResolver::Real(r) => r.delete_index(uid).await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        pub async fn get_index(&self, uid: String) -> Result<Index> {
+            match self {
+                IndexResolver::Real(r) => r.get_index(uid).await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        pub async fn get_index_creation_task_id(&self, index_uid: String) -> Result<TaskId> {
+            match self {
+                IndexResolver::Real(r) => r.get_index_creation_task_id(index_uid).await,
+                IndexResolver::Mock(_) => todo!(),
+            }
+        }
+
+        pub async fn delete_content_file(&self, content_uuid: Uuid) -> Result<()> {
+            match self {
+                IndexResolver::Real(r) => r.delete_content_file(content_uuid).await,
+                IndexResolver::Mock(m) => unsafe {
+                    m.get("delete_content_file").call(content_uuid)
+                },
+            }
+        }
+    }
 
     // TODO: ignoring this test, it has become too complex to maintain, and rather implement
     // handler logic test.

meilisearch-lib/src/tasks/handlers/index_resolver_handler.rs

@@ -38,7 +38,7 @@ where
         if let BatchContent::DocumentsAdditionBatch(ref tasks) = batch.content {
             for task in tasks {
                 if let Some(content_uuid) = task.get_content_uuid() {
-                    if let Err(e) = self.file_store.delete(content_uuid).await {
+                    if let Err(e) = self.delete_content_file(content_uuid).await {
                         log::error!("error deleting update file: {}", e);
                     }
                 }
@@ -49,7 +49,12 @@ where
 
 #[cfg(test)]
 mod test {
-    use crate::index_resolver::{index_store::MockIndexStore, meta_store::MockIndexMetaStore};
+    use crate::index_resolver::index_store::MapIndexStore;
+    use crate::index_resolver::meta_store::HeedMetaStore;
+    use crate::index_resolver::{
+        error::Result as IndexResult, index_store::MockIndexStore, meta_store::MockIndexMetaStore,
+    };
+    use crate::tasks::task::TaskResult;
     use crate::tasks::{
         handlers::test::task_to_batch,
         task::{Task, TaskContent},
@@ -142,5 +147,58 @@ mod test {
         index_resolver.process_batch(batch).await;
     }
 
-    // TODO: test perform_batch. We need a Mocker for IndexResolver.
+    proptest! {
+        #[test]
+        fn index_document_task_deletes_update_file(
+            task in any::<Task>(),
+        ) {
+            let rt = tokio::runtime::Runtime::new().unwrap();
+            let handle = rt.spawn(async {
+                let mocker = Mocker::default();
+
+                if let TaskContent::DocumentAddition{ .. } = task.content {
+                    mocker.when::<Uuid, IndexResult<()>>("delete_content_file").then(|_| Ok(()));
+                }
+
+                let index_resolver: IndexResolver<HeedMetaStore, MapIndexStore> = IndexResolver::mock(mocker);
+
+                let batch = task_to_batch(task);
+
+                index_resolver.finish(&batch).await;
+            });
+
+            rt.block_on(handle).unwrap();
+        }
+
+        #[test]
+        fn test_handle_batch(task in any::<Task>()) {
+            let rt = tokio::runtime::Runtime::new().unwrap();
+            let handle = rt.spawn(async {
+                let mocker = Mocker::default();
+                match task.content {
+                    TaskContent::DocumentAddition { .. } => {
+                        mocker.when::<Vec<Task>, Vec<Task>>("process_document_addition_batch").then(|tasks| tasks);
+                    }
+                    TaskContent::Dump { .. } => (),
+                    _ => {
+                        mocker.when::<&Task, IndexResult<TaskResult>>("process_task").then(|_| Ok(TaskResult::Other));
+                    }
+                }
+                let index_resolver: IndexResolver<HeedMetaStore, MapIndexStore> = IndexResolver::mock(mocker);
+
+
+                let batch = task_to_batch(task);
+
+                if index_resolver.accept(&batch) {
+                    index_resolver.process_batch(batch).await;
+                }
+            });
+
+            if let Err(e) = rt.block_on(handle) {
+                if e.is_panic() {
+                    std::panic::resume_unwind(e.into_panic());
+                }
+            }
+        }
+    }
 }