snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-23 02:27:40 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

reintroduce anyhow

Browse Source
This commit is contained in:
marin postma 2021-06-15 17:39:07 +02:00
parent 439db1aae0
commit 02277ec2cf
No known key found for this signature in database
GPG Key ID: 6088B7721C3E39F9
36 changed files with 110 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
meilisearch-http/Cargo.toml
View File

@ -27,7 +27,7 @@ actix-http = { version = "=3.0.0-beta.6" }
actix-service = "2.0.0"
actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] }
actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
#anyhow = "1.0.36"
anyhow = "1.0.36"
async-stream = "0.3.0"
async-trait = "0.1.42"
arc-swap = "1.2.0"

5
meilisearch-http/src/data/mod.rs
View File

@ -5,8 +5,7 @@ use sha2::Digest;
use crate::index::{Checked, Settings};
use crate::index_controller::{
DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
error::Result
error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
};
use crate::option::Opt;
@ -57,7 +56,7 @@ impl ApiKeys {
}
impl Data {
pub fn new(options: Opt) -> std::result::Result<Data, Box<dyn std::error::Error>> {
pub fn new(options: Opt) -> anyhow::Result<Data> {
let path = options.db_path.clone();
let index_controller = IndexController::new(&path, &options)?;

6
meilisearch-http/src/data/search.rs
View File

@ -5,11 +5,7 @@ use crate::index::{SearchQuery, SearchResult};
use crate::index_controller::error::Result;
impl Data {
pub async fn search(
&self,
index: String,
search_query: SearchQuery,
) -> Result<SearchResult> {
pub async fn search(&self, index: String, search_query: SearchQuery) -> Result<SearchResult> {
self.index_controller.search(index, search_query).await
}

2
meilisearch-http/src/data/updates.rs
View File

@ -3,7 +3,7 @@ use milli::update::{IndexDocumentsMethod, UpdateFormat};
use super::Data;
use crate::index::{Checked, Settings};
use crate::index_controller::{IndexMetadata, IndexSettings, UpdateStatus, error::Result};
use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus};
impl Data {
pub async fn add_documents(

12
meilisearch-http/src/error.rs
View File

@ -62,11 +62,7 @@ macro_rules! response_error {
};
}
response_error!(
IndexControllerError,
AuthenticationError
);
response_error!(IndexControllerError, AuthenticationError);
impl Serialize for ResponseError {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@ -114,7 +110,8 @@ impl<E: Error> ErrorCode for PayloadError<E> {
}
impl<E> From<PayloadError<E>> for ResponseError
where E: Error + Sync + Send + 'static
where
E: Error + Sync + Send + 'static,
{
fn from(other: PayloadError<E>) -> Self {
ResponseError {
@ -124,7 +121,8 @@ where E: Error + Sync + Send + 'static
}
pub fn payload_error_handler<E>(err: E) -> ResponseError
where E: Error + Sync + Send + 'static
where
E: Error + Sync + Send + 'static,
{
let error = PayloadError(err);
error.into()

8
meilisearch-http/src/helpers/authentication.rs
View File

@ -9,7 +9,7 @@ use futures::future::{ok, Future, Ready};
use futures::ready;
use pin_project::pin_project;
use crate::error::{ResponseError, AuthenticationError};
use crate::error::{AuthenticationError, ResponseError};
use crate::Data;
#[derive(Clone, Copy)]
@ -117,7 +117,8 @@ where
AuthProj::NoHeader(req) => {
match req.take() {
Some(req) => {
let response = ResponseError::from(AuthenticationError::MissingAuthorizationHeader);
let response =
ResponseError::from(AuthenticationError::MissingAuthorizationHeader);
let response = response.error_response();
let response = req.into_response(response);
Poll::Ready(Ok(response))
@ -134,7 +135,8 @@ where
.get("X-Meili-API-Key")
.map(|h| h.to_str().map(String::from).unwrap_or_default())
.unwrap_or_default();
let response = ResponseError::from(AuthenticationError::InvalidToken(bad_token));
let response =
ResponseError::from(AuthenticationError::InvalidToken(bad_token));
let response = response.error_response();
let response = req.into_response(response);
Poll::Ready(Ok(response))

4
meilisearch-http/src/helpers/compression.rs
View File

@ -5,7 +5,7 @@ use std::path::Path;
use flate2::{read::GzDecoder, write::GzEncoder, Compression};
use tar::{Archive, Builder};
pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<(), Box<dyn std::error::Error + Send + Sync + 'static>> {
pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
let mut f = File::create(dest)?;
let gz_encoder = GzEncoder::new(&mut f, Compression::default());
let mut tar_encoder = Builder::new(gz_encoder);
@ -16,7 +16,7 @@ pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<(), Bo
Ok(())
}
pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<(), Box<dyn std::error::Error>> {
pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
let f = File::open(&src)?;
let gz = GzDecoder::new(f);
let mut ar = Archive::new(gz);

13
meilisearch-http/src/index/dump.rs
View File

@ -3,6 +3,7 @@ use std::io::{BufRead, BufReader, Write};
use std::path::Path;
use std::sync::Arc;
use anyhow::Context;
use heed::RoTxn;
use indexmap::IndexMap;
use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream};
@ -10,8 +11,8 @@ use serde::{Deserialize, Serialize};
use crate::option::IndexerOpts;
use super::{update_handler::UpdateHandler, Index, Settings, Unchecked};
use super::error::{IndexError, Result};
use super::{update_handler::UpdateHandler, Index, Settings, Unchecked};
#[derive(Serialize, Deserialize)]
struct DumpMeta {
@ -37,7 +38,8 @@ impl Index {
let document_file_path = path.as_ref().join(DATA_FILE_NAME);
let mut document_file = File::create(&document_file_path)?;
let documents = self.all_documents(txn)
let documents = self
.all_documents(txn)
.map_err(|e| IndexError::Internal(e.into()))?;
let fields_ids_map = self.fields_ids_map(txn)?;
@ -82,13 +84,12 @@ impl Index {
dst: impl AsRef<Path>,
size: usize,
indexing_options: &IndexerOpts,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
let dir_name = src
.as_ref()
.file_name()
// TODO: remove
//.with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
.unwrap();
.with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
let dst_dir_path = dst.as_ref().join("indexes").join(dir_name);
create_dir_all(&dst_dir_path)?;

10
meilisearch-http/src/index/error.rs
View File

@ -27,12 +27,7 @@ macro_rules! internal_error {
}
}
internal_error!(
std::io::Error,
heed::Error,
fst::Error,
serde_json::Error
);
internal_error!(std::io::Error, heed::Error, fst::Error, serde_json::Error);
impl ErrorCode for IndexError {
fn error_code(&self) -> Code {
@ -47,7 +42,7 @@ impl ErrorCode for IndexError {
#[derive(Debug, thiserror::Error)]
pub enum FacetError {
#[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))]
InvalidExpression(&'static [&'static str], Value)
InvalidExpression(&'static [&'static str], Value),
}
impl ErrorCode for FacetError {
@ -57,4 +52,3 @@ impl ErrorCode for FacetError {
}
}
}

2
meilisearch-http/src/index/update_handler.rs
View File

@ -21,7 +21,7 @@ pub struct UpdateHandler {
}
impl UpdateHandler {
pub fn new(opt: &IndexerOpts) -> std::result::Result<Self, Box<dyn std::error::Error>> {
pub fn new(opt: &IndexerOpts) -> anyhow::Result<Self> {
let thread_pool = rayon::ThreadPoolBuilder::new()
.num_threads(opt.indexing_jobs.unwrap_or(0))
.build()?;

10
meilisearch-http/src/index/updates.rs
View File

@ -1,4 +1,4 @@
use std::collections::{BTreeSet, BTreeMap, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::io;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
@ -308,9 +308,8 @@ impl Index {
}
}
builder.execute(|indexing_step, update_id| {
info!("update {}: {:?}", update_id, indexing_step)
})
builder
.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step))
.map_err(|e| IndexError::Internal(e.into()))?;
Ok(UpdateResult::Other)
@ -333,7 +332,8 @@ impl Index {
update_builder: UpdateBuilder,
) -> Result<UpdateResult> {
let mut txn = self.write_txn()?;
let mut builder = update_builder.delete_documents(&mut txn, self)
let mut builder = update_builder
.delete_documents(&mut txn, self)
.map_err(|e| IndexError::Internal(e.into()))?;
// We ignore unexisting document ids

2
meilisearch-http/src/index_controller/dump_actor/actor.rs
View File

@ -10,9 +10,9 @@ use tokio::sync::{mpsc, oneshot, RwLock};
use update_actor::UpdateActorHandle;
use uuid_resolver::UuidResolverHandle;
use super::error::{DumpActorError, Result};
use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask};
use crate::index_controller::{update_actor, uuid_resolver};
use super::error::{DumpActorError, Result};
pub const CONCURRENT_DUMP_MSG: usize = 10;

4
meilisearch-http/src/index_controller/dump_actor/error.rs
View File

@ -1,6 +1,8 @@
use meilisearch_error::{Code, ErrorCode};
use crate::index_controller::{update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError};
use crate::index_controller::{
update_actor::error::UpdateActorError, uuid_resolver::UuidResolverError,
};
pub type Result<T> = std::result::Result<T, DumpActorError>;

4
meilisearch-http/src/index_controller/dump_actor/handle_impl.rs
View File

@ -3,8 +3,8 @@ use std::path::Path;
use actix_web::web::Bytes;
use tokio::sync::{mpsc, oneshot};
use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg};
use super::error::Result;
use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg};
#[derive(Clone)]
pub struct DumpActorHandleImpl {
@ -35,7 +35,7 @@ impl DumpActorHandleImpl {
update: crate::index_controller::update_actor::UpdateActorHandleImpl<Bytes>,
index_db_size: usize,
update_db_size: usize,
) -> std::result::Result<Self, Box<dyn std::error::Error>> {
) -> anyhow::Result<Self> {
let (sender, receiver) = mpsc::channel(10);
let actor = DumpActor::new(
receiver,

6
meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs
View File

@ -31,7 +31,7 @@ impl MetadataV1 {
dst: impl AsRef<Path>,
size: usize,
indexer_options: &IndexerOpts,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
info!(
"Loading dump, dump database version: {}, dump version: V1",
self.db_version
@ -83,7 +83,7 @@ fn load_index(
primary_key: Option<&str>,
size: usize,
indexer_options: &IndexerOpts,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid));
create_dir_all(&index_path)?;
@ -172,7 +172,7 @@ impl From<Settings> for index_controller::Settings<Unchecked> {
}
/// Extract Settings from `settings.json` file present at provided `dir_path`
fn import_settings(dir_path: impl AsRef<Path>) -> std::result::Result<Settings, Box<dyn std::error::Error>> {
fn import_settings(dir_path: impl AsRef<Path>) -> anyhow::Result<Settings> {
let path = dir_path.as_ref().join("settings.json");
let file = File::open(path)?;
let reader = std::io::BufReader::new(file);

2
meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs
View File

@ -34,7 +34,7 @@ impl MetadataV2 {
index_db_size: usize,
update_db_size: usize,
indexing_options: &IndexerOpts,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
info!(
"Loading dump from {}, dump database version: {}, dump version: V2",
self.dump_date, self.db_version

2
meilisearch-http/src/index_controller/dump_actor/message.rs
View File

@ -1,7 +1,7 @@
use tokio::sync::oneshot;
use super::DumpInfo;
use super::error::Result;
use super::DumpInfo;
pub enum DumpMsg {
CreateDump {

11
meilisearch-http/src/index_controller/dump_actor/mod.rs
View File

@ -1,6 +1,7 @@
use std::fs::File;
use std::path::{Path, PathBuf};
use anyhow::Context;
use chrono::{DateTime, Utc};
use log::{info, warn};
#[cfg(test)]
@ -21,10 +22,10 @@ use crate::{helpers::compression, option::IndexerOpts};
use error::Result;
mod actor;
pub mod error;
mod handle_impl;
mod loaders;
mod message;
pub mod error;
const META_FILE_NAME: &str = "metadata.json";
@ -107,7 +108,7 @@ pub fn load_dump(
index_db_size: usize,
update_db_size: usize,
indexer_opts: &IndexerOpts,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
let tmp_src = tempfile::tempdir_in(".")?;
let tmp_src_path = tmp_src.path();
@ -120,9 +121,7 @@ pub fn load_dump(
let dst_dir = dst_path
.as_ref()
.parent()
// TODO
//.with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?;
.unwrap();
.with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?;
let tmp_dst = tempfile::tempdir_in(dst_dir)?;
@ -188,7 +187,7 @@ where
let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;
compression::to_tar_gz(temp_dump_path, temp_dump_file.path())
.map_err(|e| DumpActorError::Internal(e))?;
.map_err(|e| DumpActorError::Internal(e.into()))?;
let dump_path = self.path.join(self.uid).with_extension("dump");
temp_dump_file.persist(&dump_path)?;

14
meilisearch-http/src/index_controller/index_actor/actor.rs
View File

@ -19,8 +19,8 @@ use crate::index_controller::{
};
use crate::option::IndexerOpts;
use super::error::{IndexActorError, Result};
use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore};
use super::error::{Result, IndexActorError};
pub const CONCURRENT_INDEX_MSG: usize = 10;
@ -31,7 +31,7 @@ pub struct IndexActor<S> {
}
impl<S: IndexStore + Sync + Send> IndexActor<S> {
pub fn new(receiver: mpsc::Receiver<IndexMsg>, store: S) -> std::result::Result<Self, Box<dyn std::error::Error>> {
pub fn new(receiver: mpsc::Receiver<IndexMsg>, store: S) -> anyhow::Result<Self> {
let options = IndexerOpts::default();
let update_handler = UpdateHandler::new(&options)?;
let update_handler = Arc::new(update_handler);
@ -146,7 +146,6 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
.ok_or(IndexActorError::UnexistingIndex)?;
let result = spawn_blocking(move || index.perform_search(query)).await??;
Ok(result)
}
async fn handle_create_index(
@ -269,7 +268,8 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
}
let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
builder.set_primary_key(primary_key);
builder.execute(|_, _| ())
builder
.execute(|_, _| ())
.map_err(|e| IndexActorError::Internal(Box::new(e)))?;
let meta = IndexMeta::new_txn(&index, &txn)?;
txn.commit()?;
@ -340,10 +340,12 @@ impl<S: IndexStore + Sync + Send> IndexActor<S> {
Ok(IndexStats {
size: index.size(),
number_of_documents: index.number_of_documents(&rtxn)
number_of_documents: index
.number_of_documents(&rtxn)
.map_err(|e| IndexActorError::Internal(Box::new(e)))?,
is_indexing: None,
fields_distribution: index.fields_distribution(&rtxn)
fields_distribution: index
.fields_distribution(&rtxn)
.map_err(|e| IndexActorError::Internal(e.into()))?,
})
})

6
meilisearch-http/src/index_controller/index_actor/error.rs
View File

@ -30,11 +30,7 @@ macro_rules! internal_error {
}
}
internal_error!(
heed::Error,
tokio::task::JoinError,
std::io::Error
);
internal_error!(heed::Error, tokio::task::JoinError, std::io::Error);
impl ErrorCode for IndexActorError {
fn error_code(&self) -> Code {

16
meilisearch-http/src/index_controller/index_actor/handle_impl.rs
View File

@ -12,8 +12,8 @@ use crate::{
index_controller::{Failed, Processed},
};
use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore};
use super::error::Result;
use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore};
#[derive(Clone)]
pub struct IndexActorHandleImpl {
@ -22,11 +22,7 @@ pub struct IndexActorHandleImpl {
#[async_trait::async_trait]
impl IndexActorHandle for IndexActorHandleImpl {
async fn create_index(
&self,
uuid: Uuid,
primary_key: Option<String>,
) -> Result<IndexMeta> {
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
let (ret, receiver) = oneshot::channel();
let msg = IndexMsg::CreateIndex {
ret,
@ -118,11 +114,7 @@ impl IndexActorHandle for IndexActorHandleImpl {
Ok(receiver.await.expect("IndexActor has been killed")?)
}
async fn update_index(
&self,
uuid: Uuid,
index_settings: IndexSettings,
) -> Result<IndexMeta> {
async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta> {
let (ret, receiver) = oneshot::channel();
let msg = IndexMsg::UpdateIndex {
uuid,
@ -156,7 +148,7 @@ impl IndexActorHandle for IndexActorHandleImpl {
}
impl IndexActorHandleImpl {
pub fn new(path: impl AsRef<Path>, index_size: usize) -> std::result::Result<Self, Box<dyn std::error::Error>> {
pub fn new(path: impl AsRef<Path>, index_size: usize) -> anyhow::Result<Self> {
let (sender, receiver) = mpsc::channel(100);
let store = MapIndexStore::new(path, index_size);

2
meilisearch-http/src/index_controller/index_actor/message.rs
View File

@ -3,9 +3,9 @@ use std::path::PathBuf;
use tokio::sync::oneshot;
use uuid::Uuid;
use super::error::Result as IndexResult;
use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
use crate::index_controller::{Failed, IndexStats, Processed, Processing};
use super::error::Result as IndexResult;
use super::{IndexMeta, IndexSettings};

17
meilisearch-http/src/index_controller/index_actor/mod.rs
View File

@ -22,10 +22,10 @@ use self::error::IndexActorError;
use super::IndexSettings;
mod actor;
pub mod error;
mod handle_impl;
mod message;
mod store;
pub mod error;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
@ -60,8 +60,7 @@ impl IndexMeta {
#[async_trait::async_trait]
#[cfg_attr(test, automock)]
pub trait IndexActorHandle {
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>)
-> Result<IndexMeta>;
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>;
async fn update(
&self,
uuid: Uuid,
@ -86,11 +85,7 @@ pub trait IndexActorHandle {
) -> Result<Document>;
async fn delete(&self, uuid: Uuid) -> Result<()>;
async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta>;
async fn update_index(
&self,
uuid: Uuid,
index_settings: IndexSettings,
) -> Result<IndexMeta>;
async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta>;
async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats>;
@ -105,11 +100,7 @@ mod test {
#[async_trait::async_trait]
/// Useful for passing around an `Arc<MockIndexActorHandle>` in tests.
impl IndexActorHandle for Arc<MockIndexActorHandle> {
async fn create_index(
&self,
uuid: Uuid,
primary_key: Option<String>,
) -> Result<IndexMeta> {
async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
self.as_ref().create_index(uuid, primary_key).await
}

9
meilisearch-http/src/index_controller/mod.rs
View File

@ -29,12 +29,12 @@ use self::dump_actor::load_dump;
use self::error::IndexControllerError;
mod dump_actor;
pub mod error;
mod index_actor;
mod snapshot;
mod update_actor;
mod updates;
mod uuid_resolver;
pub mod error;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
@ -83,7 +83,7 @@ pub struct Stats {
}
impl IndexController {
pub fn new(path: impl AsRef<Path>, options: &Opt) -> std::result::Result<Self, Box<dyn std::error::Error>> {
pub fn new(path: impl AsRef<Path>, options: &Opt) -> anyhow::Result<Self> {
let index_size = options.max_mdb_size.get_bytes() as usize;
let update_store_size = options.max_udb_size.get_bytes() as usize;
@ -238,10 +238,7 @@ impl IndexController {
}
}
pub async fn create_index(
&self,
index_settings: IndexSettings,
) -> Result<IndexMetadata> {
pub async fn create_index(&self, index_settings: IndexSettings) -> Result<IndexMetadata> {
let IndexSettings { uid, primary_key } = index_settings;
let uid = uid.ok_or(IndexControllerError::MissingUid)?;
let uuid = Uuid::new_v4();

6
meilisearch-http/src/index_controller/snapshot.rs
View File

@ -52,7 +52,7 @@ where
}
}
async fn perform_snapshot(&self) -> std::result::Result<(), Box<dyn std::error::Error + Sync + Send + 'static>> {
async fn perform_snapshot(&self) -> anyhow::Result<()> {
info!("Performing snapshot.");
let snapshot_dir = self.snapshot_path.clone();
@ -77,7 +77,7 @@ where
let snapshot_path = self
.snapshot_path
.join(format!("{}.snapshot", self.db_name));
let snapshot_path = spawn_blocking(move || -> Result<PathBuf, Box<dyn std::error::Error + Sync + Send + 'static>> {
let snapshot_path = spawn_blocking(move || -> anyhow::Result<PathBuf> {
let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?;
let temp_snapshot_file_path = temp_snapshot_file.path().to_owned();
compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?;
@ -97,7 +97,7 @@ pub fn load_snapshot(
snapshot_path: impl AsRef<Path>,
ignore_snapshot_if_db_exists: bool,
ignore_missing_snapshot: bool,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() {
match compression::from_tar_gz(snapshot_path, &db_path) {
Ok(()) => Ok(()),

4
meilisearch-http/src/index_controller/update_actor/actor.rs
View File

@ -13,8 +13,8 @@ use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
use uuid::Uuid;
use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo};
use super::error::{Result, UpdateActorError};
use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo};
use crate::index_controller::index_actor::IndexActorHandle;
use crate::index_controller::{UpdateMeta, UpdateStatus};
@ -36,7 +36,7 @@ where
inbox: mpsc::Receiver<UpdateMsg<D>>,
path: impl AsRef<Path>,
index_handle: I,
) -> std::result::Result<Self, Box<dyn std::error::Error>> {
) -> anyhow::Result<Self> {
let path = path.as_ref().join("updates");
std::fs::create_dir_all(&path)?;

1
meilisearch-http/src/index_controller/update_actor/error.rs
View File

@ -45,6 +45,7 @@ impl From<tokio::sync::oneshot::error::RecvError> for UpdateActorError {
}
internal_error!(
UpdateActorError:
heed::Error,
std::io::Error,
serde_json::Error,

2
meilisearch-http/src/index_controller/update_actor/handle_impl.rs
View File

@ -22,7 +22,7 @@ where
index_handle: I,
path: impl AsRef<Path>,
update_store_size: usize,
) -> std::result::Result<Self, Box<dyn std::error::Error>>
) -> anyhow::Result<Self>
where
I: IndexActorHandle + Clone + Send + Sync + 'static,
{

2
meilisearch-http/src/index_controller/update_actor/message.rs
View File

@ -4,8 +4,8 @@ use std::path::PathBuf;
use tokio::sync::{mpsc, oneshot};
use uuid::Uuid;
use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo};
use super::error::Result;
use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo};
pub enum UpdateMsg<D> {
Update {

4
meilisearch-http/src/index_controller/update_actor/mod.rs
View File

@ -7,16 +7,16 @@ use uuid::Uuid;
use crate::index_controller::{UpdateMeta, UpdateStatus};
use actor::UpdateActor;
use message::UpdateMsg;
use error::Result;
use message::UpdateMsg;
pub use handle_impl::UpdateActorHandleImpl;
pub use store::{UpdateStore, UpdateStoreInfo};
mod actor;
pub mod error;
mod handle_impl;
mod message;
pub mod error;
pub mod store;
type PayloadData<D> = std::result::Result<D, PayloadError>;

4
meilisearch-http/src/index_controller/update_actor/store/dump.rs
View File

@ -9,7 +9,7 @@ use heed::{EnvOpenOptions, RoTxn};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::{State, UpdateStore, Result};
use super::{Result, State, UpdateStore};
use crate::index_controller::{
index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued,
UpdateStatus,
@ -125,7 +125,7 @@ impl UpdateStore {
src: impl AsRef<Path>,
dst: impl AsRef<Path>,
db_size: usize,
) -> std::result::Result<(), Box<dyn std::error::Error>> {
) -> anyhow::Result<()> {
let dst_update_path = dst.as_ref().join("updates/");
create_dir_all(&dst_update_path)?;

39
meilisearch-http/src/index_controller/update_actor/store/mod.rs
View File

@ -23,10 +23,10 @@ use uuid::Uuid;
use codec::*;
use super::UpdateMeta;
use super::error::Result;
use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
use super::UpdateMeta;
use crate::helpers::EnvSizer;
use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
#[allow(clippy::upper_case_acronyms)]
type BEU64 = U64<heed::byteorder::BE>;
@ -110,7 +110,7 @@ impl UpdateStore {
fn new(
mut options: EnvOpenOptions,
path: impl AsRef<Path>,
) -> std::result::Result<(Self, mpsc::Receiver<()>), Box<dyn std::error::Error>> {
) -> anyhow::Result<(Self, mpsc::Receiver<()>)> {
options.max_dbs(5);
let env = options.open(&path)?;
@ -141,7 +141,7 @@ impl UpdateStore {
path: impl AsRef<Path>,
index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static,
must_exit: Arc<AtomicBool>,
) -> std::result::Result<Arc<Self>, Box<dyn std::error::Error>> {
) -> anyhow::Result<Arc<Self>> {
let (update_store, mut notification_receiver) = Self::new(options, path)?;
let update_store = Arc::new(update_store);
@ -270,11 +270,8 @@ impl UpdateStore {
}
_ => {
let _update_id = self.next_update_id_raw(wtxn, index_uuid)?;
self.updates.put(
wtxn,
&(index_uuid, update.id()),
&update,
)?;
self.updates
.put(wtxn, &(index_uuid, update.id()), &update)?;
}
}
Ok(())
@ -283,10 +280,7 @@ impl UpdateStore {
/// Executes the user provided function on the next pending update (the one with the lowest id).
/// This is asynchronous as it let the user process the update with a read-only txn and
/// only writing the result meta to the processed-meta store *after* it has been processed.
fn process_pending_update(
&self,
index_handle: impl IndexActorHandle,
) -> Result<Option<()>> {
fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result<Option<()>> {
// Create a read transaction to be able to retrieve the pending update in order.
let rtxn = self.env.read_txn()?;
let first_meta = self.pending_queue.first(&rtxn)?;
@ -353,11 +347,8 @@ impl UpdateStore {
Err(res) => res.into(),
};
self.updates.put(
&mut wtxn,
&(index_uuid, update_id),
&result,
)?;
self.updates
.put(&mut wtxn, &(index_uuid, update_id), &result)?;
wtxn.commit()?;
@ -704,18 +695,10 @@ mod test {
let txn = store.env.read_txn().unwrap();
assert!(store.pending_queue.first(&txn).unwrap().is_none());
let update = store
.updates
.get(&txn, &(uuid, 0))
.unwrap()
.unwrap();
let update = store.updates.get(&txn, &(uuid, 0)).unwrap().unwrap();
assert!(matches!(update, UpdateStatus::Processed(_)));
let update = store
.updates
.get(&txn, &(uuid, 1))
.unwrap()
.unwrap();
let update = store.updates.get(&txn, &(uuid, 1)).unwrap().unwrap();
assert!(matches!(update, UpdateStatus::Failed(_)));
}

2
meilisearch-http/src/index_controller/updates.rs
View File

@ -25,7 +25,7 @@ pub enum UpdateMeta {
},
ClearDocuments,
DeleteDocuments {
ids: Vec<String>
ids: Vec<String>,
},
Settings(Settings<Unchecked>),
}

1
meilisearch-http/src/main.rs
View File

@ -30,6 +30,7 @@ async fn main() -> Result<(), MainError> {
.into(),
);
}
#[cfg(all(not(debug_assertions), feature = "analytics"))]
if !opt.no_analytics {
let logger =

16
meilisearch-http/src/routes/mod.rs
View File

@ -24,17 +24,19 @@ pub enum UpdateType {
Customs,
DocumentsAddition {
#[serde(skip_serializing_if = "Option::is_none")]
number: Option<usize>
number: Option<usize>,
},
DocumentsPartial {
#[serde(skip_serializing_if = "Option::is_none")]
number: Option<usize>
number: Option<usize>,
},
DocumentsDeletion {
#[serde(skip_serializing_if = "Option::is_none")]
number: Option<usize>
number: Option<usize>,
},
Settings {
settings: Settings<Unchecked>,
},
Settings { settings: Settings<Unchecked> },
}
impl From<&UpdateStatus> for UpdateType {
@ -60,9 +62,9 @@ impl From<&UpdateStatus> for UpdateType {
}
}
UpdateMeta::ClearDocuments => UpdateType::ClearAll,
UpdateMeta::DeleteDocuments { ids } => {
UpdateType::DocumentsDeletion { number: Some(ids.len()) }
}
UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion {
number: Some(ids.len()),
},
UpdateMeta::Settings(settings) => UpdateType::Settings {
settings: settings.clone(),
},

2
meilisearch-http/tests/index/get_index.rs
View File

@ -21,7 +21,7 @@ async fn create_and_get_index() {
assert_eq!(response.as_object().unwrap().len(), 5);
}
// TODO: partial test since we are testing error, amd error is not yet fully implemented in
// TODO: partial test since we are testing error, and error is not yet fully implemented in
// transplant
#[actix_rt::test]
async fn get_unexisting_index() {