diff --git a/Cargo.lock b/Cargo.lock index f173ce283..5bca17310 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3706,8 +3706,9 @@ checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rusty-s3" -version = "0.4.1" -source = "git+https://github.com/paolobarbolini/rusty-s3?rev=0b60b9d#0b60b9d10f2e3db01bb772768ae0afa0d57a7239" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa883f1b986a5249641e574ca0e11ac4fb9970b009c6fbb96fedaf4fa78db8" dependencies = [ "base64 0.21.2", "hmac", @@ -4008,8 +4009,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strois" -version = "0.1.0" -source = "git+http://github.com/meilisearch/strois?branch=main#f977fd50e8e082d631a1dbf98f4e75e263112981" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dde87abcdf0638bf1e0e762e5f4466f68aa9862bc061093f3914641f9db26f6" dependencies = [ "http", "log", diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index f8ec77def..ecd0e2323 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -34,7 +34,7 @@ uuid = { version = "1.3.1", features = ["serde", "v4"] } tokio = { version = "1.27.0", features = ["full"] } zookeeper = "0.8.0" parking_lot = "0.12.1" -strois = { git = "http://github.com/meilisearch/strois", branch = "main" } +strois = "0.0.4" [dev-dependencies] big_s = "1.0.2" diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 5f384c407..265919ac5 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -46,7 +46,6 @@ use dump::{KindDump, TaskDump, UpdateFile}; pub use error::Error; pub use features::RoFeatures; use file_store::FileStore; -use strois::Bucket; use meilisearch_types::error::ResponseError; use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{OwnedType, SerdeBincode, SerdeJson, Str}; @@ -57,6 +56,7 @@ use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmap use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task}; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard}; use roaring::RoaringBitmap; +use strois::Bucket; use synchronoise::SignalEvent; use tempfile::TempDir; use time::format_description::well_known::Rfc3339; @@ -577,12 +577,7 @@ impl IndexScheduler { "{snapshot_dir}/{}", meilisearch_types::VERSION_FILE_NAME ); - let mut version_file_path = - File::open(&inner.version_file_path).unwrap(); - s3.put_object_multipart(dst, &mut version_file_path, S3_PART_SIZE) - .unwrap(); - version_file_path.sync_data().unwrap(); - drop(version_file_path); + s3.put_object_file(dst, &inner.version_file_path).unwrap(); // 2. Snapshot the index-scheduler LMDB env log::info!("Snapshotting the tasks"); @@ -599,7 +594,6 @@ impl IndexScheduler { s3.put_object_multipart( format!("{snapshot_dir}/tasks.mdb"), &mut file, - S3_PART_SIZE, ) .unwrap(); temp.close().unwrap(); @@ -620,12 +614,8 @@ impl IndexScheduler { heed::CompactionOption::Enabled, ) .unwrap(); - s3.put_object_multipart( - format!("{dst}/{uuid}.mdb"), - &mut file, - S3_PART_SIZE, - ) - .unwrap(); + s3.put_object_multipart(format!("{dst}/{uuid}.mdb"), &mut file) + .unwrap(); temp.close().unwrap(); } diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index 4dfb3c0cf..f8787f4e4 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -80,7 +80,7 @@ reqwest = { version = "0.11.16", features = [ ], default-features = false } rustls = "0.20.8" rustls-pemfile = "1.0.2" -strois = { git = "http://github.com/meilisearch/strois", branch = "main" } +strois = "0.0.4" segment = { version = "0.2.2", optional = true } serde = { version = "1.0.160", features = ["derive"] } serde_json = { version = "1.0.95", features = ["preserve_order"] } diff --git a/meilisearch/src/lib.rs b/meilisearch/src/lib.rs index 478064599..b5e158dde 100644 --- a/meilisearch/src/lib.rs +++ b/meilisearch/src/lib.rs @@ -30,7 +30,6 @@ use extractors::payload::PayloadConfig; use http::header::CONTENT_TYPE; use index_scheduler::{IndexScheduler, IndexSchedulerOptions}; use log::error; -use strois::Client; use meilisearch_auth::AuthController; use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod}; @@ -40,6 +39,7 @@ use meilisearch_types::versioning::{check_version_file, create_version_file}; use meilisearch_types::{compression, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; +use strois::Client; use zookeeper::ZooKeeper; use crate::error::MeilisearchHttpError; @@ -250,7 +250,7 @@ fn open_or_create_database_unchecked( .key(opt.s3_access_key.as_ref().expect("Need s3 key to work").clone()) .secret(opt.s3_secret_key.as_ref().expect("Need s3 secret to work").clone()) .maybe_token(opt.s3_security_token.clone()) - .build() + .region(&opt.s3_region) .bucket(opt.s3_bucket.as_ref().expect("Need an s3 bucket to work")) .unwrap() .get_or_create() diff --git a/meilisearch/src/routes/indexes/documents.rs b/meilisearch/src/routes/indexes/documents.rs index ff949ca94..d370b6ae1 100644 --- a/meilisearch/src/routes/indexes/documents.rs +++ b/meilisearch/src/routes/indexes/documents.rs @@ -413,11 +413,7 @@ async fn document_addition( if let Some(s3) = s3 { update_file.seek(SeekFrom::Start(0)).unwrap(); let mut reader = BufReader::new(&*update_file); - s3.put_object_multipart( - format!("update-files/{}", uuid), - &mut reader, - 50 * 1024 * 1024, - )?; + s3.put_object_multipart(format!("update-files/{}", uuid), &mut reader)?; } // we NEED to persist the file here because we moved the `udpate_file` in another task.