From c97d51a62469764b90a4a8d0fdc06648ba05aadd Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 14 Sep 2022 01:48:58 +0200 Subject: [PATCH] add a bunch of tests --- Cargo.lock | 64 +++ index-scheduler/Cargo.toml | 1 + index-scheduler/src/autobatcher.rs | 816 ++++++++++++++++++++++++++++- 3 files changed, 880 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e18946125..74db62fda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -790,6 +790,19 @@ dependencies = [ "syn 1.0.101", ] +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size", + "winapi", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -1089,6 +1102,12 @@ dependencies = [ "void", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding" version = "0.2.33" @@ -1766,6 +1785,7 @@ dependencies = [ "csv", "file-store", "index", + "insta", "log", "milli 0.33.0", "nelson", @@ -1788,6 +1808,19 @@ dependencies = [ "serde", ] +[[package]] +name = "insta" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581d4e3314cae4536e5d22ffd23189d4a374696c5ef733eadafae0ed273fd303" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "similar", + "yaml-rust", +] + [[package]] name = "instant" version = "0.1.12" @@ -2110,6 +2143,12 @@ dependencies = [ "yada", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "lmdb-rkv-sys" version = "0.15.0" @@ -3521,6 +3560,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ac7f900db32bf3fd12e0117dd3dc4da74bc52ebaac97f39668446d89694803" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -3742,6 +3787,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "termtree" version = "0.2.4" @@ -4317,6 +4372,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d12cb7a57bbf2ab670ed9545bae3648048547f9039279a89ce000208e585c1" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yaup" version = "0.2.1" diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index 45d21e0ec..770ae4424 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -22,3 +22,4 @@ uuid = { version = "1.1.2", features = ["serde", "v4"] } [dev-dependencies] nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} +insta = "1.19.1" diff --git a/index-scheduler/src/autobatcher.rs b/index-scheduler/src/autobatcher.rs index 9a57dbe7b..44a338f77 100644 --- a/index-scheduler/src/autobatcher.rs +++ b/index-scheduler/src/autobatcher.rs @@ -2,6 +2,7 @@ use std::ops::ControlFlow; use crate::{task::Kind, TaskId}; +#[derive(Debug)] pub enum BatchKind { DocumentClear { ids: Vec, @@ -367,5 +368,818 @@ pub fn autobatch(enqueued: Vec<(TaskId, Kind)>) -> Option { }; } - None + Some(acc) +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::*; + use Kind::*; + + fn input_from(input: impl IntoIterator) -> Vec<(TaskId, Kind)> { + input + .into_iter() + .enumerate() + .map(|(id, kind)| (id as TaskId, kind)) + .collect() + } + + #[test] + fn autobatch_simple_operation_together() { + // we can autobatch one or multiple DocumentAddition together + assert_debug_snapshot!(autobatch(input_from([DocumentAddition])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentAddition])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + // we can autobatch one or multiple DocumentUpdate together + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentUpdate])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + // we can autobatch one or multiple DocumentDeletion together + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentDeletion, DocumentDeletion])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + // we can autobatch one or multiple Settings together + assert_debug_snapshot!(autobatch(input_from([Settings])), @r###" + Some( + Settings { + settings_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([Settings, Settings, Settings])), @r###" + Some( + Settings { + settings_ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + } + + #[test] + fn simple_document_operation_dont_autobatch_with_other() { + // addition, updates and deletion can't batch together + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentUpdate])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentDeletion])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentAddition])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentDeletion])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentAddition])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, DocumentUpdate])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexCreation])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexCreation])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexCreation])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexUpdate])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexUpdate])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexUpdate])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexRename])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexRename])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexRename])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexSwap])), @r###" + Some( + DocumentAddition { + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexSwap])), @r###" + Some( + DocumentUpdate { + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexSwap])), @r###" + Some( + DocumentDeletion { + deletion_ids: [ + 0, + ], + }, + ) + "###); + } + + #[test] + fn document_addition_batch_with_settings() { + // simple case + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + + // multiple settings and doc addition + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, Settings, Settings])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 2, + 3, + ], + addition_ids: [ + 0, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, Settings, Settings])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 2, + 3, + ], + addition_ids: [ + 0, + 1, + ], + }, + ) + "###); + + // addition and setting unordered + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentAddition, Settings])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + 3, + ], + addition_ids: [ + 0, + 2, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentUpdate, Settings])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + 3, + ], + update_ids: [ + 0, + 2, + ], + }, + ) + "###); + + // We ensure this kind of batch doesn't batch with forbidden operations + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentUpdate])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentAddition])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentDeletion])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentDeletion])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexCreation])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexCreation])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexUpdate])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexUpdate])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexRename])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexRename])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexSwap])), @r###" + Some( + SettingsAndDocumentAddition { + settings_ids: [ + 1, + ], + addition_ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexSwap])), @r###" + Some( + SettingsAndDocumentUpdate { + settings_ids: [ + 1, + ], + update_ids: [ + 0, + ], + }, + ) + "###); + } + + #[test] + fn clear_and_additions() { + // these two doesn't need to batch + assert_debug_snapshot!(autobatch(input_from([DocumentClear, DocumentAddition])), @r###" + Some( + DocumentClear { + ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentClear, DocumentUpdate])), @r###" + Some( + DocumentClear { + ids: [ + 0, + ], + }, + ) + "###); + + // Basic use case + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + + // This batch kind doesn't mix with other document addition + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentAddition])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentUpdate])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + ], + }, + ) + "###); + + // But you can batch multiple clear together + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, DocumentAddition, DocumentClear, DocumentClear, DocumentClear])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + 3, + 4, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, DocumentUpdate, DocumentClear, DocumentClear, DocumentClear])), @r###" + Some( + DocumentClear { + ids: [ + 0, + 1, + 2, + 3, + 4, + ], + }, + ) + "###); + } + + #[test] + fn clear_and_additions_and_settings() { + // A clear don't need to autobatch the settings that happens AFTER there is no documents + assert_debug_snapshot!(autobatch(input_from([DocumentClear, Settings])), @r###" + Some( + DocumentClear { + ids: [ + 0, + ], + }, + ) + "###); + + assert_debug_snapshot!(autobatch(input_from([Settings, DocumentClear, Settings])), @r###" + Some( + ClearAndSettings { + other: [ + 1, + ], + settings_ids: [ + 0, + 2, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentClear])), @r###" + Some( + ClearAndSettings { + other: [ + 0, + 2, + ], + settings_ids: [ + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentClear])), @r###" + Some( + ClearAndSettings { + other: [ + 0, + 2, + ], + settings_ids: [ + 1, + ], + }, + ) + "###); + } + + #[test] + fn anything_and_index_deletion() { + // The indexdeletion doesn't batch with anything that happens AFTER + assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentAddition])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentUpdate])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([IndexDeletion, DocumentClear])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([IndexDeletion, Settings])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + ], + }, + ) + "###); + + // The index deletion can accept almost any type of BatchKind and transform it to an IndexDeletion + // First, the basic cases + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentDeletion, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentClear, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([Settings, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 1, + ], + }, + ) + "###); + + // Then the mixed cases + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 2, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 0, + 2, + 1, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentAddition, Settings, DocumentClear, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 1, + 3, + 0, + 2, + ], + }, + ) + "###); + assert_debug_snapshot!(autobatch(input_from([DocumentUpdate, Settings, DocumentClear, IndexDeletion])), @r###" + Some( + IndexDeletion { + ids: [ + 1, + 3, + 0, + 2, + ], + }, + ) + "###); + } }