meilisearch/crates/index-scheduler/src/queue/test.rs

use big_s::S;
use meili_snap::{json_string, snapshot};
use meilisearch_types::error::ErrorCode;
use meilisearch_types::tasks::{KindWithContent, Status};
use roaring::RoaringBitmap;

use crate::insta_snapshot::snapshot_index_scheduler;
use crate::test_utils::Breakpoint::*;
use crate::test_utils::{index_creation_task, replace_document_import_task};
use crate::{IndexScheduler, Query};

#[test]
fn register() {
    // In this test, the handle doesn't make any progress, we only check that the tasks are registered
    let (index_scheduler, mut _handle) = IndexScheduler::test(true, vec![]);

    let kinds = [
        index_creation_task("catto", "mouse"),
        replace_document_import_task("catto", None, 0, 12),
        replace_document_import_task("catto", None, 1, 50),
        replace_document_import_task("doggo", Some("bone"), 2, 5000),
    ];
    let (_, file) = index_scheduler.queue.create_update_file_with_uuid(0).unwrap();
    file.persist().unwrap();
    let (_, file) = index_scheduler.queue.create_update_file_with_uuid(1).unwrap();
    file.persist().unwrap();
    let (_, file) = index_scheduler.queue.create_update_file_with_uuid(2).unwrap();
    file.persist().unwrap();

    for (idx, kind) in kinds.into_iter().enumerate() {
        let k = kind.as_kind();
        let task = index_scheduler.register(kind, None, false).unwrap();
        index_scheduler.assert_internally_consistent();

        assert_eq!(task.uid, idx as u32);
        assert_eq!(task.status, Status::Enqueued);
        assert_eq!(task.kind.as_kind(), k);
    }

    snapshot!(snapshot_index_scheduler(&index_scheduler), name: "everything_is_successfully_registered");
}

#[test]
fn dry_run() {
    let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]);

    let kind = KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None };
    let task = index_scheduler.register(kind, None, true).unwrap();
    snapshot!(task.uid, @"0");
    snapshot!(snapshot_index_scheduler(&index_scheduler), @r"
        ### Autobatching Enabled = true
        ### Processing batch None:
        []
        ----------------------------------------------------------------------
        ### All Tasks:
        ----------------------------------------------------------------------
        ### Status:
        ----------------------------------------------------------------------
        ### Kind:
        ----------------------------------------------------------------------
        ### Index Tasks:
        ----------------------------------------------------------------------
        ### Index Mapper:

        ----------------------------------------------------------------------
        ### Canceled By:

        ----------------------------------------------------------------------
        ### Enqueued At:
        ----------------------------------------------------------------------
        ### Started At:
        ----------------------------------------------------------------------
        ### Finished At:
        ----------------------------------------------------------------------
        ### All Batches:
        ----------------------------------------------------------------------
        ### Batch to tasks mapping:
        ----------------------------------------------------------------------
        ### Batches Status:
        ----------------------------------------------------------------------
        ### Batches Kind:
        ----------------------------------------------------------------------
        ### Batches Index Tasks:
        ----------------------------------------------------------------------
        ### Batches Enqueued At:
        ----------------------------------------------------------------------
        ### Batches Started At:
        ----------------------------------------------------------------------
        ### Batches Finished At:
        ----------------------------------------------------------------------
        ### File Store:

        ----------------------------------------------------------------------
        ");

    let kind = KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None };
    let task = index_scheduler.register(kind, Some(12), true).unwrap();
    snapshot!(task.uid, @"12");
    snapshot!(snapshot_index_scheduler(&index_scheduler), @r"
        ### Autobatching Enabled = true
        ### Processing batch None:
        []
        ----------------------------------------------------------------------
        ### All Tasks:
        ----------------------------------------------------------------------
        ### Status:
        ----------------------------------------------------------------------
        ### Kind:
        ----------------------------------------------------------------------
        ### Index Tasks:
        ----------------------------------------------------------------------
        ### Index Mapper:

        ----------------------------------------------------------------------
        ### Canceled By:

        ----------------------------------------------------------------------
        ### Enqueued At:
        ----------------------------------------------------------------------
        ### Started At:
        ----------------------------------------------------------------------
        ### Finished At:
        ----------------------------------------------------------------------
        ### All Batches:
        ----------------------------------------------------------------------
        ### Batch to tasks mapping:
        ----------------------------------------------------------------------
        ### Batches Status:
        ----------------------------------------------------------------------
        ### Batches Kind:
        ----------------------------------------------------------------------
        ### Batches Index Tasks:
        ----------------------------------------------------------------------
        ### Batches Enqueued At:
        ----------------------------------------------------------------------
        ### Batches Started At:
        ----------------------------------------------------------------------
        ### Batches Finished At:
        ----------------------------------------------------------------------
        ### File Store:

        ----------------------------------------------------------------------
        ");
}

#[test]
fn basic_set_taskid() {
    let (index_scheduler, _handle) = IndexScheduler::test(true, vec![]);

    let kind = KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None };
    let task = index_scheduler.register(kind, None, false).unwrap();
    snapshot!(task.uid, @"0");

    let kind = KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None };
    let task = index_scheduler.register(kind, Some(12), false).unwrap();
    snapshot!(task.uid, @"12");

    let kind = KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None };
    let error = index_scheduler.register(kind, Some(5), false).unwrap_err();
    snapshot!(error, @"Received bad task id: 5 should be >= to 13.");
}

#[test]
fn test_disable_auto_deletion_of_tasks() {
    let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| {
        config.cleanup_enabled = false;
        config.max_number_of_tasks = 2;
    });

    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_successful_batch();

    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_failed_batch();

    // at this point the max number of tasks is reached
    // we can still enqueue multiple tasks
    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();

    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
    drop(rtxn);
    drop(proc);

    // now we're above the max number of tasks
    // and if we try to advance in the tick function no new task deletion should be enqueued
    handle.advance_till([Start, BatchCreated]);
    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_not_been_enqueued");
    drop(rtxn);
    drop(proc);
}

#[test]
fn test_auto_deletion_of_tasks() {
    let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| {
        config.max_number_of_tasks = 2;
    });

    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_successful_batch();

    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_failed_batch();

    // at this point the max number of tasks is reached
    // we can still enqueue multiple tasks
    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();

    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]" }), name: "task_queue_is_full");
    drop(rtxn);
    drop(proc);

    // now we're above the max number of tasks
    // and if we try to advance in the tick function a new task deletion should be enqueued
    handle.advance_till([Start, BatchCreated]);
    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_enqueued");
    drop(rtxn);
    drop(proc);

    handle.advance_till([InsideProcessBatch, ProcessBatchSucceeded, AfterProcessing]);
    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "task_deletion_have_been_processed");
    drop(rtxn);
    drop(proc);

    handle.advance_one_failed_batch();
    // a new task deletion has been enqueued
    handle.advance_one_successful_batch();
    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "after_the_second_task_deletion");
    drop(rtxn);
    drop(proc);

    handle.advance_one_failed_batch();
    handle.advance_one_successful_batch();
    let rtxn = index_scheduler.env.read_txn().unwrap();
    let proc = index_scheduler.processing_tasks.read().unwrap();
    let tasks =
        index_scheduler.queue.get_task_ids(&rtxn, &Query { ..Default::default() }, &proc).unwrap();
    let tasks = index_scheduler.queue.tasks.get_existing_tasks(&rtxn, tasks).unwrap();
    snapshot!(json_string!(tasks, { "[].enqueuedAt" => "[date]", "[].startedAt" => "[date]", "[].finishedAt" => "[date]", ".**.original_filter" => "[filter]", ".**.query" => "[query]" }), name: "everything_has_been_processed");
    drop(rtxn);
    drop(proc);
}

#[test]
fn test_task_queue_is_full() {
    let (index_scheduler, mut handle) = IndexScheduler::test_with_custom_config(vec![], |config| {
        // that's the minimum map size possible
        config.task_db_size = 1048576;
    });

    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_successful_batch();
    // on average this task takes ~600 bytes
    loop {
        let result = index_scheduler.register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        );
        if result.is_err() {
            break;
        }
        handle.advance_one_failed_batch();
    }
    index_scheduler.assert_internally_consistent();

    // at this point the task DB shoud have reached its limit and we should not be able to register new tasks
    let result = index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap_err();
    snapshot!(result, @"Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.");
    // we won't be able to test this error in an integration test thus as a best effort test I still ensure the error return the expected error code
    snapshot!(format!("{:?}", result.error_code()), @"NoSpaceLeftOnDevice");

    // Even the task deletion that doesn't delete anything shouldn't be accepted
    let result = index_scheduler
        .register(
            KindWithContent::TaskDeletion { query: S("test"), tasks: RoaringBitmap::new() },
            None,
            false,
        )
        .unwrap_err();
    snapshot!(result, @"Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.");
    // we won't be able to test this error in an integration test thus as a best effort test I still ensure the error return the expected error code
    snapshot!(format!("{:?}", result.error_code()), @"NoSpaceLeftOnDevice");

    // But a task deletion that delete something should works
    index_scheduler
        .register(
            KindWithContent::TaskDeletion { query: S("test"), tasks: (0..100).collect() },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_successful_batch();

    // Now we should be able to enqueue a few tasks again
    index_scheduler
        .register(
            KindWithContent::IndexCreation { index_uid: S("doggo"), primary_key: None },
            None,
            false,
        )
        .unwrap();
    handle.advance_one_failed_batch();
}