snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-25 11:35:05 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

external changes

Browse Source
This commit is contained in:
Louis Dureuil 2024-10-14 15:41:31 +02:00
parent 6ad3f57bc1
commit c01ee7b732
No known key found for this signature in database
3 changed files with 27 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
milli/src/update/new/indexer/document_deletion.rs
View File

@ -5,9 +5,8 @@ use roaring::RoaringBitmap;
use super::document_changes::{DocumentChangeContext, DocumentChanges, MostlySend};
use crate::documents::PrimaryKey;
use crate::index::db_name::EXTERNAL_DOCUMENTS_IDS;
use crate::update::new::{Deletion, DocumentChange};
use crate::{DocumentId, InternalError, Result};
use crate::{DocumentId, Result};
#[derive(Default)]
pub struct DocumentDeletion {
@ -61,12 +60,15 @@ impl<'pl> DocumentChanges<'pl> for DocumentDeletionChanges<'pl> {
'pl: 'doc, // the payload must survive the process calls
{
let current = context.index.document(&context.txn, docid)?;
let new_fields_ids_map = context.new_fields_ids_map.borrow();
let new_fields_ids_map = new_fields_ids_map.local_map();
let external_document_id =
self.primary_key.document_id(current, new_fields_ids_map)?.map_err(|_| {
InternalError::DatabaseMissingEntry { db_name: EXTERNAL_DOCUMENTS_IDS, key: None }
})?;
let external_document_id = self.primary_key.extract_docid_from_db(
current,
&context.db_fields_ids_map,
&context.doc_alloc,
)?;
let external_document_id = external_document_id.to_bump(&context.doc_alloc);
Ok(DocumentChange::Deletion(Deletion::create(docid, external_document_id)))
}
}

33
milli/src/update/new/indexer/document_operation.rs
View File

@ -11,10 +11,10 @@ use super::document_changes::{DocumentChangeContext, DocumentChanges, MostlySend
use crate::documents::{DocumentIdExtractionError, PrimaryKey};
use crate::update::new::document::DocumentFromVersions;
use crate::update::new::document_change::Versions;
use crate::update::new::indexer::de::DocumentVisitor;
use crate::update::new::indexer::de::FieldAndDocidExtractor;
use crate::update::new::{Deletion, Insertion, Update};
use crate::update::{AvailableIds, IndexDocumentsMethod};
use crate::{DocumentId, Error, FieldsIdsMap, Index, Result, UserError};
use crate::{external_documents_ids, DocumentId, Error, FieldsIdsMap, Index, Result, UserError};
pub struct DocumentOperation<'pl> {
operations: Vec<Payload<'pl>>,
@ -98,7 +98,7 @@ impl<'pl> DocumentOperation<'pl> {
iter.next().transpose().map_err(UserError::SerdeJson)?
{
let res = document
.deserialize_map(DocumentVisitor::new(
.deserialize_map(FieldAndDocidExtractor::new(
new_fields_ids_map,
primary_key,
indexer,
@ -122,6 +122,8 @@ impl<'pl> DocumentOperation<'pl> {
}
}?;
let external_document_id = external_document_id.to_de();
let current_offset = iter.byte_offset();
let document_operation = InnerDocOp::Addition(DocumentOffset {
content: &payload[previous_offset..current_offset],
@ -310,23 +312,14 @@ impl MergeChanges for MergeDocumentForReplacement {
let document = DocumentFromVersions::new(Versions::Single(document));
if is_new {
Ok(DocumentChange::Insertion(Insertion::create(
docid,
external_doc.to_owned(),
document,
)))
Ok(DocumentChange::Insertion(Insertion::create(docid, external_doc, document)))
} else {
Ok(DocumentChange::Update(Update::create(
docid,
external_doc.to_owned(),
document,
true,
)))
Ok(DocumentChange::Update(Update::create(docid, external_doc, document, true)))
}
}
Some(InnerDocOp::Deletion) => {
let deletion = if is_new {
Deletion::create(docid, external_doc.to_owned())
Deletion::create(docid, external_doc)
} else {
todo!("Do that with Louis")
};
@ -373,7 +366,7 @@ impl MergeChanges for MergeDocumentForUpdates {
if operations.is_empty() {
let deletion = if !is_new {
Deletion::create(docid, external_docid.to_owned())
Deletion::create(docid, external_docid)
} else {
todo!("Do that with Louis")
};
@ -408,15 +401,11 @@ impl MergeChanges for MergeDocumentForUpdates {
let document = DocumentFromVersions::new(versions);
if is_new {
Ok(DocumentChange::Insertion(Insertion::create(
docid,
external_docid.to_owned(),
document,
)))
Ok(DocumentChange::Insertion(Insertion::create(docid, external_docid, document)))
} else {
Ok(DocumentChange::Update(Update::create(
docid,
external_docid.to_owned(),
external_docid,
document,
has_deletion,
)))

28
milli/src/update/new/indexer/partial_dump.rs
View File

@ -4,7 +4,7 @@ use rayon::iter::IndexedParallelIterator;
use serde::Deserializer;
use serde_json::value::RawValue;
use super::de::DocumentVisitor;
use super::de::FieldAndDocidExtractor;
use super::document_changes::{DocumentChangeContext, DocumentChanges, MostlySend};
use crate::documents::{DocumentIdExtractionError, PrimaryKey};
use crate::update::concurrent_available_ids::ConcurrentAvailableIds;
@ -66,36 +66,20 @@ where
let mut fields_ids_map = context.new_fields_ids_map.borrow_mut();
let fields_ids_map = fields_ids_map.deref_mut();
let res = document
.deserialize_map(DocumentVisitor::new(fields_ids_map, self.primary_key, doc_alloc))
.map_err(UserError::SerdeJson)?;
let external_document_id = match res {
Ok(document_id) => Ok(document_id),
Err(DocumentIdExtractionError::InvalidDocumentId(e)) => Err(e),
Err(DocumentIdExtractionError::MissingDocumentId) => {
Err(UserError::MissingDocumentId {
primary_key: self.primary_key.name().to_string(),
document: serde_json::from_str(document.get()).unwrap(),
})
}
Err(DocumentIdExtractionError::TooManyDocumentIds(_)) => {
Err(UserError::TooManyDocumentIds {
primary_key: self.primary_key.name().to_string(),
document: serde_json::from_str(document.get()).unwrap(),
})
}
}?;
let document = doc_alloc.alloc_str(document.get());
let document: &RawValue = unsafe { std::mem::transmute(document) };
let external_document_id =
self.primary_key.extract_fields_and_docid(document, fields_ids_map, doc_alloc)?;
let external_document_id = external_document_id.to_de();
let document = raw_collections::RawMap::from_raw_value(document, doc_alloc)
.map_err(InternalError::SerdeJson)?;
let document = document.into_bump_slice();
let document = DocumentFromVersions::new(Versions::Single(document));
let insertion = Insertion::create(docid, external_document_id.to_owned(), document);
let insertion = Insertion::create(docid, external_document_id, document);
Ok(DocumentChange::Insertion(insertion))
}
}