snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2024-11-26 20:15:07 +08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Read NDJson update file more efficiently

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-07-04 08:15:54 +02:00
parent 5572f0c2c8
commit c147724384
1 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
meilisearch-lib/src/document_formats.rs
View File

@ -1,6 +1,6 @@
use std::borrow::Borrow; use std::borrow::Borrow;
use std::fmt::{self, Debug, Display}; use std::fmt::{self, Debug, Display};
use std::io::{self, BufReader, Read, Seek, Write}; use std::io::{self, BufRead, BufReader, Read, Seek, Write};
use meilisearch_types::error::{Code, ErrorCode}; use meilisearch_types::error::{Code, ErrorCode};
use meilisearch_types::internal_error; use meilisearch_types::internal_error;
@ -98,16 +98,14 @@ pub fn read_csv(input: impl Read, writer: impl Write + Seek) -> Result<usize> {
/// Reads JSON Lines from input and write an obkv batch to writer. /// Reads JSON Lines from input and write an obkv batch to writer.
pub fn read_ndjson(input: impl Read, writer: impl Write + Seek) -> Result<usize> { pub fn read_ndjson(input: impl Read, writer: impl Write + Seek) -> Result<usize> {
let mut builder = DocumentsBatchBuilder::new(writer); let mut builder = DocumentsBatchBuilder::new(writer);
let reader = BufReader::new(input); let mut reader = BufReader::new(input);
let mut buf = String::with_capacity(1024);
for result in serde_json::Deserializer::from_reader(reader).into_iter() { while reader.read_line(&mut buf)? > 0 {
let object = result
.map_err(Error::Json)
.map_err(|e| (PayloadType::Ndjson, e))?;
builder builder
.append_json_object(&object) .append_unparsed_json_object(&buf)
.map_err(Into::into) .map_err(Into::into)
.map_err(DocumentFormatError::Internal)?; .map_err(DocumentFormatError::Internal)?;
buf.clear();
} }
let count = builder.documents_count(); let count = builder.documents_count();