diff --git a/src/subcommand/infos.rs b/src/subcommand/infos.rs index f8138660b..54ce620cb 100644 --- a/src/subcommand/infos.rs +++ b/src/subcommand/infos.rs @@ -135,15 +135,16 @@ enum Command { word2: String, }, - /// Outputs the words FST to disk. + /// Outputs the words FST to standard output. /// /// One can use the FST binary helper to dissect and analyze it, /// you can install it using `cargo install fst-bin`. - ExportWordsFst { - /// The path where the FST will be written. - #[structopt(short, long, default_value = "words.fst")] - output: PathBuf, - }, + ExportWordsFst, + + /// Outputs the documents as JSON lines to the standard output. + /// + /// All of the fields are extracted, not just the displayed ones. + ExportDocuments, /// A command that patches the old external ids /// into the new external ids format. @@ -182,14 +183,15 @@ pub fn run(opt: Opt) -> anyhow::Result<()> { WordPairProximitiesDocids { full_display, word1, word2 } => { word_pair_proximities_docids(&index, &rtxn, !full_display, word1, word2) }, - ExportWordsFst { output } => export_words_fst(&index, &rtxn, output), + ExportWordsFst => export_words_fst(&index, &rtxn), + ExportDocuments => export_documents(&index, &rtxn), PatchToNewExternalIds => { drop(rtxn); let mut wtxn = index.write_txn()?; let result = patch_to_new_external_ids(&index, &mut wtxn); wtxn.commit()?; result - } + }, } } @@ -479,15 +481,34 @@ fn facet_stats(index: &Index, rtxn: &heed::RoTxn, field_name: String) -> anyhow: Ok(()) } -fn export_words_fst(index: &Index, rtxn: &heed::RoTxn, output: PathBuf) -> anyhow::Result<()> { - use std::fs::File; +fn export_words_fst(index: &Index, rtxn: &heed::RoTxn) -> anyhow::Result<()> { use std::io::Write as _; - let mut output = File::create(&output) - .with_context(|| format!("failed to create {} file", output.display()))?; - + let mut stdout = io::stdout(); let words_fst = index.words_fst(rtxn)?; - output.write_all(words_fst.as_fst().as_bytes())?; + stdout.write_all(words_fst.as_fst().as_bytes())?; + + Ok(()) +} + +fn export_documents(index: &Index, rtxn: &heed::RoTxn) -> anyhow::Result<()> { + use std::io::{BufWriter, Write as _}; + use crate::obkv_to_json; + + let stdout = io::stdout(); + let mut out = BufWriter::new(stdout); + + let fields_ids_map = index.fields_ids_map(rtxn)?; + let displayed_fields: Vec<_> = fields_ids_map.iter().map(|(id, _name)| id).collect(); + + for result in index.documents.iter(rtxn)? { + let (_id, obkv) = result?; + let document = obkv_to_json(&displayed_fields, &fields_ids_map, obkv)?; + serde_json::to_writer(&mut out, &document)?; + writeln!(&mut out)?; + } + + out.into_inner()?; Ok(()) }