From d43dc4824cf9aab5e122d1915944985d5ab8d871 Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 3 Feb 2021 17:44:20 +0100 Subject: [PATCH] implement list indexes --- src/data/mod.rs | 8 ++- .../local_index_controller/index_store.rs | 67 +++++++++++++++---- .../local_index_controller/mod.rs | 27 +++++++- src/index_controller/mod.rs | 14 ++++ src/index_controller/updates.rs | 7 ++ src/routes/index.rs | 23 ++++--- 6 files changed, 120 insertions(+), 26 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index de24d0a06..cada4f559 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use sha2::Digest; -use crate::index_controller::{IndexController, LocalIndexController}; -use crate::{option::Opt, index_controller::Settings}; +use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings}; +use crate::option::Opt; #[derive(Clone)] pub struct Data { @@ -114,6 +114,10 @@ impl Data { }) } + pub fn list_indexes(&self) -> anyhow::Result> { + self.index_controller.list_indexes() + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 483b6f5d6..138fe57fc 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -2,8 +2,10 @@ use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use chrono::{DateTime, Utc}; use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; +use log::error; use milli::Index; use rayon::ThreadPool; use serde::{Serialize, Deserialize}; @@ -16,10 +18,11 @@ use super::{UpdateMeta, UpdateResult}; type UpdateStore = super::update_store::UpdateStore; #[derive(Serialize, Deserialize, Debug, PartialEq)] -struct IndexMeta { - update_store_size: u64, - index_store_size: u64, - uuid: Uuid, +pub struct IndexMeta { + update_size: u64, + index_size: u64, + pub uuid: Uuid, + pub created_at: DateTime, } impl IndexMeta { @@ -168,7 +171,8 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() }; + let created_at = Utc::now(); + let meta = IndexMeta { update_size, index_size, uuid: uuid.clone(), created_at }; self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?; self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; @@ -186,6 +190,29 @@ impl IndexStore { Ok((index, update_store)) } + + /// Returns each index associated with it's metadata; + pub fn list_indexes(&self) -> anyhow::Result> { + let txn = self.env.read_txn()?; + let indexes = self.name_to_uuid_db + .iter(&txn)? + .filter_map(|entry| entry + .map_err(|e| { + error!("error decoding entry while listing indexes: {}", e); + e + }) + .ok()) + .map(|(name, uuid)| { + let meta = self.uuid_to_index_db + .get(&txn, &uuid) + .ok() + .flatten() + .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name)); + (name.to_owned(), meta) + }) + .collect(); + Ok(indexes) + } } fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { @@ -264,7 +291,12 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { + update_size: 4096 * 100, + index_size: 4096 * 100, + uuid: uuid.clone(), + created_at: Utc::now(), + }; let mut txn = store.env.write_txn().unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -286,7 +318,12 @@ mod test { assert!(store.index(&name).unwrap().is_none()); let uuid = Uuid::new_v4(); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { + update_size: 4096 * 100, + index_size: 4096 * 100, + uuid: uuid.clone(), + created_at: Utc::now(), + }; let mut txn = store.env.write_txn().unwrap(); store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); @@ -301,14 +338,18 @@ mod test { let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); let name = "foobar"; - store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); + let update_size = 4096 * 100; + let index_size = 4096 * 100; + store.get_or_create_index(&name, update_size, index_size).unwrap(); let txn = store.env.read_txn().unwrap(); let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; - assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_size, update_size); + assert_eq!(meta.index_size, index_size); + assert_eq!(meta.uuid, uuid); } #[test] @@ -326,8 +367,10 @@ mod test { assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() }; - assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap(); + assert_eq!(meta.update_size, update_size); + assert_eq!(meta.index_size, index_size); + assert_eq!(meta.uuid, uuid); } } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index b59eb2a99..6d6700639 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -13,7 +13,7 @@ use crate::option::IndexerOpts; use index_store::IndexStore; use super::IndexController; use super::updates::UpdateStatus; -use super::{UpdateMeta, UpdateResult}; +use super::{UpdateMeta, UpdateResult, IndexMetadata}; pub struct LocalIndexController { indexes: IndexStore, @@ -102,4 +102,29 @@ impl IndexController for LocalIndexController { } } + + fn list_indexes(&self) -> anyhow::Result> { + let metas = self.indexes.list_indexes()?; + let mut output_meta = Vec::new(); + for (name, meta) in metas { + let created_at = meta.created_at; + let uuid = meta.uuid; + let updated_at = self + .all_update_status(&name)? + .iter() + .filter_map(|u| u.processed().map(|u| u.processed_at)) + .max() + .unwrap_or(created_at); + + let index_meta = IndexMetadata { + name, + created_at, + updated_at, + uuid, + primary_key: None, + }; + output_meta.push(index_meta); + } + Ok(output_meta) + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d348ee876..38ff149bf 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -8,14 +8,26 @@ use std::num::NonZeroUsize; use std::sync::Arc; use anyhow::Result; +use chrono::{DateTime, Utc}; use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; +use uuid::Uuid; pub use updates::{Processed, Processing, Failed}; pub type UpdateStatus = updates::UpdateStatus; +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IndexMetadata { + pub name: String, + uuid: Uuid, + created_at: DateTime, + updated_at: DateTime, + pub primary_key: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { @@ -140,4 +152,6 @@ pub trait IndexController { fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; + + fn list_indexes(&self) -> anyhow::Result>; } diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index 4eb94dc4a..900987ba6 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -134,6 +134,13 @@ impl UpdateStatus { UpdateStatus::Failed(u) => u.id(), } } + + pub fn processed(&self) -> Option<&Processed> { + match self { + UpdateStatus::Processed(p) => Some(p), + _ => None, + } + } } impl From> for UpdateStatus { diff --git a/src/routes/index.rs b/src/routes/index.rs index 774039f2b..653d51475 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -19,19 +19,20 @@ pub fn services(cfg: &mut web::ServiceConfig) { .service(get_all_updates_status); } -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct IndexResponse { - pub name: String, - pub uid: String, - created_at: DateTime, - updated_at: DateTime, - pub primary_key: Option, -} #[get("/indexes", wrap = "Authentication::Private")] -async fn list_indexes(_data: web::Data) -> Result { - todo!() +async fn list_indexes(data: web::Data) -> Result { + match data.list_indexes() { + Ok(indexes) => { + let json = serde_json::to_string(&indexes).unwrap(); + Ok(HttpResponse::Ok().body(&json)) + } + Err(e) => { + error!("error listing indexes: {}", e); + unimplemented!() + } + } + } #[get("/indexes/{index_uid}", wrap = "Authentication::Private")]