diff --git a/Cargo.lock b/Cargo.lock index 80a0117b4..ab7d9c23e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,14 @@ dependencies = [ "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "levenshtein_automata" +version = "0.1.0" +source = "git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst#5e8183a7634c4a0182ea7bb398140b2fe9854f77" +dependencies = [ + "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", +] + [[package]] name = "libc" version = "0.2.40" @@ -57,6 +65,7 @@ version = "0.1.0" dependencies = [ "bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", + "levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)", "serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -114,6 +123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bda13183df33055cbb84b847becce220d392df502ebe7a4a78d7021771ed94d0" "checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" "checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "" +"checksum levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)" = "" "checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" diff --git a/Cargo.toml b/Cargo.toml index 7bf4fda09..6afd53fd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,3 +11,8 @@ serde_derive = "1.0" [dependencies.fst] git = "https://github.com/Kerollmops/fst.git" branch = "op-builder-with-state" + +[dependencies.levenshtein_automata] +git = "https://github.com/Kerollmops/levenshtein-automata.git" +branch = "custom-fst" +features = ["fst_automaton"] diff --git a/raptor-http/Cargo.lock b/raptor-http/Cargo.lock index cbd8162c9..99f567e66 100644 --- a/raptor-http/Cargo.lock +++ b/raptor-http/Cargo.lock @@ -80,6 +80,7 @@ dependencies = [ [[package]] name = "fst" version = "0.3.0" +source = "git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state#6e0ab4e4ee5443cc55079996bf9f703086322c33" dependencies = [ "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -150,8 +151,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "levenshtein_automata" version = "0.1.0" +source = "git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst#5e8183a7634c4a0182ea7bb398140b2fe9854f77" dependencies = [ - "fst 0.3.0", + "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", ] [[package]] @@ -292,7 +294,8 @@ name = "raptor" version = "0.1.0" dependencies = [ "bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "fst 0.3.0", + "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", + "levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)", "serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -302,9 +305,8 @@ name = "raptor-http" version = "0.1.0" dependencies = [ "env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "fst 0.3.0", + "fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)", "futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", - "levenshtein_automata 0.1.0", "raptor 0.1.0", "serde 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", @@ -627,6 +629,7 @@ dependencies = [ "checksum crossbeam-epoch 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9b4e2817eb773f770dcb294127c011e22771899c21d18fce7dd739c0b9832e81" "checksum crossbeam-utils 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d636a8b3bcc1b409d7ffd3facef8f21dcb4009626adbd0c5e6c4305c07253c7b" "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f" +"checksum fst 0.3.0 (git+https://github.com/Kerollmops/fst.git?branch=op-builder-with-state)" = "" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "1a70b146671de62ec8c8ed572219ca5d594d9b06c0b364d5e67b722fc559b48c" @@ -636,6 +639,7 @@ dependencies = [ "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" +"checksum levenshtein_automata 0.1.0 (git+https://github.com/Kerollmops/levenshtein-automata.git?branch=custom-fst)" = "" "checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" diff --git a/raptor-http/Cargo.toml b/raptor-http/Cargo.toml index 0c1a63a68..0eab61c3d 100644 --- a/raptor-http/Cargo.toml +++ b/raptor-http/Cargo.toml @@ -14,11 +14,6 @@ tokio-proto = "0.1" tokio-service = "0.1" url = "1.7" -[dependencies.levenshtein_automata] -git = "https://github.com/Kerollmops/levenshtein-automata.git" -branch = "custom-fst" -features = ["fst"] - [dependencies.fst] git = "https://github.com/Kerollmops/fst.git" branch = "op-builder-with-state" diff --git a/raptor-http/src/main.rs b/raptor-http/src/main.rs index bef8ef7db..4db122789 100644 --- a/raptor-http/src/main.rs +++ b/raptor-http/src/main.rs @@ -1,7 +1,6 @@ extern crate env_logger; extern crate fst; extern crate futures; -extern crate levenshtein_automata; extern crate raptor; extern crate tokio_minihttp; extern crate tokio_proto; @@ -9,29 +8,22 @@ extern crate tokio_service; extern crate url; use std::{io, fs}; +use std::sync::Arc; use fst::Streamer; use futures::future; -use levenshtein_automata::LevenshteinAutomatonBuilder as LevBuilder; use tokio_minihttp::{Request, Response, Http}; use tokio_proto::TcpServer; use tokio_service::Service; -use raptor::{Map, OpWithStateBuilder}; +use raptor::{Map, OpWithStateBuilder, LevBuilder}; -static mut MAP: Option> = None; -static mut LEV_BUILDER_0: Option = None; -static mut LEV_BUILDER_1: Option = None; -static mut LEV_BUILDER_2: Option = None; - -struct MainService<'a> { - map: &'a Map, - lev_builder_0: &'a LevBuilder, - lev_builder_1: &'a LevBuilder, - lev_builder_2: &'a LevBuilder, +struct MainService { + map: Arc>, + lev_builder: Arc, } -impl<'a> Service for MainService<'a> { +impl Service for MainService { type Request = Request; type Response = Response; type Error = io::Error; @@ -52,13 +44,7 @@ impl<'a> Service for MainService<'a> { let mut automatons = Vec::new(); for query in query.split_whitespace() { - let lev = if query.len() <= 4 { - self.lev_builder_0.build_dfa(&query) - } else if query.len() <= 8 { - self.lev_builder_1.build_dfa(&query) - } else { - self.lev_builder_2.build_dfa(&query) - }; + let lev = self.lev_builder.build_automaton(query); automatons.push(lev); } @@ -100,27 +86,18 @@ impl<'a> Service for MainService<'a> { fn main() { drop(env_logger::init()); - // initialize all static variables - unsafe { - MAP = { - let map = fs::read("map.fst").unwrap(); - let values = fs::read("values.vecs").unwrap(); - - Some(Map::from_bytes(map, &values).unwrap()) - }; - LEV_BUILDER_0 = Some(LevBuilder::new(0, false)); - LEV_BUILDER_1 = Some(LevBuilder::new(1, false)); - LEV_BUILDER_2 = Some(LevBuilder::new(2, false)); - } - let addr = "0.0.0.0:8080".parse().unwrap(); - unsafe { - TcpServer::new(Http, addr).serve(|| Ok(MainService { - map: MAP.as_ref().unwrap(), - lev_builder_0: LEV_BUILDER_0.as_ref().unwrap(), - lev_builder_1: LEV_BUILDER_1.as_ref().unwrap(), - lev_builder_2: LEV_BUILDER_2.as_ref().unwrap(), - })) - } + let lev_builder = Arc::new(LevBuilder::new()); + let map = { + let fst = fs::read("map.fst").unwrap(); + let values = fs::read("values.vecs").unwrap(); + let map = Map::from_bytes(fst, &values).unwrap(); + Arc::new(map) + }; + + TcpServer::new(Http, addr).serve(move || Ok(MainService { + map: map.clone(), + lev_builder: lev_builder.clone(), + })) } diff --git a/src/levenshtein.rs b/src/levenshtein.rs new file mode 100644 index 000000000..0cc1334a7 --- /dev/null +++ b/src/levenshtein.rs @@ -0,0 +1,27 @@ +use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA}; + +pub struct LevBuilder { + automatons: [LevenshteinAutomatonBuilder; 3], +} + +impl LevBuilder { + pub fn new() -> Self { + Self { + automatons: [ + LevenshteinAutomatonBuilder::new(0, false), + LevenshteinAutomatonBuilder::new(1, false), + LevenshteinAutomatonBuilder::new(2, false), + ], + } + } + + pub fn build_automaton(&self, query: &str) -> DFA { + if query.len() <= 4 { + self.automatons[0].build_dfa(query) + } else if query.len() <= 8 { + self.automatons[1].build_dfa(query) + } else { + self.automatons[2].build_dfa(query) + } + } +} diff --git a/src/lib.rs b/src/lib.rs index b7d15bd6b..cfcaba1cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,16 @@ #[macro_use] extern crate serde_derive; extern crate bincode; extern crate fst; +extern crate levenshtein_automata; extern crate serde; pub mod map; +mod levenshtein; pub use self::map::{Map, MapBuilder, Values}; pub use self::map::{ OpBuilder, IndexedValues, OpWithStateBuilder, IndexedValuesWithState, }; + +pub use self::levenshtein::LevBuilder;