diff --git a/Cargo.lock b/Cargo.lock index f48920953..ced0c6de5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1696,6 +1696,22 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.1.0" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.1.0#32b2aee7e4a819478226eab5525232f09dd61bf5" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "meilisearch-types" version = "0.17.0" diff --git a/meilisearch-core/Cargo.toml b/meilisearch-core/Cargo.toml index ecfecfdc1..3f074587f 100644 --- a/meilisearch-core/Cargo.toml +++ b/meilisearch-core/Cargo.toml @@ -26,7 +26,7 @@ levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } log = "0.4.11" meilisearch-error = { path = "../meilisearch-error", version = "0.17.0" } meilisearch-schema = { path = "../meilisearch-schema", version = "0.17.0" } -meilisearch-tokenizer = { path = "../../Tokenizer" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.1.0" } meilisearch-types = { path = "../meilisearch-types", version = "0.17.0" } once_cell = "1.5.2" ordered-float = { version = "2.0.1", features = ["serde"] }