diff --git a/Cargo.lock b/Cargo.lock index 94a8fbcec..3b99965f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1626,7 +1626,7 @@ dependencies = [ "log", "main_error", "meilisearch-error", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.3", "memmap", "milli", "mime", @@ -1680,6 +1680,22 @@ dependencies = [ "whatlang", ] +[[package]] +name = "meilisearch-tokenizer" +version = "0.2.3" +source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781" +dependencies = [ + "character_converter", + "cow-utils", + "deunicode", + "fst", + "jieba-rs", + "once_cell", + "slice-group-by", + "unicode-segmentation", + "whatlang", +] + [[package]] name = "memchr" version = "2.4.0" @@ -1726,7 +1742,7 @@ dependencies = [ "linked-hash-map", "log", "logging_timer", - "meilisearch-tokenizer", + "meilisearch-tokenizer 0.2.2", "memmap", "obkv", "once_cell", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index efe78c70f..62625b2f1 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -49,7 +49,7 @@ itertools = "0.10.0" log = "0.4.8" main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } -meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" } +meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" } memmap = "0.7.0" milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" } mime = "0.3.16"