From 0353fbb5df04146bd9297a5886a351e1d502be6f Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Thu, 22 Jul 2021 16:23:36 +0200
Subject: [PATCH] Bump the tokenizer version to v0.2.4

---
 Cargo.lock       | 31 ++++++++++++++++++++++++++++---
 milli/Cargo.toml |  2 +-
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 18d42029f..cd1913ec8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -990,7 +990,7 @@ dependencies = [
  "jemallocator",
  "log",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
  "memmap",
  "milli",
  "once_cell",
@@ -1353,7 +1353,23 @@ dependencies = [
  "once_cell",
  "slice-group-by",
  "unicode-segmentation",
- "whatlang",
+ "whatlang 0.9.0",
+]
+
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.4"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.4#135d08dce465a756abaf6a1bcad70f315bda99b9"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang 0.12.0",
 ]
 
 [[package]]
@@ -1404,7 +1420,7 @@ dependencies = [
  "log",
  "logging_timer",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.4",
  "memmap",
  "obkv",
  "once_cell",
@@ -3087,6 +3103,15 @@ dependencies = [
  "hashbrown 0.7.2",
 ]
 
+[[package]]
+name = "whatlang"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703"
+dependencies = [
+ "hashbrown 0.7.2",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"
diff --git a/milli/Cargo.toml b/milli/Cargo.toml
index 6af928041..dc839a209 100644
--- a/milli/Cargo.toml
+++ b/milli/Cargo.toml
@@ -19,7 +19,7 @@ heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-fe
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.4" }
 memmap = "0.7.0"
 obkv = "0.2.0"
 once_cell = "1.5.2"