Merge #230

230: Update Tokenizer version to v0.2.3 r=ManyTheFish a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -971,7 +971,7 @@ dependencies = [
  "heed",
  "log",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
  "memmap",
  "milli",
  "once_cell",
@@ -1332,6 +1332,22 @@ dependencies = [
  "whatlang",
 ]
 
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.3"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang",
+]
+
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -1380,7 +1396,7 @@ dependencies = [
  "log",
  "logging_timer",
  "maplit",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.2",
  "memmap",
  "obkv",
  "once_cell",

http-ui/Cargo.toml

@@ -11,7 +11,7 @@ byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
 crossbeam-channel = "0.5.0"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
 heed = "0.10.6"
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
 milli = { path = "../milli" }
 once_cell = "1.5.2"