Merge #145

145: Update tokenizer to v0.2.1 r=MarinPostma a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -1847,7 +1847,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.1",
  "memmap",
  "milli",
  "mime",
@@ -1894,6 +1894,22 @@ dependencies = [
  "whatlang",
 ]
 
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.1"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang",
+]
+
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -1942,7 +1958,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.1.1",
  "memmap",
  "num-traits",
  "obkv",

meilisearch-http/Cargo.toml

@@ -40,7 +40,7 @@ itertools = "0.10.0"
 log = "0.4.8"
 main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" }
 memmap = "0.7.0"
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" }
 mime = "0.3.16"