From b0717b75d94f6f10769edafdb35b81fe8dd7b5d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= <clementine@meilisearch.com>
Date: Wed, 14 Apr 2021 19:06:18 +0200
Subject: [PATCH] Update tokenizer to v0.2.1

---
 Cargo.lock                  | 20 ++++++++++++++++++--
 meilisearch-http/Cargo.toml |  2 +-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7ef514ffe..e2a98faed 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1847,7 +1847,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.1",
  "memmap",
  "milli",
  "mime",
@@ -1894,6 +1894,22 @@ dependencies = [
  "whatlang",
 ]
 
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.1"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang",
+]
+
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -1942,7 +1958,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.1.1",
  "memmap",
  "num-traits",
  "obkv",
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index 9ab386882..efaafd077 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -40,7 +40,7 @@ itertools = "0.10.0"
 log = "0.4.8"
 main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" }
 memmap = "0.7.0"
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" }
 mime = "0.3.16"