From e2844f3a928d5a7cc0af77267329d3d006b5cdb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9mentine=20Urquizar?= <clementine@meilisearch.com>
Date: Mon, 21 Jun 2021 17:00:09 +0200
Subject: [PATCH] Update tokenizer version to v0.2.3

---
 Cargo.lock                  | 20 ++++++++++++++++++--
 meilisearch-http/Cargo.toml |  2 +-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 94a8fbcec..3b99965f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1626,7 +1626,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.3",
  "memmap",
  "milli",
  "mime",
@@ -1680,6 +1680,22 @@ dependencies = [
  "whatlang",
 ]
 
+[[package]]
+name = "meilisearch-tokenizer"
+version = "0.2.3"
+source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
+dependencies = [
+ "character_converter",
+ "cow-utils",
+ "deunicode",
+ "fst",
+ "jieba-rs",
+ "once_cell",
+ "slice-group-by",
+ "unicode-segmentation",
+ "whatlang",
+]
+
 [[package]]
 name = "memchr"
 version = "2.4.0"
@@ -1726,7 +1742,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer",
+ "meilisearch-tokenizer 0.2.2",
  "memmap",
  "obkv",
  "once_cell",
diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml
index efe78c70f..62625b2f1 100644
--- a/meilisearch-http/Cargo.toml
+++ b/meilisearch-http/Cargo.toml
@@ -49,7 +49,7 @@ itertools = "0.10.0"
 log = "0.4.8"
 main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.5.0" }
 mime = "0.3.16"