Merge #1578

1578: Update tokenizer version to v0.2.4 r=ManyTheFish a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Cargo.lock

@@ -1658,7 +1658,7 @@ dependencies = [
  "log",
  "main_error",
  "meilisearch-error",
- "meilisearch-tokenizer 0.2.3",
+ "meilisearch-tokenizer",
  "memmap",
  "milli",
  "mime",
@@ -1697,22 +1697,6 @@ dependencies = [
  "zip",
 ]
 
-[[package]]
-name = "meilisearch-tokenizer"
-version = "0.2.3"
-source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
-dependencies = [
- "character_converter",
- "cow-utils",
- "deunicode",
- "fst",
- "jieba-rs",
- "once_cell",
- "slice-group-by",
- "unicode-segmentation",
- "whatlang 0.9.0",
-]
-
 [[package]]
 name = "meilisearch-tokenizer"
 version = "0.2.4"
@@ -1726,7 +1710,7 @@ dependencies = [
  "once_cell",
  "slice-group-by",
  "unicode-segmentation",
- "whatlang 0.12.0",
+ "whatlang",
 ]
 
 [[package]]
@@ -1776,7 +1760,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer 0.2.4",
+ "meilisearch-tokenizer",
  "memmap",
  "obkv",
  "once_cell",
@@ -3554,15 +3538,6 @@ dependencies = [
  "webpki",
 ]
 
-[[package]]
-name = "whatlang"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
-dependencies = [
- "hashbrown 0.7.2",
-]
-
 [[package]]
 name = "whatlang"
 version = "0.12.0"

meilisearch-http/Cargo.toml

@@ -49,7 +49,7 @@ itertools = "0.10.0"
 log = "0.4.8"
 main_error = "0.1.0"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.4" }
 memmap = "0.7.0"
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.9.0" }
 mime = "0.3.16"