meilisearch/milli/Cargo.toml

[package]
name = "milli"
edition = "2018"
publish = false

version.workspace = true
authors.workspace = true
description.workspace = true
homepage.workspace = true
readme.workspace = true
# edition.workspace = true
license.workspace = true

[dependencies]
bimap = { version = "0.6.3", features = ["serde"] }
bincode = "1.3.3"
bstr = "1.4.0"
byteorder = "1.4.3"
charabia = { version = "0.7.2", default-features = false }
concat-arrays = "0.1.2"
crossbeam-channel = "0.5.8"
deserr = "0.5.0"
either = "1.8.1"
flatten-serde-json = { path = "../flatten-serde-json" }
fst = "0.4.7"
fxhash = "0.2.1"
geoutils = "0.5.1"
grenad = { version = "0.4.4", default-features = false, features = ["tempfile"] }
heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.5", default-features = false, features = ["lmdb", "sync-read-txn"] }
json-depth-checker = { path = "../json-depth-checker" }
levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
memmap2 = "0.5.10"
obkv = "0.2.0"
once_cell = "1.17.1"
ordered-float = "3.6.0"
rayon = "1.7.0"
roaring = "0.10.1"
rstar = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.160", features = ["derive"] }
serde_json = { version = "1.0.95", features = ["preserve_order"] }
slice-group-by = "0.3.0"
smallstr =  { version = "0.3.0", features = ["serde"] }
smallvec = "1.10.0"
smartstring = "1.0.1"
tempfile = "3.5.0"
thiserror = "1.0.40"
time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
uuid = { version = "1.3.1", features = ["v4"] }

filter-parser = { path = "../filter-parser" }

# documents words self-join
itertools = "0.10.5"

# logging
log = "0.4.17"
logging_timer = "1.1.0"
csv = "1.2.1"

[dev-dependencies]
mimalloc = { version = "0.1.29", default-features = false }
big_s = "1.0.2"
insta = "1.29.0"
maplit = "1.0.2"
md5 = "0.7.0"
rand = {version = "0.8.5", features = ["small_rng"] }

[target.'cfg(fuzzing)'.dev-dependencies]
fuzzcheck = "0.12.1"

[features]
all-tokenizations = [ "charabia/default" ]

# Use POSIX semaphores instead of SysV semaphores in LMDB
# For more information on this feature, see heed's Cargo.toml
lmdb-posix-sem = ["heed/posix-sem"]

# allow chinese specialized tokenization
chinese = ["charabia/chinese"]

# allow hebrew specialized tokenization
hebrew = ["charabia/hebrew"]

# allow japanese specialized tokenization
japanese = ["charabia/japanese"]
japanese-transliteration = ["charabia/japanese-transliteration"]

# allow korean specialized tokenization
korean = ["charabia/korean"]

# allow thai specialized tokenization
thai = ["charabia/thai"]

# allow greek specialized tokenization
greek = ["charabia/greek"]
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00			`[package]`
			`name = "milli"`
			`edition = "2018"`
Use the workspace inheritance feature of rust 1.64 2023-02-15 20:51:07 +08:00			`publish = false`

			`version.workspace = true`
			`authors.workspace = true`
			`description.workspace = true`
			`homepage.workspace = true`
			`readme.workspace = true`
			`# edition.workspace = true`
			`license.workspace = true`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
			`[dependencies]`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`bimap = { version = "0.6.3", features = ["serde"] }`
Implement documents format document reader transform remove update format support document sequences fix document transform clean transform improve error handling add documents! macro fix transform bug fix tests remove csv dependency Add comments on the transform process replace search cli fmt review edits fix http ui fix clippy warnings Revert "fix clippy warnings" This reverts commit a1ce3cd96e603633dbf43e9e0b12b2453c9c5620. fix review comments remove smallvec in transform loop review edits 2021-08-31 17:44:15 +08:00			`bincode = "1.3.3"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`bstr = "1.4.0"`
Upgrade the dependencies 2022-03-15 00:00:53 +08:00			`byteorder = "1.4.3"`
Update charabia in milli 2023-04-26 20:56:54 +08:00			`charabia = { version = "0.7.2", default-features = false }`
Use an u16 field id instead of one byte 2021-07-06 17:31:24 +08:00			`concat-arrays = "0.1.2"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`crossbeam-channel = "0.5.8"`
implements the csv delimiter without tests Co-authored-by: Maxi Barmetler <maxi.barmetler@gmail.com> 2023-02-16 20:12:18 +08:00			`deserr = "0.5.0"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`either = "1.8.1"`
Add Charabia in Cargo.toml 2022-06-02 21:46:44 +08:00			`flatten-serde-json = { path = "../flatten-serde-json" }`
Upgrade the dependencies 2022-03-15 00:00:53 +08:00			`fst = "0.4.7"`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00			`fxhash = "0.2.1"`
Upgrade all dependencies 2022-10-04 17:29:39 +08:00			`geoutils = "0.5.1"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`grenad = { version = "0.4.4", default-features = false, features = ["tempfile"] }`
feat: add Cargo feature for LMDB's POSIX semaphores 2023-01-20 01:08:38 +08:00			`heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.5", default-features = false, features = ["lmdb", "sync-read-txn"] }`
only flatten an object if it's nested 2022-04-12 17:22:36 +08:00			`json-depth-checker = { path = "../json-depth-checker" }`
Upgrade the dependencies 2022-03-15 00:00:53 +08:00			`levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`memmap2 = "0.5.10"`
Use an u16 field id instead of one byte 2021-07-06 17:31:24 +08:00			`obkv = "0.2.0"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`once_cell = "1.17.1"`
			`ordered-float = "3.6.0"`
			`rayon = "1.7.0"`
Upgrade all dependencies 2022-10-04 17:29:39 +08:00			`roaring = "0.10.1"`
Upgrade the incompatible versions of the dependencies 2023-04-11 21:55:23 +08:00			`rstar = { version = "0.10.0", features = ["serde"] }`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`serde = { version = "1.0.160", features = ["derive"] }`
			`serde_json = { version = "1.0.95", features = ["preserve_order"] }`
Upgrade the dependencies 2022-03-15 00:00:53 +08:00			`slice-group-by = "0.3.0"`
			`smallstr = { version = "0.3.0", features = ["serde"] }`
Upgrade all dependencies 2022-10-04 17:29:39 +08:00			`smallvec = "1.10.0"`
Use smartstring to store the external id in our hashmap We need to store all the external id (primary key) in a hashmap associated to their internal id during. The smartstring remove heap allocation / memory usage and should improve the cache locality. 2022-04-11 21:43:18 +08:00			`smartstring = "1.0.1"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`tempfile = "3.5.0"`
			`thiserror = "1.0.40"`
			`time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }`
			`uuid = { version = "1.3.1", features = ["v4"] }`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
Rename the filter_parser crate into filter-parser Co-authored-by: Clément Renault <clement@meilisearch.com> 2021-11-09 23:16:28 +08:00			`filter-parser = { path = "../filter-parser" }`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
			`# documents words self-join`
Upgrade all dependencies 2022-10-04 17:29:39 +08:00			`itertools = "0.10.5"`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
			`# logging`
Upgrade all dependencies 2022-10-04 17:29:39 +08:00			`log = "0.4.17"`
Upgrade the dependencies 2022-03-15 00:00:53 +08:00			`logging_timer = "1.1.0"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`csv = "1.2.1"`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
			`[dev-dependencies]`
Use MiMalloc in milli tests 2023-03-20 16:41:37 +08:00			`mimalloc = { version = "0.1.29", default-features = false }`
Introduce the Attribute criterion 2021-03-11 18:48:55 +08:00			`big_s = "1.0.2"`
Upgrade the compatible versions of the dependencies 2023-04-11 21:23:51 +08:00			`insta = "1.29.0"`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00			`maplit = "1.0.2"`
Add index snapshot test helper function 2022-08-03 14:45:26 +08:00			`md5 = "0.7.0"`
Add range search and incremental indexing algorithm 2022-08-30 21:22:39 +08:00			`rand = {version = "0.8.5", features = ["small_rng"] }`
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00
Fix bugs in incremental facet indexing with variable parameters e.g. add one facet value incrementally with a group_size = X and then add another one with group_size = Y It is not actually possible to do so with the public API of milli, but I wanted to make sure the algorithm worked well in those cases anyway. The bugs were found by fuzzing the code with fuzzcheck, which I've added to milli as a conditional dev-dependency. But it can be removed later. 2022-09-08 17:53:01 +08:00			`[target.'cfg(fuzzing)'.dev-dependencies]`
Depend on released version of fuzzcheck from crates.io 2022-10-26 20:06:59 +08:00			`fuzzcheck = "0.12.1"`
Fix bugs in incremental facet indexing with variable parameters e.g. add one facet value incrementally with a group_size = X and then add another one with group_size = Y It is not actually possible to do so with the public API of milli, but I wanted to make sure the algorithm worked well in those cases anyway. The bugs were found by fuzzing the code with fuzzcheck, which I've added to milli as a conditional dev-dependency. But it can be removed later. 2022-09-08 17:53:01 +08:00
Change the project to become a workspace with milli as a default-member 2021-02-12 23:15:09 +08:00			`[features]`
Allow to disable specialized tokenizations (again) In PR #2773, I added the `chinese`, `hebrew`, `japanese` and `thai` feature flags to allow melisearch to be built without huge specialed tokenizations that took up 90% of the melisearch binary size. Unfortunately, due to some recent changes, this doesn't work anymore. The problem lies in excessive use of the `default` feature flag, which infects the dependency graph. Instead of adding `default-features = false` here and there, it's easier and more future-proof to not declare `default` in `milli` and `meilisearch-types`. I've renamed it to `all-tokenizers`, which also makes it a bit clearer what it's about. 2023-04-24 06:26:08 +08:00			`all-tokenizations = [ "charabia/default" ]`
Make charabia languages configurable 2022-09-08 18:19:44 +08:00
feat: add Cargo feature for LMDB's POSIX semaphores 2023-01-20 01:08:38 +08:00			`# Use POSIX semaphores instead of SysV semaphores in LMDB`
			`# For more information on this feature, see heed's Cargo.toml`
			`lmdb-posix-sem = ["heed/posix-sem"]`

Make charabia languages configurable 2022-09-08 18:19:44 +08:00			`# allow chinese specialized tokenization`
			`chinese = ["charabia/chinese"]`

			`# allow hebrew specialized tokenization`
			`hebrew = ["charabia/hebrew"]`

			`# allow japanese specialized tokenization`
			`japanese = ["charabia/japanese"]`
Update charabia 2022-12-12 21:53:08 +08:00			`japanese-transliteration = ["charabia/japanese-transliteration"]`

			`# allow korean specialized tokenization`
			`korean = ["charabia/korean"]`
Make charabia languages configurable 2022-09-08 18:19:44 +08:00
			`# allow thai specialized tokenization`
			`thai = ["charabia/thai"]`
Update charabia in milli 2023-04-26 20:56:54 +08:00
			`# allow greek specialized tokenization`
Update feature flags 2023-04-26 20:58:32 +08:00			`greek = ["charabia/greek"]`