From c25f7e345006916145b0c9d59c48fa9ceb6b64e3 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 15 Jul 2024 15:05:56 +0200 Subject: [PATCH 1/6] When `retrieveVectors` is true, retrieve `_vectors.embedder` even if there are no vector for that embedder --- milli/src/index.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/milli/src/index.rs b/milli/src/index.rs index 0a7a20ce0..a9b118827 100644 --- a/milli/src/index.rs +++ b/milli/src/index.rs @@ -1660,9 +1660,7 @@ impl Index { } } - if !embeddings.is_empty() { - res.insert(embedder_name.to_owned(), embeddings); - } + res.insert(embedder_name.to_owned(), embeddings); } Ok(res) } From 0e0e29459c5586eac432e1575622ba8c3c43f4a2 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 27 Aug 2024 16:27:05 +0200 Subject: [PATCH 2/6] Update time --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c728f348..51df0ea7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5098,9 +5098,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -5121,9 +5121,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", From e669af1e4975fa10c0a6d9a4289b1d49333121fd Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 8 Jul 2024 11:04:11 +0200 Subject: [PATCH 3/6] CI: Add ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION workaround to keep using Ubuntu 18.04 --- .github/workflows/flaky-tests.yml | 2 ++ .github/workflows/fuzzer-indexing.yml | 3 ++- .github/workflows/publish-apt-brew-pkg.yml | 2 ++ .github/workflows/publish-binaries.yml | 4 ++++ .github/workflows/test-suite.yml | 8 ++++++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.yml index c7e81aacc..dda1a86dc 100644 --- a/.github/workflows/flaky-tests.yml +++ b/.github/workflows/flaky-tests.yml @@ -1,4 +1,6 @@ name: Look for flaky tests +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true on: workflow_dispatch: schedule: diff --git a/.github/workflows/fuzzer-indexing.yml b/.github/workflows/fuzzer-indexing.yml index 1d01a6ea5..f3cc5af37 100644 --- a/.github/workflows/fuzzer-indexing.yml +++ b/.github/workflows/fuzzer-indexing.yml @@ -1,5 +1,6 @@ name: Run the indexing fuzzer - +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true on: push: branches: diff --git a/.github/workflows/publish-apt-brew-pkg.yml b/.github/workflows/publish-apt-brew-pkg.yml index 11893bae0..e99e196f2 100644 --- a/.github/workflows/publish-apt-brew-pkg.yml +++ b/.github/workflows/publish-apt-brew-pkg.yml @@ -15,6 +15,8 @@ jobs: debian: name: Publish debian packagge + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest needs: check-version container: diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 2372ce497..4480d4918 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -35,6 +35,8 @@ jobs: publish-linux: name: Publish binary for Linux runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true needs: check-version container: # Use ubuntu-18.04 to compile with glibc 2.27 @@ -132,6 +134,8 @@ jobs: name: Publish binary for aarch64 runs-on: ubuntu-latest needs: check-version + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true container: # Use ubuntu-18.04 to compile with glibc 2.27 image: ubuntu:18.04 diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 84a82250e..0b697f584 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -21,6 +21,8 @@ jobs: test-linux: name: Tests on ubuntu-18.04 runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true container: # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations image: ubuntu:18.04 @@ -77,6 +79,8 @@ jobs: test-all-features: name: Tests almost all features runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true container: # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations image: ubuntu:18.04 @@ -100,6 +104,8 @@ jobs: test-disabled-tokenization: name: Test disabled tokenization + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: image: ubuntu:18.04 @@ -127,6 +133,8 @@ jobs: # We run tests in debug also, to make sure that the debug_assertions are hit test-debug: name: Run tests in debug + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true runs-on: ubuntu-latest container: # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations From 5dac8e71681b9a9979904903943fd73a201df67a Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 27 Aug 2024 16:43:44 +0200 Subject: [PATCH 4/6] Allow fuzzing cfg --- milli/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/milli/Cargo.toml b/milli/Cargo.toml index fd7bde99b..b07c05d10 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -141,3 +141,6 @@ swedish-recomposition = ["charabia/swedish-recomposition"] # allow CUDA support, see cuda = ["candle-core/cuda"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } From 6287f5b2043978d991cb487ace874ae9151857ee Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 27 Aug 2024 16:54:33 +0200 Subject: [PATCH 5/6] Remove unexecuted test --- milli/src/update/index_documents/mod.rs | 27 ------------------------- 1 file changed, 27 deletions(-) diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs index 089b56025..c2c49be17 100644 --- a/milli/src/update/index_documents/mod.rs +++ b/milli/src/update/index_documents/mod.rs @@ -2180,33 +2180,6 @@ mod tests { index.add_documents(doc1).unwrap(); } - #[cfg(feature = "default")] - #[test] - fn store_detected_script_and_language_per_document_during_indexing() { - use charabia::{Language, Script}; - let index = TempIndex::new(); - index - .add_documents(documents!([ - { "id": 1, "title": "The quick (\"brown\") fox can't jump 32.3 feet, right? Brr, it's 29.3°F!" }, - { "id": 2, "title": "人人生而自由﹐在尊嚴和權利上一律平等。他們賦有理性和良心﹐並應以兄弟關係的精神互相對待。" }, - { "id": 3, "title": "הַשּׁוּעָל הַמָּהִיר (״הַחוּם״) לֹא יָכוֹל לִקְפֹּץ 9.94 מֶטְרִים, נָכוֹן? ברר, 1.5°C- בַּחוּץ!" }, - { "id": 4, "title": "関西国際空港限定トートバッグ すもももももももものうち" }, - { "id": 5, "title": "ภาษาไทยง่ายนิดเดียว" }, - { "id": 6, "title": "The quick 在尊嚴和權利上一律平等。" }, - ])) - .unwrap(); - - let rtxn = index.read_txn().unwrap(); - let key_jpn = (Script::Cj, Language::Jpn); - let key_cmn = (Script::Cj, Language::Cmn); - let cj_jpn_docs = index.script_language_documents_ids(&rtxn, &key_jpn).unwrap().unwrap(); - let cj_cmn_docs = index.script_language_documents_ids(&rtxn, &key_cmn).unwrap().unwrap(); - let expected_cj_jpn_docids = [3].iter().collect(); - assert_eq!(cj_jpn_docs, expected_cj_jpn_docids); - let expected_cj_cmn_docids = [1, 5].iter().collect(); - assert_eq!(cj_cmn_docs, expected_cj_cmn_docids); - } - #[test] fn add_and_delete_documents_in_single_transform() { let mut index = TempIndex::new(); From 575b7b7a0b7554ef9df3b8aec18e2330baa8a5a4 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 27 Aug 2024 17:14:10 +0200 Subject: [PATCH 6/6] Fix tests --- meilisearch/tests/documents/get_documents.rs | 14 +++++++++++-- meilisearch/tests/vector/mod.rs | 21 +++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/meilisearch/tests/documents/get_documents.rs b/meilisearch/tests/documents/get_documents.rs index efe4cf8e9..ccf31e764 100644 --- a/meilisearch/tests/documents/get_documents.rs +++ b/meilisearch/tests/documents/get_documents.rs @@ -644,7 +644,12 @@ async fn get_document_with_vectors() { { "id": 1, "name": "echo", - "_vectors": {} + "_vectors": { + "manual": { + "embeddings": [], + "regenerate": false + } + } } ], "offset": 0, @@ -700,7 +705,12 @@ async fn get_document_with_vectors() { }, { "name": "echo", - "_vectors": {} + "_vectors": { + "manual": { + "embeddings": [], + "regenerate": false + } + } } ], "offset": 0, diff --git a/meilisearch/tests/vector/mod.rs b/meilisearch/tests/vector/mod.rs index 0343ab785..44e815585 100644 --- a/meilisearch/tests/vector/mod.rs +++ b/meilisearch/tests/vector/mod.rs @@ -119,7 +119,12 @@ async fn add_remove_user_provided() { { "id": 1, "name": "echo", - "_vectors": {} + "_vectors": { + "manual": { + "embeddings": [], + "regenerate": false + } + } } ], "offset": 0, @@ -141,7 +146,12 @@ async fn add_remove_user_provided() { { "id": 1, "name": "echo", - "_vectors": {} + "_vectors": { + "manual": { + "embeddings": [], + "regenerate": false + } + } } ], "offset": 0, @@ -577,7 +587,12 @@ async fn add_remove_one_vector_4588() { { "id": 0, "name": "kefir", - "_vectors": {} + "_vectors": { + "manual": { + "embeddings": [], + "regenerate": false + } + } } ], "offset": 0,