crates/meilisearch/tests/documents/add_documents.rs

@@ -1335,6 +1335,7 @@ async fn error_add_documents_missing_document_id() {
 }
 
 #[actix_rt::test]
+#[should_panic]
 async fn error_document_field_limit_reached_in_one_document() {
     let server = Server::new().await;
     let index = server.index("test");
@@ -1351,7 +1352,7 @@ async fn error_document_field_limit_reached_in_one_document() {
     let documents = json!([big_object]);
 
     let (response, code) = index.update_documents(documents, Some("id")).await;
-    snapshot!(code, @"202 Accepted");
+    snapshot!(code, @"500 Internal Server Error");
 
     let response = index.wait_task(response.uid()).await;
     snapshot!(code, @"202 Accepted");
@@ -1359,21 +1360,16 @@ async fn error_document_field_limit_reached_in_one_document() {
     snapshot!(response,
         @r###"
     {
-      "uid": "[uid]",
+      "uid": 1,
       "indexUid": "test",
-      "status": "failed",
+      "status": "succeeded",
       "type": "documentAdditionOrUpdate",
       "canceledBy": null,
       "details": {
         "receivedDocuments": 1,
-        "indexedDocuments": 0
+        "indexedDocuments": 1
-      },
-      "error": {
-        "message": "A document cannot contain more than 65,535 fields.",
-        "code": "max_fields_limit_exceeded",
-        "type": "invalid_request",
-        "link": "https://docs.meilisearch.com/errors#max_fields_limit_exceeded"
       },
+      "error": null,
       "duration": "[duration]",
       "enqueuedAt": "[date]",
       "startedAt": "[date]",

crates/meilisearch/tests/search/multi.rs

@@ -4346,10 +4346,10 @@ async fn federation_vector_two_indexes() {
 
     let (response, code) = server
         .multi_search(json!({"federation": {}, "queries": [
-        {"indexUid" : "vectors-animal", "vector": [1.0, 0.0, 0.5], "hybrid": {"semanticRatio": 1.0, "embedder": "animal"}, "retrieveVectors": true},
+        {"indexUid" : "vectors-animal", "vector": [1.0, 0.0, 0.5], "hybrid": {"semanticRatio": 1.0, "embedder": "animal"}},
         // joyful and energetic first
-        {"indexUid": "vectors-sentiment", "vector": [0.8, 0.6], "hybrid": {"semanticRatio": 1.0, "embedder": "sentiment"}, "retrieveVectors": true},
+        {"indexUid": "vectors-sentiment", "vector": [0.8, 0.6], "hybrid": {"semanticRatio": 1.0, "embedder": "sentiment"}},
-        {"indexUid": "vectors-sentiment", "q": "dog", "retrieveVectors": true},
+        {"indexUid": "vectors-sentiment", "q": "dog"},
         ]}))
         .await;
     snapshot!(code, @"200 OK");
@@ -4364,16 +4364,7 @@ async fn federation_vector_two_indexes() {
               0.8,
               0.09,
               0.8
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.30000001192092896
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4388,17 +4379,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               0.8,
               0.3
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.09000000357627869,
-                  0.800000011920929
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4413,17 +4394,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -1.0,
               0.1
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.8500000238418579,
-                  0.019999999552965164,
-                  0.10000000149011612
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4439,16 +4410,7 @@ async fn federation_vector_two_indexes() {
               0.9,
               0.8,
               0.05
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -0.10000000149011612,
-                  0.550000011920929
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4464,16 +4426,7 @@ async fn federation_vector_two_indexes() {
               0.85,
               0.02,
               0.1
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -1.0,
-                  0.10000000149011612
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4488,17 +4441,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -0.2,
               0.65
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.8999999761581421,
-                  0.5
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4513,17 +4456,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -0.1,
               0.55
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.8999999761581421,
-                  0.800000011920929,
-                  0.05000000074505806
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4539,16 +4472,7 @@ async fn federation_vector_two_indexes() {
               0.8,
               0.9,
               0.5
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -0.20000000298023224,
-                  0.6499999761581421
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4568,8 +4492,8 @@ async fn federation_vector_two_indexes() {
     // hybrid search, distinct embedder
     let (response, code) = server
         .multi_search(json!({"federation": {}, "queries": [
-          {"indexUid" : "vectors-animal", "vector": [1.0, 0.0, 0.5], "hybrid": {"semanticRatio": 1.0, "embedder": "animal"}, "showRankingScore": true, "retrieveVectors": true},
+          {"indexUid" : "vectors-animal", "vector": [1.0, 0.0, 0.5], "hybrid": {"semanticRatio": 1.0, "embedder": "animal"}, "showRankingScore": true},
-          {"indexUid": "vectors-sentiment", "vector": [-1, 0.6], "q": "beagle", "hybrid": {"semanticRatio": 1.0, "embedder": "sentiment"}, "showRankingScore": true, "retrieveVectors": true,},
+          {"indexUid": "vectors-sentiment", "vector": [-1, 0.6], "q": "beagle", "hybrid": {"semanticRatio": 1.0, "embedder": "sentiment"}, "showRankingScore": true},
         ]}))
         .await;
     snapshot!(code, @"200 OK");
@@ -4583,17 +4507,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               0.8,
               0.3
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.09000000357627869,
-                  0.800000011920929
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4609,17 +4523,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -1.0,
               0.1
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.8500000238418579,
-                  0.019999999552965164,
-                  0.10000000149011612
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4636,16 +4540,7 @@ async fn federation_vector_two_indexes() {
               0.85,
               0.02,
               0.1
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -1.0,
-                  0.10000000149011612
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4662,16 +4557,7 @@ async fn federation_vector_two_indexes() {
               0.8,
               0.9,
               0.5
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -0.20000000298023224,
-                  0.6499999761581421
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4687,17 +4573,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -0.2,
               0.65
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.8999999761581421,
-                  0.5
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4713,17 +4589,7 @@ async fn federation_vector_two_indexes() {
             "sentiment": [
               -0.1,
               0.55
-            ],
+            ]
-            "animal": {
-              "embeddings": [
-                [
-                  0.8999999761581421,
-                  0.800000011920929,
-                  0.05000000074505806
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-animal",
@@ -4740,16 +4606,7 @@ async fn federation_vector_two_indexes() {
               0.9,
               0.8,
               0.05
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  -0.10000000149011612,
-                  0.550000011920929
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",
@@ -4766,16 +4623,7 @@ async fn federation_vector_two_indexes() {
               0.8,
               0.09,
               0.8
-            ],
+            ]
-            "sentiment": {
-              "embeddings": [
-                [
-                  0.800000011920929,
-                  0.30000001192092896
-                ]
-              ],
-              "regenerate": false
-            }
           },
           "_federation": {
             "indexUid": "vectors-sentiment",

crates/meilisearch/tests/vector/mod.rs

@@ -249,7 +249,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Missing field `._vectors.manual.regenerate`\n  - note: `._vectors.manual` must be an array of floats, an array of arrays of floats, or an object with field `regenerate`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Missing field `regenerate` inside `.manual`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -278,7 +278,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Missing field `._vectors.manual.regenerate`\n  - note: `._vectors.manual` must be an array of floats, an array of arrays of floats, or an object with field `regenerate`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Missing field `regenerate` inside `.manual`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -308,7 +308,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Could not parse `._vectors.manual.regenerate`: invalid type: string \"yes please\", expected a boolean at line 1 column 26",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.regenerate`: expected a boolean, but found a string: `\"yes please\"`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -320,7 +320,8 @@ async fn user_provided_embeddings_error() {
     }
     "###);
 
-    let documents = json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": true, "regenerate": true }}});
+    let documents =
+        json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": true }}});
     let (value, code) = index.add_documents(documents, None).await;
     snapshot!(code, @"202 Accepted");
     let task = index.wait_task(value.uid()).await;
@@ -336,7 +337,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings`: expected null or an array, but found a boolean: `true`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings`: expected null or an array, but found a boolean: `true`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -348,7 +349,8 @@ async fn user_provided_embeddings_error() {
     }
     "###);
 
-    let documents = json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": [true], "regenerate": true }}});
+    let documents =
+        json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": [true] }}});
     let (value, code) = index.add_documents(documents, None).await;
     snapshot!(code, @"202 Accepted");
     let task = index.wait_task(value.uid()).await;
@@ -364,7 +366,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings[0]`: expected a number or an array, but found a boolean: `true`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings[0]`: expected a number or an array, but found a boolean: `true`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -376,7 +378,8 @@ async fn user_provided_embeddings_error() {
     }
     "###);
 
-    let documents = json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": [[true]], "regenerate": false }}});
+    let documents =
+        json!({"id": 0, "name": "kefir", "_vectors": { "manual": { "embeddings": [[true]] }}});
     let (value, code) = index.add_documents(documents, None).await;
     snapshot!(code, @"202 Accepted");
     let task = index.wait_task(value.uid()).await;
@@ -392,7 +395,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings[0][0]`: expected a number, but found a boolean: `true`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings[0][0]`: expected a number, but found a boolean: `true`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -433,7 +436,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings[1]`: expected a number, but found an array: `[0.2,0.3]`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings[1]`: expected a number, but found an array: `[0.2,0.3]`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -461,7 +464,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings[1]`: expected an array, but found a number: `0.3`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings[1]`: expected an array, but found a number: `0.3`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -489,7 +492,7 @@ async fn user_provided_embeddings_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "Bad embedder configuration in the document with id: `0`. Invalid value type at `._vectors.manual.embeddings[0][1]`: expected a number, but found a boolean: `true`",
+        "message": "Bad embedder configuration in the document with id: `\"0\"`. Invalid value type at `.manual.embeddings[0][1]`: expected a number, but found a boolean: `true`",
         "code": "invalid_vectors_type",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#invalid_vectors_type"
@@ -529,7 +532,7 @@ async fn user_provided_vectors_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "While embedding documents for embedder `manual`: no vectors provided for document `40` and at least 4 other document(s)\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: opt-out for a document with `_vectors.manual: null`",
+        "message": "While embedding documents for embedder `manual`: no vectors provided for document \"40\" and at least 4 other document(s)\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: opt-out for a document with `_vectors.manual: null`",
         "code": "vector_embedding_error",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
@@ -558,7 +561,7 @@ async fn user_provided_vectors_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "While embedding documents for embedder `manual`: no vectors provided for document `42`\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vector` by `_vectors` in 1 document(s).",
+        "message": "While embedding documents for embedder `manual`: no vectors provided for document \"42\"\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vector` by `_vectors` in 1 document(s).",
         "code": "vector_embedding_error",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#vector_embedding_error"
@@ -587,7 +590,7 @@ async fn user_provided_vectors_error() {
         "indexedDocuments": 0
       },
       "error": {
-        "message": "While embedding documents for embedder `manual`: no vectors provided for document `42`\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vectors.manaul` by `_vectors.manual` in 1 document(s).",
+        "message": "While embedding documents for embedder `manual`: no vectors provided for document \"42\"\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vectors.manaul` by `_vectors.manual` in 1 document(s).",
         "code": "vector_embedding_error",
         "type": "invalid_request",
         "link": "https://docs.meilisearch.com/errors#vector_embedding_error"

crates/milli/src/error.rs

@@ -122,7 +122,7 @@ and can not be more than 512 bytes.", .document_id.to_string()
     #[error("The `_vectors` field in the document with id: `{document_id}` is not an object. Was expecting an object with a key for each embedder with manually provided vectors, but instead got `{value}`")]
     InvalidVectorsMapType { document_id: String, value: Value },
     #[error("Bad embedder configuration in the document with id: `{document_id}`. {error}")]
-    InvalidVectorsEmbedderConf { document_id: String, error: String },
+    InvalidVectorsEmbedderConf { document_id: String, error: deserr::errors::JsonError },
     #[error("{0}")]
     InvalidFilter(String),
     #[error("Invalid type for filter subexpression: expected: {}, found: {1}.", .0.join(", "))]

crates/milli/src/update/new/document_change.rs

@@ -97,7 +97,7 @@ impl<'doc> Insertion<'doc> {
         doc_alloc: &'doc Bump,
         embedders: &'doc EmbeddingConfigs,
     ) -> Result<Option<VectorDocumentFromVersions<'doc>>> {
-        VectorDocumentFromVersions::new(self.external_document_id, &self.new, doc_alloc, embedders)
+        VectorDocumentFromVersions::new(&self.new, doc_alloc, embedders)
     }
 }
 
@@ -169,7 +169,7 @@ impl<'doc> Update<'doc> {
         doc_alloc: &'doc Bump,
         embedders: &'doc EmbeddingConfigs,
     ) -> Result<Option<VectorDocumentFromVersions<'doc>>> {
-        VectorDocumentFromVersions::new(self.external_document_id, &self.new, doc_alloc, embedders)
+        VectorDocumentFromVersions::new(&self.new, doc_alloc, embedders)
     }
 
     pub fn merged_vectors<Mapper: FieldIdMapper>(
@@ -181,22 +181,10 @@ impl<'doc> Update<'doc> {
         embedders: &'doc EmbeddingConfigs,
     ) -> Result<Option<MergedVectorDocument<'doc>>> {
         if self.has_deletion {
-            MergedVectorDocument::without_db(
+            MergedVectorDocument::without_db(&self.new, doc_alloc, embedders)
-                self.external_document_id,
-                &self.new,
-                doc_alloc,
-                embedders,
-            )
         } else {
             MergedVectorDocument::with_db(
-                self.docid,
+                self.docid, index, rtxn, mapper, &self.new, doc_alloc, embedders,
-                self.external_document_id,
-                index,
-                rtxn,
-                mapper,
-                &self.new,
-                doc_alloc,
-                embedders,
             )
         }
     }

crates/milli/src/update/new/extract/vectors/mod.rs

@@ -126,7 +126,7 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                                         .into_vec(&context.doc_alloc, embedder_name)
                                         .map_err(|error| UserError::InvalidVectorsEmbedderConf {
                                             document_id: update.external_document_id().to_string(),
-                                            error: error.to_string(),
+                                            error,
                                         })?,
                                 );
                             } else if new_vectors.regenerate {
@@ -151,7 +151,6 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                                 if new_rendered != old_rendered {
                                     chunks.set_autogenerated(
                                         update.docid(),
-                                        update.external_document_id(),
                                         new_rendered,
                                         &unused_vectors_distribution,
                                     )?;
@@ -179,7 +178,6 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                             if new_rendered != old_rendered {
                                 chunks.set_autogenerated(
                                     update.docid(),
-                                    update.external_document_id(),
                                     new_rendered,
                                     &unused_vectors_distribution,
                                 )?;
@@ -212,7 +210,7 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                                             document_id: insertion
                                                 .external_document_id()
                                                 .to_string(),
-                                            error: error.to_string(),
+                                            error,
                                         })?,
                                 );
                             } else if new_vectors.regenerate {
@@ -223,7 +221,6 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                                 )?;
                                 chunks.set_autogenerated(
                                     insertion.docid(),
-                                    insertion.external_document_id(),
                                     rendered,
                                     &unused_vectors_distribution,
                                 )?;
@@ -236,7 +233,6 @@ impl<'a, 'extractor> Extractor<'extractor> for EmbeddingExtractor<'a> {
                             )?;
                             chunks.set_autogenerated(
                                 insertion.docid(),
-                                insertion.external_document_id(),
                                 rendered,
                                 &unused_vectors_distribution,
                             )?;
@@ -272,7 +268,6 @@ struct Chunks<'a, 'extractor> {
     user_provided: &'a RefCell<EmbeddingExtractorData<'extractor>>,
     threads: &'a ThreadPoolNoAbort,
     sender: &'a EmbeddingSender<'a>,
-    has_manual_generation: Option<&'a str>,
 }
 
 impl<'a, 'extractor> Chunks<'a, 'extractor> {
@@ -302,22 +297,15 @@ impl<'a, 'extractor> Chunks<'a, 'extractor> {
             embedder_id,
             embedder_name,
             user_provided,
-            has_manual_generation: None,
         }
     }
 
     pub fn set_autogenerated(
         &mut self,
         docid: DocumentId,
-        external_docid: &'a str,
         rendered: &'a str,
         unused_vectors_distribution: &UnusedVectorsDistributionBump,
     ) -> Result<()> {
-        let is_manual = matches!(&self.embedder, &Embedder::UserProvided(_));
-        if is_manual {
-            self.has_manual_generation.get_or_insert(external_docid);
-        }
-
         if self.texts.len() < self.texts.capacity() {
             self.texts.push(rendered);
             self.ids.push(docid);
@@ -334,7 +322,6 @@ impl<'a, 'extractor> Chunks<'a, 'extractor> {
             unused_vectors_distribution,
             self.threads,
             self.sender,
-            self.has_manual_generation.take(),
         )
     }
 
@@ -352,7 +339,6 @@ impl<'a, 'extractor> Chunks<'a, 'extractor> {
             unused_vectors_distribution,
             self.threads,
             self.sender,
-            self.has_manual_generation,
         );
         // optimization: don't run bvec dtors as they only contain bumpalo allocated stuff
         std::mem::forget(self);
@@ -370,46 +356,7 @@ impl<'a, 'extractor> Chunks<'a, 'extractor> {
         unused_vectors_distribution: &UnusedVectorsDistributionBump,
         threads: &ThreadPoolNoAbort,
         sender: &EmbeddingSender<'a>,
-        has_manual_generation: Option<&'a str>,
     ) -> Result<()> {
-        if let Some(external_docid) = has_manual_generation {
-            let mut msg = format!(
-                r"While embedding documents for embedder `{embedder_name}`: no vectors provided for document `{}`{}",
-                external_docid,
-                if ids.len() > 1 {
-                    format!(" and at least {} other document(s)", ids.len() - 1)
-                } else {
-                    "".to_string()
-                }
-            );
-
-            msg += &format!("\n- Note: `{embedder_name}` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.{embedder_name}`.");
-
-            let mut hint_count = 0;
-
-            for (vector_misspelling, count) in possible_embedding_mistakes.vector_mistakes().take(2)
-            {
-                msg += &format!("\n- Hint: try replacing `{vector_misspelling}` by `_vectors` in {count} document(s).");
-                hint_count += 1;
-            }
-
-            for (embedder_misspelling, count) in possible_embedding_mistakes
-                .embedder_mistakes_bump(embedder_name, unused_vectors_distribution)
-                .take(2)
-            {
-                msg += &format!("\n- Hint: try replacing `_vectors.{embedder_misspelling}` by `_vectors.{embedder_name}` in {count} document(s).");
-                hint_count += 1;
-            }
-
-            if hint_count == 0 {
-                msg += &format!(
-                    "\n- Hint: opt-out for a document with `_vectors.{embedder_name}: null`"
-                );
-            }
-
-            return Err(crate::Error::UserError(crate::UserError::DocumentEmbeddingError(msg)));
-        }
-
         let res = match embedder.embed_chunks_ref(texts.as_slice(), threads) {
             Ok(embeddings) => {
                 for (docid, embedding) in ids.into_iter().zip(embeddings) {

crates/milli/src/update/new/indexer/de.rs

@@ -41,11 +41,6 @@ impl<'de, 'p, 'indexer: 'de, Mapper: MutFieldIdMapper> Visitor<'de>
     where
         A: serde::de::MapAccess<'de>,
     {
-        // We need to remember if we encountered a semantic error, because raw values don't like to be parsed partially
-        // (trying to do so results in parsing errors).
-        // So we'll exhaust all keys and values even if we encounter an error, and we'll then return any error we detected.
-        let mut attribute_limit_reached = false;
-        let mut document_id_extraction_error = None;
         let mut docid = None;
 
         while let Some(((level_name, right), (fid, fields_ids_map))) =
@@ -54,36 +49,20 @@ impl<'de, 'p, 'indexer: 'de, Mapper: MutFieldIdMapper> Visitor<'de>
                 visitor: MutFieldIdMapVisitor(self.fields_ids_map),
             })?
         {
+            let Some(_fid) = fid else {
+                return Ok(Err(crate::UserError::AttributeLimitReached));
+            };
             self.fields_ids_map = fields_ids_map;
 
             let value: &'de RawValue = map.next_value()?;
-            if attribute_limit_reached || document_id_extraction_error.is_some() {
-                continue;
-            }
-
-            let Some(_fid) = fid else {
-                attribute_limit_reached = true;
-                continue;
-            };
 
             match match_component(level_name, right, value, self.indexer, &mut docid) {
                 ControlFlow::Continue(()) => continue,
                 ControlFlow::Break(Err(err)) => return Err(serde::de::Error::custom(err)),
-                ControlFlow::Break(Ok(err)) => {
+                ControlFlow::Break(Ok(err)) => return Ok(Ok(Err(err))),
-                    document_id_extraction_error = Some(err);
-                    continue;
-                }
             }
         }
 
-        // return previously detected errors
-        if attribute_limit_reached {
-            return Ok(Err(UserError::AttributeLimitReached));
-        }
-        if let Some(document_id_extraction_error) = document_id_extraction_error {
-            return Ok(Ok(Err(document_id_extraction_error)));
-        }
-
         Ok(Ok(match docid {
             Some(docid) => Ok(docid),
             None => Err(DocumentIdExtractionError::MissingDocumentId),

crates/milli/src/update/new/vector_document.rs

@@ -12,7 +12,7 @@ use super::indexer::de::DeserrRawValue;
 use crate::documents::FieldIdMapper;
 use crate::index::IndexEmbeddingConfig;
 use crate::vector::parsed_vectors::{
-    RawVectors, RawVectorsError, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
+    RawVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
 };
 use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfigs};
 use crate::{DocumentId, Index, InternalError, Result, UserError};
@@ -143,14 +143,7 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> {
                 Ok((&*config_name, entry))
             })
             .chain(self.vectors_field.iter().flat_map(|map| map.iter()).map(|(name, value)| {
-                Ok((
+                Ok((name, entry_from_raw_value(value, false).map_err(InternalError::SerdeJson)?))
-                    name,
-                    entry_from_raw_value(value, false).map_err(|_| {
-                        InternalError::Serialization(crate::SerializationError::Decoding {
-                            db_name: Some(crate::index::db_name::VECTOR_ARROY),
-                        })
-                    })?,
-                ))
             }))
     }
 
@@ -162,38 +155,20 @@ impl<'t> VectorDocument<'t> for VectorDocumentFromDb<'t> {
                 Some(self.entry_from_db(embedder_id, config)?)
             }
             None => match self.vectors_field.as_ref().and_then(|obkv| obkv.get(key)) {
-                Some(embedding_from_doc) => {
+                Some(embedding_from_doc) => Some(
-                    Some(entry_from_raw_value(embedding_from_doc, false).map_err(|_| {
+                    entry_from_raw_value(embedding_from_doc, false)
-                        InternalError::Serialization(crate::SerializationError::Decoding {
+                        .map_err(InternalError::SerdeJson)?,
-                            db_name: Some(crate::index::db_name::VECTOR_ARROY),
+                ),
-                        })
-                    })?)
-                }
                 None => None,
             },
         })
     }
 }
 
-fn entry_from_raw_value_user<'doc>(
-    external_docid: &str,
-    embedder_name: &str,
-    value: &'doc RawValue,
-    has_configured_embedder: bool,
-) -> Result<VectorEntry<'doc>> {
-    entry_from_raw_value(value, has_configured_embedder).map_err(|error| {
-        UserError::InvalidVectorsEmbedderConf {
-            document_id: external_docid.to_string(),
-            error: error.msg(embedder_name),
-        }
-        .into()
-    })
-}
-
 fn entry_from_raw_value(
     value: &RawValue,
     has_configured_embedder: bool,
-) -> std::result::Result<VectorEntry<'_>, RawVectorsError> {
+) -> std::result::Result<VectorEntry<'_>, serde_json::Error> {
     let value: RawVectors = RawVectors::from_raw_value(value)?;
 
     Ok(match value {
@@ -219,14 +194,12 @@ fn entry_from_raw_value(
 }
 
 pub struct VectorDocumentFromVersions<'doc> {
-    external_document_id: &'doc str,
     vectors: RawMap<'doc>,
     embedders: &'doc EmbeddingConfigs,
 }
 
 impl<'doc> VectorDocumentFromVersions<'doc> {
     pub fn new(
-        external_document_id: &'doc str,
         versions: &Versions<'doc>,
         bump: &'doc Bump,
         embedders: &'doc EmbeddingConfigs,
@@ -235,7 +208,7 @@ impl<'doc> VectorDocumentFromVersions<'doc> {
         if let Some(vectors_field) = document.vectors_field()? {
             let vectors =
                 RawMap::from_raw_value(vectors_field, bump).map_err(UserError::SerdeJson)?;
-            Ok(Some(Self { external_document_id, vectors, embedders }))
+            Ok(Some(Self { vectors, embedders }))
         } else {
             Ok(None)
         }
@@ -245,24 +218,16 @@ impl<'doc> VectorDocumentFromVersions<'doc> {
 impl<'doc> VectorDocument<'doc> for VectorDocumentFromVersions<'doc> {
     fn iter_vectors(&self) -> impl Iterator<Item = Result<(&'doc str, VectorEntry<'doc>)>> {
         self.vectors.iter().map(|(embedder, vectors)| {
-            let vectors = entry_from_raw_value_user(
+            let vectors = entry_from_raw_value(vectors, self.embedders.contains(embedder))
-                self.external_document_id,
+                .map_err(UserError::SerdeJson)?;
-                embedder,
-                vectors,
-                self.embedders.contains(embedder),
-            )?;
             Ok((embedder, vectors))
         })
     }
 
     fn vectors_for_key(&self, key: &str) -> Result<Option<VectorEntry<'doc>>> {
         let Some(vectors) = self.vectors.get(key) else { return Ok(None) };
-        let vectors = entry_from_raw_value_user(
+        let vectors = entry_from_raw_value(vectors, self.embedders.contains(key))
-            self.external_document_id,
+            .map_err(UserError::SerdeJson)?;
-            key,
-            vectors,
-            self.embedders.contains(key),
-        )?;
         Ok(Some(vectors))
     }
 }
@@ -273,10 +238,8 @@ pub struct MergedVectorDocument<'doc> {
 }
 
 impl<'doc> MergedVectorDocument<'doc> {
-    #[allow(clippy::too_many_arguments)]
     pub fn with_db<Mapper: FieldIdMapper>(
         docid: DocumentId,
-        external_document_id: &'doc str,
         index: &'doc Index,
         rtxn: &'doc RoTxn,
         db_fields_ids_map: &'doc Mapper,
@@ -285,20 +248,16 @@ impl<'doc> MergedVectorDocument<'doc> {
         embedders: &'doc EmbeddingConfigs,
     ) -> Result<Option<Self>> {
         let db = VectorDocumentFromDb::new(docid, index, rtxn, db_fields_ids_map, doc_alloc)?;
-        let new_doc =
+        let new_doc = VectorDocumentFromVersions::new(versions, doc_alloc, embedders)?;
-            VectorDocumentFromVersions::new(&external_document_id, versions, doc_alloc, embedders)?;
         Ok(if db.is_none() && new_doc.is_none() { None } else { Some(Self { new_doc, db }) })
     }
 
     pub fn without_db(
-        external_document_id: &'doc str,
         versions: &Versions<'doc>,
         doc_alloc: &'doc Bump,
         embedders: &'doc EmbeddingConfigs,
     ) -> Result<Option<Self>> {
-        let Some(new_doc) =
+        let Some(new_doc) = VectorDocumentFromVersions::new(versions, doc_alloc, embedders)? else {
-            VectorDocumentFromVersions::new(external_document_id, versions, doc_alloc, embedders)?
-        else {
             return Ok(None);
         };
         Ok(Some(Self { new_doc: Some(new_doc), db: None }))

crates/milli/src/vector/mod.rs

@@ -648,7 +648,7 @@ impl Embedder {
             Embedder::HuggingFace(embedder) => embedder.chunk_count_hint(),
             Embedder::OpenAi(embedder) => embedder.chunk_count_hint(),
             Embedder::Ollama(embedder) => embedder.chunk_count_hint(),
-            Embedder::UserProvided(_) => 100,
+            Embedder::UserProvided(_) => 1,
             Embedder::Rest(embedder) => embedder.chunk_count_hint(),
         }
     }

crates/milli/src/vector/parsed_vectors.rs

@@ -19,54 +19,10 @@ pub enum RawVectors<'doc> {
     ImplicitlyUserProvided(#[serde(borrow)] Option<&'doc RawValue>),
 }
 
-pub enum RawVectorsError {
-    DeserializeSeq { index: usize, error: String },
-    DeserializeKey { error: String },
-    DeserializeRegenerate { error: String },
-    DeserializeEmbeddings { error: String },
-    UnknownField { field: String },
-    MissingRegenerate,
-    WrongKind { kind: &'static str, value: String },
-    Parsing(serde_json::Error),
-}
-
-impl RawVectorsError {
-    pub fn msg(self, embedder_name: &str) -> String {
-        match self {
-            RawVectorsError::DeserializeSeq { index, error } => format!(
-                "Could not parse `._vectors.{embedder_name}[{index}]`: {error}"
-            ),
-            RawVectorsError::DeserializeKey { error } => format!(
-                "Could not parse a field at `._vectors.{embedder_name}`: {error}"
-            ),
-            RawVectorsError::DeserializeRegenerate { error } => format!(
-                "Could not parse `._vectors.{embedder_name}.regenerate`: {error}"
-            ),
-            RawVectorsError::DeserializeEmbeddings { error } => format!(
-                "Could not parse `._vectors.{embedder_name}.embeddings`: {error}"
-            ),
-            RawVectorsError::UnknownField { field } => format!(
-                "Unexpected field `._vectors.{embedder_name}.{field}`\n  \
-                  - note: the allowed fields are `regenerate` and `embeddings`"
-            ),
-            RawVectorsError::MissingRegenerate => format!(
-                "Missing field `._vectors.{embedder_name}.regenerate`\n  \
-                - note: `._vectors.{embedder_name}` must be an array of floats, an array of arrays of floats, or an object with field `regenerate`"
-            ),
-            RawVectorsError::WrongKind { kind, value } => format!(
-                "Expected `._vectors.{embedder_name}` to be an array of floats, an array of arrays of floats, or an object with at least the field `regenerate`, but got the {kind} `{value}`"
-            ),
-            RawVectorsError::Parsing(error) => format!(
-                "Could not parse `._vectors.{embedder_name}`: {error}"
-            ),
-        }
-    }
-}
-
 impl<'doc> RawVectors<'doc> {
-    pub fn from_raw_value(raw: &'doc RawValue) -> Result<Self, RawVectorsError> {
+    pub fn from_raw_value(raw: &'doc RawValue) -> Result<Self, serde_json::Error> {
         use serde::de::Deserializer as _;
-        Ok(match raw.deserialize_any(RawVectorsVisitor).map_err(RawVectorsError::Parsing)?? {
+        Ok(match raw.deserialize_any(RawVectorsVisitor)? {
             RawVectorsVisitorValue::ImplicitNone => RawVectors::ImplicitlyUserProvided(None),
             RawVectorsVisitorValue::Implicit => RawVectors::ImplicitlyUserProvided(Some(raw)),
             RawVectorsVisitorValue::Explicit { regenerate, embeddings } => {
@@ -85,7 +41,7 @@ enum RawVectorsVisitorValue<'doc> {
 }
 
 impl<'doc> serde::de::Visitor<'doc> for RawVectorsVisitor {
-    type Value = std::result::Result<RawVectorsVisitorValue<'doc>, RawVectorsError>;
+    type Value = RawVectorsVisitorValue<'doc>;
 
     fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(formatter, "a map containing at least `regenerate`, or an array of floats`")
@@ -95,7 +51,7 @@ impl<'doc> serde::de::Visitor<'doc> for RawVectorsVisitor {
     where
         E: serde::de::Error,
     {
-        Ok(Ok(RawVectorsVisitorValue::ImplicitNone))
+        Ok(RawVectorsVisitorValue::ImplicitNone)
     }
 
     fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
@@ -109,150 +65,42 @@ impl<'doc> serde::de::Visitor<'doc> for RawVectorsVisitor {
     where
         E: serde::de::Error,
     {
-        Ok(Ok(RawVectorsVisitorValue::ImplicitNone))
+        Ok(RawVectorsVisitorValue::ImplicitNone)
     }
 
     fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
     where
         A: serde::de::SeqAccess<'doc>,
     {
-        let mut index = 0;
         // must consume all elements or parsing fails
-        loop {
+        while let Some(_) = seq.next_element::<&RawValue>()? {}
-            match seq.next_element::<&RawValue>() {
+        Ok(RawVectorsVisitorValue::Implicit)
-                Ok(Some(_)) => index += 1,
-                Err(error) => {
-                    return Ok(Err(RawVectorsError::DeserializeSeq {
-                        index,
-                        error: error.to_string(),
-                    }))
-                }
-                Ok(None) => break,
-            };
-        }
-        Ok(Ok(RawVectorsVisitorValue::Implicit))
     }
 
     fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
     where
         A: serde::de::MapAccess<'doc>,
     {
+        use serde::de::Error as _;
         let mut regenerate = None;
         let mut embeddings = None;
-        loop {
+        while let Some(s) = map.next_key()? {
-            match map.next_key::<&str>() {
+            match s {
-                Ok(Some("regenerate")) => {
+                "regenerate" => {
-                    let value: bool = match map.next_value() {
+                    let value: bool = map.next_value()?;
-                        Ok(value) => value,
-                        Err(error) => {
-                            return Ok(Err(RawVectorsError::DeserializeRegenerate {
-                                error: error.to_string(),
-                            }))
-                        }
-                    };
                     regenerate = Some(value);
                 }
-                Ok(Some("embeddings")) => {
+                "embeddings" => {
-                    let value: &RawValue = match map.next_value() {
+                    let value: &RawValue = map.next_value()?;
-                        Ok(value) => value,
-                        Err(error) => {
-                            return Ok(Err(RawVectorsError::DeserializeEmbeddings {
-                                error: error.to_string(),
-                            }))
-                        }
-                    };
                     embeddings = Some(value);
                 }
-                Ok(Some(other)) => {
+                other => return Err(A::Error::unknown_field(other, &["regenerate", "embeddings"])),
-                    return Ok(Err(RawVectorsError::UnknownField { field: other.to_string() }))
-                }
-                Ok(None) => break,
-                Err(error) => {
-                    return Ok(Err(RawVectorsError::DeserializeKey { error: error.to_string() }))
-                }
             }
         }
         let Some(regenerate) = regenerate else {
-            return Ok(Err(RawVectorsError::MissingRegenerate));
+            return Err(A::Error::missing_field("regenerate"));
         };
-        Ok(Ok(RawVectorsVisitorValue::Explicit { regenerate, embeddings }))
+        Ok(RawVectorsVisitorValue::Explicit { regenerate, embeddings })
-    }
-
-    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "boolean", value: v.to_string() }))
-    }
-
-    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "integer", value: v.to_string() }))
-    }
-
-    fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "integer", value: v.to_string() }))
-    }
-
-    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "integer", value: v.to_string() }))
-    }
-
-    fn visit_u128<E>(self, v: u128) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "integer", value: v.to_string() }))
-    }
-
-    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "number", value: v.to_string() }))
-    }
-
-    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "string", value: v.to_string() }))
-    }
-
-    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "string", value: v }))
-    }
-
-    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
-    where
-        E: serde::de::Error,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "bytes", value: format!("{v:?}") }))
-    }
-
-    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
-    where
-        D: serde::Deserializer<'doc>,
-    {
-        deserializer.deserialize_any(self)
-    }
-
-    fn visit_enum<A>(self, _data: A) -> Result<Self::Value, A::Error>
-    where
-        A: serde::de::EnumAccess<'doc>,
-    {
-        Ok(Err(RawVectorsError::WrongKind { kind: "enum", value: "a variant".to_string() }))
     }
 }
 
@@ -495,7 +343,7 @@ impl Error {
             Error::InvalidEmbedderConf { error } => {
                 crate::Error::UserError(UserError::InvalidVectorsEmbedderConf {
                     document_id,
-                    error: error.to_string(),
+                    error,
                 })
             }
             Error::InternalSerdeJson(error) => {