From cbd065ed4668bfedb516f838c19dc9f4bf53a19a Mon Sep 17 00:00:00 2001
From: Louis Dureuil <louis@meilisearch.com>
Date: Mon, 29 Jan 2024 11:22:24 +0100
Subject: [PATCH] Truncate HuggingFace vectors that are too long

---
 milli/src/vector/hf.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/milli/src/vector/hf.rs b/milli/src/vector/hf.rs
index 0a6bcbe93..08804e515 100644
--- a/milli/src/vector/hf.rs
+++ b/milli/src/vector/hf.rs
@@ -145,7 +145,8 @@ impl Embedder {
         let token_ids = tokens
             .iter()
             .map(|tokens| {
-                let tokens = tokens.get_ids().to_vec();
+                let mut tokens = tokens.get_ids().to_vec();
+                tokens.truncate(512);
                 Tensor::new(tokens.as_slice(), &self.model.device).map_err(EmbedError::tensor_shape)
             })
             .collect::<Result<Vec<_>, EmbedError>>()?;