From 1555870088cbd7c5a66cdc2cff4bb347f5a7967d Mon Sep 17 00:00:00 2001
From: Louis Dureuil <louis@meilisearch.com>
Date: Mon, 29 Jan 2024 11:22:24 +0100
Subject: [PATCH] Truncate HuggingFace vectors that are too long

---
 milli/src/vector/hf.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/milli/src/vector/hf.rs b/milli/src/vector/hf.rs
index 7acb09aa8..cdfdbfb75 100644
--- a/milli/src/vector/hf.rs
+++ b/milli/src/vector/hf.rs
@@ -151,7 +151,8 @@ impl Embedder {
         let token_ids = tokens
             .iter()
             .map(|tokens| {
-                let tokens = tokens.get_ids().to_vec();
+                let mut tokens = tokens.get_ids().to_vec();
+                tokens.truncate(512);
                 Tensor::new(tokens.as_slice(), &self.model.device).map_err(EmbedError::tensor_shape)
             })
             .collect::<Result<Vec<_>, EmbedError>>()?;