diff --git a/milli/src/update/new/word_fst_builder.rs b/milli/src/update/new/word_fst_builder.rs
index 6c415c17e..97cd47e73 100644
--- a/milli/src/update/new/word_fst_builder.rs
+++ b/milli/src/update/new/word_fst_builder.rs
@@ -234,12 +234,18 @@ impl PrefixFstBuilder {
                 *current_prefix_is_modified = false;
             }
 
-            *current_prefix_is_modified |= is_modified;
-
             if deladd == DelAdd::Addition {
                 *current_prefix_count += 1;
             }
 
+            if is_modified && !*current_prefix_is_modified {
+                if *current_prefix_count > self.prefix_count_threshold {
+                    self.modified_prefixes.insert(current_prefix.clone());
+                }
+
+                *current_prefix_is_modified = true;
+            }
+
             // There is enough words corresponding to this prefix to add it to the cache.
             if *current_prefix_count == self.prefix_count_threshold {
                 builder.insert(prefix)?;