diff --git a/src/api.rs b/src/api.rs
index 4e7d533..001db8c 100644
--- a/src/api.rs
+++ b/src/api.rs
@@ -891,6 +891,7 @@ fn decrypt_with_cached_messages(
             let _ = Prd::extract_from_message_with_commitment(&plain, &commitment)?;
             // Update the message status
             message.status = CachedMessageStatus::NoStatus;
+            message.shared_secrets = vec![]; // this way we won't check it again
 
             return Ok(Some((
                 plain,