diff --git a/models/vision_language_model.py b/models/vision_language_model.py
index f22de5cc..252fc10e 100644
--- a/models/vision_language_model.py
+++ b/models/vision_language_model.py
@@ -227,7 +227,10 @@ def from_pretrained(
 
         # Load config
         with open(config_path, "r") as f:
-            cfg = VLMConfig(**json.load(f))
+            
+            config_json = json.load(f)
+            config_json.pop('lm_eos_token_id', None)
+            cfg = VLMConfig(**config_json)
 
         # Initialize model without loading the backbone
         model = cls(cfg, load_backbone=False)