Added Falcon model converter and Added Falcon 7b support

keras_hub/src/utils/transformers/convert_falcon.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+from keras_hub.src.models.falcon import FalconBackbone
+from keras_hub.src.utils.preset_utils import load_json
+
+backbone_cls = FalconBackbone
+
+
+def convert_backbone_config(transformers_config):
+    return {
+        "vocabulary_size": transformers_config["vocab_size"],
+        "num_layers": transformers_config["num_hidden_layers"],
+        "num_attention_heads": transformers_config["num_attention_heads"],
+        "hidden_dim": transformers_config["hidden_size"],
+        "intermediate_dim": 32 * 4,
+    }
+
+
+def transpose_and_reshape(x, shape):
+    return np.reshape(np.transpose(x), shape)
+
+
+def convert_weights(backbone, loader, transformers_config):
+    # Embeddings
+    loader.port_weight(
+        keras_variable=backbone.get_layer("token_embedding").embeddings,
+        hf_weight_key="word_embeddings.weight",
+    )
+
+    for i in range(backbone.num_layers):
+        decoder_layer = backbone.get_layer(f"transformer_layer_{i}")
+
+        # Norm layer
+        loader.port_weight(
+            keras_variable=decoder_layer.input_layernorm.gamma,
+            hf_weight_key=f"h.{i}.input_layernorm.weight",
+        )
+
+        # Attention layers
+        loader.port_weight(
+            keras_variable=decoder_layer.attention_layer.output_dense.kernel,
+            hf_weight_key=f"h.{i}.self_attention.dense.weight",
+        )
+
+        loader.port_weight(
+            keras_variable=decoder_layer.post_attention_layernorm.gamma,
+            hf_weight_key=f"h.{i}.self_attention.query_key_value.weight",
+            hook_fn=lambda hf_tensor, keras_shape: np.mean(
+                np.reshape(hf_tensor, (-1, keras_shape[0])), axis=0
+            ),
+        )
+
+
+def convert_tokenizer(cls, preset, **kwargs):
+    tokenizer_data = load_json(preset, "tokenizer.json")
+    vocab = tokenizer_data["model"]["vocab"]
+    merges = tokenizer_data["model"].get("merges", None)
+
+    tokenizer_kwargs = {"vocabulary": vocab, "merges": merges}
+    return cls(**tokenizer_kwargs)

keras_hub/src/utils/transformers/convert_falcon_test.py

@@ -0,0 +1,23 @@
+import pytest
+
+from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
+from keras_hub.src.models.falcon.falcon_causal_lm import FalconCausalLM
+from keras_hub.src.tests.test_case import TestCase
+
+
+class TestTask(TestCase):
+    @pytest.mark.large
+    def test_convert_tiny_preset(self):
+        model = FalconCausalLM.from_preset("hf://tiiuae/falcon-7b")
+        prompt = "What is your favorite condiment?"
+        model.generate([prompt], max_length=15)
+
+    @pytest.mark.large
+    def test_class_detection(self):
+        model = FalconCausalLM.from_preset("hf://tiiuae/falcon-7b")
+        self.assertIsInstance(model, FalconCausalLM)
+        model = FalconBackbone.from_preset(
+            "hf://tiiuae/falcon-7b",
+            load_weights=False,
+        )
+        self.assertIsInstance(model, FalconBackbone)

keras_hub/src/utils/transformers/preset_loader.py

@@ -7,6 +7,7 @@
 from keras_hub.src.utils.transformers import convert_bart
 from keras_hub.src.utils.transformers import convert_bert
 from keras_hub.src.utils.transformers import convert_distilbert
+from keras_hub.src.utils.transformers import convert_falcon
 from keras_hub.src.utils.transformers import convert_gemma
 from keras_hub.src.utils.transformers import convert_gpt2
 from keras_hub.src.utils.transformers import convert_llama3
@@ -41,6 +42,8 @@ def __init__(self, preset, config):
             self.converter = convert_pali_gemma
         elif model_type == "vit":
             self.converter = convert_vit
+        elif model_type == "falcon":
+            self.converter = convert_falcon
         else:
             raise ValueError(
                 "KerasHub has no converter for huggingface/transformers models "