From 8318270f4dc1d626885430de08b6decf3263c5b4 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Wed, 15 Oct 2025 17:21:36 -0400 Subject: [PATCH 1/2] update --- .../quantization_w4a4_fp4/llama3_mxfp4.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 examples/quantization_w4a4_fp4/llama3_mxfp4.py diff --git a/examples/quantization_w4a4_fp4/llama3_mxfp4.py b/examples/quantization_w4a4_fp4/llama3_mxfp4.py new file mode 100644 index 0000000000..c6de83d2d8 --- /dev/null +++ b/examples/quantization_w4a4_fp4/llama3_mxfp4.py @@ -0,0 +1,35 @@ +from transformers import AutoModelForCausalLM, AutoTokenizer + +from llmcompressor import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier +from llmcompressor.utils import dispatch_for_generation + +MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct" + +# Load model. +model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +# Configure the quantization algorithm and scheme. +# In this case, we: +# * quantize the weights to fp4 with per group 16 via ptq +recipe = QuantizationModifier(targets="Linear", scheme="MXFP4A16", ignore=["lm_head"]) + +# Apply quantization. +oneshot(model=model, recipe=recipe) + +print("\n\n") +print("========== SAMPLE GENERATION ==============") +dispatch_for_generation(model) +input_ids = tokenizer("Hello my name is", return_tensors="pt").input_ids.to( + model.device +) +output = model.generate(input_ids, max_new_tokens=100) +print(tokenizer.decode(output[0])) +print("==========================================\n\n") + + +# Save to disk in compressed-tensors format. +SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-MXFP4A16" +model.save_pretrained(SAVE_DIR, save_compressed=True) +tokenizer.save_pretrained(SAVE_DIR) From 0208fdeb0008a8a6432d90530862f7c2f74b287c Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Thu, 16 Oct 2025 16:36:07 -0400 Subject: [PATCH 2/2] update --- examples/quantization_w4a4_fp4/llama3_mxfp4.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/quantization_w4a4_fp4/llama3_mxfp4.py b/examples/quantization_w4a4_fp4/llama3_mxfp4.py index c6de83d2d8..becc71c95e 100644 --- a/examples/quantization_w4a4_fp4/llama3_mxfp4.py +++ b/examples/quantization_w4a4_fp4/llama3_mxfp4.py @@ -4,7 +4,7 @@ from llmcompressor.modifiers.quantization import QuantizationModifier from llmcompressor.utils import dispatch_for_generation -MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct" +MODEL_ID = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" # Load model. model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto") @@ -13,7 +13,7 @@ # Configure the quantization algorithm and scheme. # In this case, we: # * quantize the weights to fp4 with per group 16 via ptq -recipe = QuantizationModifier(targets="Linear", scheme="MXFP4A16", ignore=["lm_head"]) +recipe = QuantizationModifier(targets="Linear", scheme="MXFP4", ignore=["lm_head"]) # Apply quantization. oneshot(model=model, recipe=recipe) @@ -30,6 +30,6 @@ # Save to disk in compressed-tensors format. -SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-MXFP4A16" +SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-MXFP4" model.save_pretrained(SAVE_DIR, save_compressed=True) tokenizer.save_pretrained(SAVE_DIR)