[Transformers] Support load mode from HF Hub when use Neural Speed (#1449)

Co-authored-by: Wenxin Zhang <[email protected]>
Co-authored-by: changwangss <[email protected]>

Author: 3 people

examples/.config/pytorch_optimize.json

@@ -1580,7 +1580,8 @@
       "params": {
         "topology": "mistral_7b_autoround",
         "task": "generation",
-        "output_model": "saved_results"
+        "output_model": "saved_results",
+        "weight_dtype": "int4_clip"
       }
     },
     "benchmark": {
@@ -1590,11 +1591,10 @@
         "task": "generation",
         "backend": "neuralspeed",
         "mode": "benchmark",
-        "batch_size": "112",
+        "batch_size": "10",
         "iters": "100",
         "int8": "false",
-        "config": "saved_results",
-        "weight_dtype": "int4_clip"
+        "config": "saved_results"
       }
     }
   },
@@ -1616,7 +1616,7 @@
         "task": "generation",
         "mode": "benchmark",
         "backend": "neuralspeed",
-        "batch_size": "112",
+        "batch_size": "10",
         "iters": "100",
         "int8": "false",
         "config": "saved_results"
@@ -1642,7 +1642,7 @@
         "task": "generation",
         "backend": "neuralspeed",
         "mode": "benchmark",
-        "batch_size": "112",
+        "batch_size": "10",
         "iters": "100",
         "int8": "false",
         "config": "saved_results"
@@ -1732,7 +1732,7 @@
         "task": "generation",
         "backend": "neuralspeed",
         "mode": "benchmark",
-        "batch_size": "112",
+        "batch_size": "10",
         "iters": "100",
         "int8": "false",
         "config": "saved_results",
@@ -1750,7 +1750,7 @@
         "task": "generation",
         "mode": "benchmark",
         "backend": "neuralspeed",
-        "batch_size": "112",
+        "batch_size": "10",
         "iters": "100",
         "int8": "false",
         "config": "saved_results",

examples/huggingface/neural_speed/perplexity/requirements.txt

@@ -13,4 +13,4 @@ tiktoken
 py-cpuinfo
 cmake
 gguf
-neural-speed==1.0a0
+neural-speed

examples/huggingface/neural_speed/requirements.txt

@@ -1,5 +1,5 @@
 intel_extension_for_transformers
-neural-speed==1.0a0
+neural-speed
 git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
 sentencepiece
 gguf

examples/huggingface/neural_speed/run_accuracy.py

@@ -19,15 +19,15 @@
     parser = argparse.ArgumentParser(description="Evaluate diff for a model")
     parser.add_argument('--model_name', type=str, default="~/Llama-2-7b-chat-hf", help="path to model")
     parser.add_argument('--tasks', type=str, default="lambada_openai")
-    parser.add_argument('--model_format', type=str, default="runtime")
+    parser.add_argument('--model_format', type=str, default="neural_speed")
     parser.add_argument('--use_gptq', action='store_true')
     parser.add_argument('--batch_size', type=int, default=1)
     args = parser.parse_args()
     print(args)
     model_args=f'pretrained="{args.model_name}",dtype=float32,trust_remote_code=True'
     if args.use_gptq:
         model_args += ",use_gptq=True"
-    if args.model_format == "runtime":
+    if args.model_format == "neural_speed":
         results = evaluate(
             model="hf-causal",
             model_args=model_args,

examples/huggingface/pytorch/text-generation/quantization/requirements.txt

@@ -13,7 +13,8 @@ bitsandbytes  #baichuan
 transformers_stream_generator
 tiktoken  #qwen
 einops  #qwen
-neural-speed
+git+https://github.com/intel/neural-speed[email protected]
 auto-round
 git+https://github.com/intel/neural-compressor.git
 git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2
+huggingface_hub

examples/huggingface/pytorch/text-generation/quantization/run_benchmark.sh

@@ -163,6 +163,8 @@ function run_benchmark {
         model_name_or_path="/tf_dataset2/models/pytorch/Mistral-7B-v0.1"
     elif [ "${topology}" = "mistral_7b_rtn" ] && [ "$model_source" != "huggingface" ]; then
         model_name_or_path="/tf_dataset2/models/pytorch/Mistral-7B-v0.1"
+    elif [ "${topology}" = "mistral_7b_gptq" ] && [ "$model_source" != "huggingface" ]; then
+        model_name_or_path="/tf_dataset2/models/pytorch/Mistral-7B-v0.1"
     fi
 
     if [[ ${int8} == "true" ]]; then

examples/huggingface/pytorch/text-generation/quantization/run_generation.py

@@ -250,7 +250,10 @@
 args.model = args.peft_model_id if args.peft_model_id is not None else args.model
 
 # Generation
-generate_kwargs = dict(do_sample=False, temperature=0.9, num_beams=4)
+if args.use_neural_speed:
+    generate_kwargs = dict(do_sample=False, temperature=0.9, num_beams=1)
+else:
+    generate_kwargs = dict(do_sample=False, temperature=0.9, num_beams=4)
 
 # mp/sq/woq/bitsandbytes config setting
 quantization_config = None
@@ -478,10 +481,9 @@
 
 if args.benchmark:
     user_model = (
-        user_model.eval() if not (args.int8 or args.int8_bf16_mixed) else user_model
+        user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) else user_model
     )
     prompt = "Once upon a time, there existed a little girl, who liked to have adventures. She wanted to go to places and meet new people, and have fun."
-
     input_size = tokenizer(prompt, return_tensors="pt").input_ids.size(dim=1)
     print("---- Prompt size:", input_size)
 
@@ -521,7 +523,7 @@
             toc = time.time()
             # please check the gen_ids if include input_ids.
             input_tokens_num = input_ids.numel()
-            output_tokens_num = gen_ids.numel() - input_tokens_num
+            output_tokens_num = torch.tensor(gen_ids).numel() - input_tokens_num
             print(gen_text, flush=True)
             if i >= num_warmup:
                 total_time += toc - tic
@@ -534,18 +536,30 @@
     print("Throughput: {} samples/sec".format(throughput))
 
 if args.accuracy:
-    user_model = (user_model.eval() if not (args.int8 or args.int8_bf16_mixed) else user_model)
+    user_model = (user_model.eval() if (not (args.int8 or args.int8_bf16_mixed) and hasattr(user_model, "eval")) \
+                  else user_model)
     args.model = (peft_config.base_model_name_or_path if args.peft_model_id else args.model)
     from intel_extension_for_transformers.transformers.llm.evaluation.lm_eval import evaluate
-
+    pretrained = ',pretrained=' + args.model
     args._commit_hash = "main" if args._commit_hash is None else args._commit_hash
+    eval_args = "tokenizer=" + args.model + ",dtype=float32" + ",_commit_hash=" + \
+                args._commit_hash + ",trust_remote_code=" + str(args.trust_remote_code)
+    if args.use_neural_speed:
+        eval_args += pretrained
+        q_conf = user_model.config.quantization_config
+        if isinstance(q_conf, dict):
+            q_algo = q_conf.get("quant_method", None)
+        else:
+            q_algo = q_conf.quant_method.value
+        if q_algo.upper() in ["AWQ", "GPTQ", "AUTOROUND"]:
+            eval_args += ",use_gptq=True"
     results = evaluate(
         model="hf-causal",
-        model_args="pretrained=" + args.model + ",tokenizer=" + args.model + ",dtype=float32" + ",_commit_hash=" + args._commit_hash +
-        ",trust_remote_code=" + str(args.trust_remote_code),
+        model_args=eval_args,
         user_model=user_model,
         batch_size=args.batch_size,
         tasks=args.tasks,
+        model_format="neural_speed" if args.use_neural_speed else "torch",
     )
     dumped = json.dumps(results, indent=2)
     if args.save_accuracy_path:

examples/huggingface/pytorch/text-generation/quantization/run_generation_gpu_woq.py

@@ -323,7 +323,7 @@
 
     results = evaluate(
         model="hf-causal",
-        model_args='pretrained=' + args.model + ',tokenizer=' + args.model + \
+        model_args='tokenizer=' + args.model + \
             ',dtype=float32,trust_remote_code=' + str(args.trust_remote_code),
         user_model=user_model,
         batch_size=args.batch_size,

examples/huggingface/pytorch/text-generation/quantization/run_tuning.sh

@@ -220,7 +220,7 @@ function run_tuning {
         extra_cmd=$extra_cmd" --weight_dtype ${weight_dtype}"
     elif [ "${topology}" = "mistral_7b_rtn" ]; then
         model_name_or_path="/tf_dataset2/models/pytorch/Mistral-7B-v0.1"
-        extra_cmd=$extra_cmd" --woq --bits 4 -compute_dtype fp32 --scheme asym "
+        extra_cmd=$extra_cmd" --woq --bits 4 --compute_dtype fp32 --scheme asym "
         extra_cmd=$extra_cmd" --woq_algo "Rtn" --desc_act --blocksize 128 --max_input_length 2048 "
         extra_cmd=$extra_cmd" --output_dir ${tuned_checkpoint}"
         extra_cmd=$extra_cmd" --trust_remote_code"

intel_extension_for_transformers/transformers/llm/evaluation/lm_eval/evaluator.py

@@ -124,6 +124,8 @@ def evaluate(model,
         }
         if user_model:
             kwargs["init_empty_weights"] = True
+            if "pretrained" not in model_args:
+                model_args = "pretrained='Muennighoff/tiny-random-bert'," + model_args
 
         if device == "hpu":
             # if hpu, set user_model