Address PR feedback: simplify dependencies and configuration

- Remove torch and accelerate from installation (dependencies of TRL)
- Remove pad token check (handled automatically)
- Restore num_generations to default value (8)
- Remove remove_unused_columns parameter (false by default)
- Remove processing_class parameter (loaded automatically)

Author: behroozazarkhalili

notebooks/en/trl_grpo_reasoning_advanced_reward.ipynb

@@ -17,10 +17,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "# Install required packages\n",
-    "!pip install transformers datasets trl accelerate bitsandbytes peft torch"
-   ]
+   "source": "# Install required packages\n!pip install transformers datasets trl bitsandbytes peft"
   },
   {
    "cell_type": "markdown",
@@ -62,69 +59,14 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import torch\n",
-    "import re\n",
-    "from transformers import (\n",
-    "    AutoModelForCausalLM, \n",
-    "    AutoTokenizer, \n",
-    "    BitsAndBytesConfig,\n",
-    ")\n",
-    "from peft import LoraConfig, get_peft_model, TaskType\n",
-    "from datasets import load_dataset\n",
-    "from trl import GRPOConfig, GRPOTrainer\n",
-    "import logging\n",
-    "\n",
-    "# Set up logging\n",
-    "logging.basicConfig(level=logging.INFO)\n",
-    "logger = logging.getLogger(__name__)"
-   ]
+   "source": "import torch\nimport re\nfrom transformers import (\n    AutoModelForCausalLM, \n    AutoTokenizer, \n    BitsAndBytesConfig,\n)\nfrom peft import LoraConfig, get_peft_model, TaskType\nfrom datasets import load_dataset\nfrom trl import GRPOConfig, GRPOTrainer\nimport logging\n\n# Set up logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)"
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "# Model configuration\n",
-    "model_name = \"Qwen/Qwen2.5-3B-Instruct\"  # You can change this to any model you prefer\n",
-    "# Alternative models:\n",
-    "# model_name = \"microsoft/DialoGPT-small\"\n",
-    "# model_name = \"gpt2\"\n",
-    "# model_name = \"google/gemma-2b\"\n",
-    "\n",
-    "max_seq_length = 2048\n",
-    "\n",
-    "# Quantization config for memory efficiency\n",
-    "bnb_config = BitsAndBytesConfig(\n",
-    "    load_in_4bit=True,\n",
-    "    bnb_4bit_quant_type=\"nf4\",\n",
-    "    bnb_4bit_compute_dtype=torch.float16,\n",
-    "    bnb_4bit_use_double_quant=True,\n",
-    ")\n",
-    "\n",
-    "# Load model and tokenizer with correct device mapping\n",
-    "# Since CUDA_VISIBLE_DEVICES=\"1\" is set, GPU 1 becomes device 0 from PyTorch's perspective\n",
-    "model = AutoModelForCausalLM.from_pretrained(\n",
-    "    model_name,\n",
-    "    quantization_config=bnb_config,\n",
-    "    device_map={\"\": 0},  # Use device 0 (which is actually GPU 1 due to CUDA_VISIBLE_DEVICES)\n",
-    "    trust_remote_code=True\n",
-    ")\n",
-    "\n",
-    "tokenizer = AutoTokenizer.from_pretrained(\n",
-    "    model_name,\n",
-    "    trust_remote_code=True\n",
-    ")\n",
-    "\n",
-    "# Add pad token if it doesn't exist\n",
-    "if tokenizer.pad_token is None:\n",
-    "    tokenizer.pad_token = tokenizer.eos_token\n",
-    "\n",
-    "print(f\"Model loaded: {model_name}\")\n",
-    "print(f\"Model device: {model.device}\")\n",
-    "print(f\"Tokenizer vocab size: {len(tokenizer)}\")"
-   ]
+   "source": "# Model configuration\nmodel_name = \"Qwen/Qwen2.5-3B-Instruct\"  # You can change this to any model you prefer\n# Alternative models:\n# model_name = \"microsoft/DialoGPT-small\"\n# model_name = \"gpt2\"\n# model_name = \"google/gemma-2b\"\n\nmax_seq_length = 2048\n\n# Quantization config for memory efficiency\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    bnb_4bit_quant_type=\"nf4\",\n    bnb_4bit_compute_dtype=torch.float16,\n    bnb_4bit_use_double_quant=True,\n)\n\n# Load model and tokenizer with correct device mapping\n# Since CUDA_VISIBLE_DEVICES=\"1\" is set, GPU 1 becomes device 0 from PyTorch's perspective\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_name,\n    quantization_config=bnb_config,\n    device_map={\"\": 0},  # Use device 0 (which is actually GPU 1 due to CUDA_VISIBLE_DEVICES)\n    trust_remote_code=True\n)\n\ntokenizer = AutoTokenizer.from_pretrained(\n    model_name,\n    trust_remote_code=True\n)\n\nprint(f\"Model loaded: {model_name}\")\nprint(f\"Model device: {model.device}\")\nprint(f\"Tokenizer vocab size: {len(tokenizer)}\")"
   },
   {
    "cell_type": "markdown",
@@ -341,51 +283,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "# GRPO Training configuration with enhanced logging\n",
-    "training_args = GRPOConfig(\n",
-    "    learning_rate=5e-6,\n",
-    "    adam_beta1=0.9,\n",
-    "    adam_beta2=0.99,\n",
-    "    weight_decay=0.1,\n",
-    "    warmup_ratio=0.1,\n",
-    "    lr_scheduler_type=\"cosine\",\n",
-    "    optim=\"adamw_torch_fused\",\n",
-    "    logging_steps=1,  # Log every step\n",
-    "    per_device_train_batch_size=2,  # Start small to avoid memory issues\n",
-    "    gradient_accumulation_steps=8,  # Increase to maintain effective batch size\n",
-    "    num_generations=4,  # Reduce to save memory\n",
-    "    max_prompt_length=1024,  # Reduce if needed\n",
-    "    max_completion_length=1024,  # Reduce if needed\n",
-    "    max_steps=10,  # Reduce for testing\n",
-    "    save_steps=10,\n",
-    "    eval_steps=1,  # Enable evaluation logging\n",
-    "    max_grad_norm=0.1,\n",
-    "    report_to=\"none\",  # Disable reporting to external services\n",
-    "    output_dir=\"./trl_grpo_outputs\",\n",
-    "    logging_dir=\"./logs\",  # Directory for logs\n",
-    "    remove_unused_columns=False,\n",
-    "    dataloader_drop_last=True,\n",
-    "    # Enhanced logging options\n",
-    "    log_level=\"info\",\n",
-    "    logging_first_step=True,\n",
-    "    logging_nan_inf_filter=True,\n",
-    "    metric_for_best_model=\"reward\",\n",
-    "    greater_is_better=True,\n",
-    "    # Keep default progress bar enabled\n",
-    "    disable_tqdm=False,\n",
-    ")\n",
-    "\n",
-    "print(\"Training configuration with enhanced default progress bar:\")\n",
-    "print(f\"Batch size: {training_args.per_device_train_batch_size}\")\n",
-    "print(f\"Gradient accumulation: {training_args.gradient_accumulation_steps}\")\n",
-    "print(f\"Effective batch size: {training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps}\")\n",
-    "print(f\"Max steps: {training_args.max_steps}\")\n",
-    "print(f\"Learning rate: {training_args.learning_rate}\")\n",
-    "print(f\"Logging every: {training_args.logging_steps} steps\")\n",
-    "print(f\"Evaluation every: {training_args.eval_steps} steps\")\n",
-    "print(f\"Default tqdm enabled: {not training_args.disable_tqdm}\")"
-   ]
+   "source": "# GRPO Training configuration with enhanced logging\ntraining_args = GRPOConfig(\n    learning_rate=5e-6,\n    adam_beta1=0.9,\n    adam_beta2=0.99,\n    weight_decay=0.1,\n    warmup_ratio=0.1,\n    lr_scheduler_type=\"cosine\",\n    optim=\"adamw_torch_fused\",\n    logging_steps=1,  # Log every step\n    per_device_train_batch_size=2,  # Start small to avoid memory issues\n    gradient_accumulation_steps=8,  # Increase to maintain effective batch size\n    max_prompt_length=1024,  # Reduce if needed\n    max_completion_length=1024,  # Reduce if needed\n    max_steps=10,  # Reduce for testing\n    save_steps=10,\n    eval_steps=1,  # Enable evaluation logging\n    max_grad_norm=0.1,\n    report_to=\"none\",  # Disable reporting to external services\n    output_dir=\"./trl_grpo_outputs\",\n    logging_dir=\"./logs\",  # Directory for logs\n    dataloader_drop_last=True,\n    # Enhanced logging options\n    log_level=\"info\",\n    logging_first_step=True,\n    logging_nan_inf_filter=True,\n    metric_for_best_model=\"reward\",\n    greater_is_better=True,\n    # Keep default progress bar enabled\n    disable_tqdm=False,\n)\n\nprint(\"Training configuration with enhanced default progress bar:\")\nprint(f\"Batch size: {training_args.per_device_train_batch_size}\")\nprint(f\"Gradient accumulation: {training_args.gradient_accumulation_steps}\")\nprint(f\"Effective batch size: {training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps}\")\nprint(f\"Max steps: {training_args.max_steps}\")\nprint(f\"Learning rate: {training_args.learning_rate}\")\nprint(f\"Logging every: {training_args.logging_steps} steps\")\nprint(f\"Evaluation every: {training_args.eval_steps} steps\")\nprint(f\"Default tqdm enabled: {not training_args.disable_tqdm}\")"
   },
   {
    "cell_type": "markdown",
@@ -540,37 +438,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "max_steps is given, it will override any value given in num_train_epochs\n",
-      "Using auto half precision backend\n",
-      "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Initialize GRPO trainer with HuggingFace-style interactive table callback\n",
-    "hf_table_callback = HuggingFaceStyleTableCallback()\n",
-    "\n",
-    "trainer = GRPOTrainer(\n",
-    "    model=model,\n",
-    "    processing_class=tokenizer,\n",
-    "    reward_funcs=[\n",
-    "        match_format_exactly,\n",
-    "        match_format_approximately,\n",
-    "        check_answer_correctness,\n",
-    "        check_numbers_extraction,\n",
-    "    ],\n",
-    "    args=training_args,\n",
-    "    train_dataset=dataset,\n",
-    "    callbacks=[hf_table_callback],  # Add HuggingFace-style table callback\n",
-    ")"
-   ]
+   "outputs": [],
+   "source": "# Initialize GRPO trainer with HuggingFace-style interactive table callback\nhf_table_callback = HuggingFaceStyleTableCallback()\n\ntrainer = GRPOTrainer(\n    model=model,\n    reward_funcs=[\n        match_format_exactly,\n        match_format_approximately,\n        check_answer_correctness,\n        check_numbers_extraction,\n    ],\n    args=training_args,\n    train_dataset=dataset,\n    callbacks=[hf_table_callback],  # Add HuggingFace-style table callback\n)"
   },
   {
    "cell_type": "code",