Gather per-token entropy before computing stats

Author: SamComber

verifiers/trainers/grpo_trainer.py

@@ -1327,6 +1327,7 @@ def compute_loss(  # type: ignore
         if self.log_policy_entropy:
             masked_entropy = per_token_entropy * completion_mask
             total_completion_tokens = completion_mask.sum()
+
             if total_completion_tokens > 0:
                 mean_entropy = masked_entropy.sum() / total_completion_tokens
                 gathered_entropy = self.accelerator.gather_for_metrics(mean_entropy)