-
Notifications
You must be signed in to change notification settings - Fork 3
Description
Why do we have evaluation method twice in evalTypes? https://github.com/evaleval/evalHub/blob/main/schema/eval_types.py
One of them should be deleted (which?)
class EvaluationMethod(BaseModel):
method_name: str = Field(
...,
description='Name of the evaluation method. Can be a predefined method or a user-defined method.',
)
description: str = Field(
...,
description='Detailed explanation of how the evaluation method works. For user-defined methods, this is required.',
)
parameters: Optional[Dict[str, Any]] = Field(
None,
description='Optional parameters used by the evaluation method. Allows custom configuration.',
)
class Evaluation(BaseModel):
evaluation_method: EvaluationMethod = Field(
...,
description='Method used to evaluate the answer, including predefined methods and user-defined methods.',
)
ground_truth: str = Field(..., description='The correct answer for evaluation')
score: confloat(ge=0.0, le=1.0) = Field(
...,
description="Binary score indicating whether the model's answer was correct (1.0) or incorrect (0.0)",
)