[RFC] Rework the dependencies to be more versatile (#951)

This PR proposes a change in the way optional dependencies are managed so that it's easier to shield tasks that have specific dependencies. See below for an explanation of the different changes.

The first change renames the `is_xxx_available() -> bool` methods to a more versatile `is_package_available(package_name: str) -> bool`.

This unbloats the `imports.py` module while adding two additional methods that work on top of it:
- `is_multilingual_package_available(language_code: Optional[str] = None) -> bool` which checks the presence of `spacy`, and optionally takes in a language code to check whether the language-related dependencies are installed.
- the `@requires(package_name)` decorator which can be added on top of classes/methods to raise an error as soon as any method of the class/the method is called, if the package isn't installed.

FYI: this is a draft PR that isn't tested at this point, and that I need to test in a real-world scenario. I'd like to validate the direction before adding relevant tests.

Author: LysandreJik

docs/source/using-the-python-api.mdx

@@ -12,9 +12,9 @@ import lighteval
 from lighteval.logging.evaluation_tracker import EvaluationTracker
 from lighteval.models.vllm.vllm_model import VLLMModelConfig
 from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
-from lighteval.utils.imports import is_accelerate_available
+from lighteval.utils.imports import is_package_available
 
-if is_accelerate_available():
+if is_package_available("accelerate"):
     from datetime import timedelta
     from accelerate import Accelerator, InitProcessGroupKwargs
     accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))])

pyproject.toml

@@ -84,6 +84,7 @@ dependencies = [
     "fsspec>=2023.12.2",
     "httpx>=0.27.2",
     "latex2sympy2_extended==1.0.6",
+    "langcodes",
 ]
 
 [project.optional-dependencies]
@@ -98,6 +99,7 @@ nanotron = [
 ]
 tensorboardX = ["tensorboardX"]
 vllm = ["vllm>=0.10.0,<0.10.2", "ray", "more_itertools"]
+sglang = ["sglang"]
 quality = ["ruff>=v0.11.0","pre-commit"]
 tests = ["pytest>=7.4.0","deepdiff","pip>=25.2"]
 dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks,vllm]"]

src/lighteval/logging/evaluation_tracker.py

@@ -43,13 +43,13 @@
     TaskConfigLogger,
     VersionsLogger,
 )
-from lighteval.utils.imports import NO_TENSORBOARDX_WARN_MSG, is_nanotron_available, is_tensorboardX_available
+from lighteval.utils.imports import is_package_available, not_installed_error_message
 from lighteval.utils.utils import obj_to_markdown
 
 
 logger = logging.getLogger(__name__)
 
-if is_nanotron_available():
+if is_package_available("nanotron"):
     from nanotron.config import GeneralArgs  # type: ignore
 
 try:
@@ -659,11 +659,11 @@ def recreate_metadata_card(self, repo_id: str) -> None:  # noqa: C901
     def push_to_tensorboard(  # noqa: C901
         self, results: dict[str, dict[str, float]], details: dict[str, DetailsLogger.CompiledDetail]
     ):
-        if not is_tensorboardX_available:
-            logger.warning(NO_TENSORBOARDX_WARN_MSG)
+        if not is_package_available("tensorboardX"):
+            logger.warning(not_installed_error_message("tensorboardX"))
             return
 
-        if not is_nanotron_available():
+        if not is_package_available("nanotron"):
             logger.warning("You cannot push results to tensorboard without having nanotron installed. Skipping")
             return
 

src/lighteval/logging/info_loggers.py

@@ -34,13 +34,13 @@
 from lighteval.models.model_output import ModelResponse
 from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig
 from lighteval.tasks.requests import Doc
-from lighteval.utils.imports import is_nanotron_available
+from lighteval.utils.imports import is_package_available
 
 
 logger = logging.getLogger(__name__)
 
 
-if is_nanotron_available():
+if is_package_available("nanotron"):
     pass
 
 

src/lighteval/main_nanotron.py

@@ -32,11 +32,13 @@
     reasoning_tags,
     remove_reasoning_tags,
 )
+from lighteval.utils.imports import requires
 
 
 SEED = 1234
 
 
+@requires("nanotron")
 def nanotron(
     checkpoint_config_path: Annotated[
         str, Option(help="Path to the nanotron checkpoint YAML or python config file, potentially on s3.")
@@ -45,12 +47,9 @@ def nanotron(
     remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default,
     reasoning_tags: reasoning_tags.type = reasoning_tags.default,
 ):
-    """Evaluate models using nanotron as backend."""
-    from lighteval.utils.imports import NO_NANOTRON_ERROR_MSG, is_nanotron_available
-
-    if not is_nanotron_available():
-        raise ImportError(NO_NANOTRON_ERROR_MSG)
-
+    """
+    Evaluate models using nanotron as backend.
+    """
     from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs, get_config_from_dict, get_config_from_file
 
     from lighteval.logging.evaluation_tracker import EvaluationTracker

src/lighteval/metrics/imports/data_stats_metric.py

@@ -30,7 +30,7 @@
 from typing import Literal
 
 from lighteval.metrics.imports.data_stats_utils import Fragments
-from lighteval.utils.imports import NO_SPACY_ERROR_MSG, is_spacy_available
+from lighteval.utils.imports import Extra, requires
 
 
 logger = logging.getLogger(__name__)
@@ -55,6 +55,7 @@ def find_ngrams(input_list, n):
     return zip(*[input_list[i:] for i in range(n)])
 
 
+@requires(Extra.MULTILINGUAL)
 class DataStatsMetric(Metric):
     def __init__(
         self,
@@ -86,8 +87,6 @@ def __init__(
                 determines the spaCy model used for tokenization. Currently supports English,
                 German, French, and Italian.
         """
-        if not is_spacy_available():
-            raise ImportError(NO_SPACY_ERROR_MSG)
         import spacy
 
         self.n_gram = n_gram

src/lighteval/metrics/normalizations.py

@@ -28,6 +28,7 @@
 from typing import Callable
 
 from lighteval.metrics.utils.linguistic_tokenizers import get_word_tokenizer
+from lighteval.utils.imports import Extra, requires
 from lighteval.utils.language import Language
 
 
@@ -444,15 +445,16 @@ def remove_punc(text: str) -> str:
     return "".join(ch for ch in text if ch not in PUNCT)
 
 
+@requires(Extra.MULTILINGUAL)
 def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]:
     """Get a normalizer function for the specified language.
 
     Returns:
         Callable[[str], str]: A function that normalizes text for the specified language
     """
-    tokenizer = get_word_tokenizer(lang)
 
     def _inner_normalizer(text: str) -> str:
+        tokenizer = get_word_tokenizer(lang)
         text = remove_articles(text, lang)
         text = remove_punc(text)
         if lower:

src/lighteval/metrics/utils/extractive_match_utils.py

@@ -34,12 +34,12 @@
 from lighteval.tasks.requests import Doc
 from lighteval.tasks.templates.utils.formulation import ChoicePrefix, get_prefix
 from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS
-from lighteval.utils.imports import requires_latex2sympy2_extended
+from lighteval.utils.imports import requires
 from lighteval.utils.language import Language
 from lighteval.utils.timeout import timeout
 
 
-@requires_latex2sympy2_extended
+@requires("latex2sympy2_extended")
 def latex_normalization_config_default_factory():
     from latex2sympy2_extended.latex2sympy2 import NormalizationConfig
 
@@ -373,7 +373,7 @@ def get_target_type_order(target_type: ExtractionTarget) -> int:
 
 # Small cache, to catche repeated calls invalid parsing
 @lru_cache(maxsize=20)
-@requires_latex2sympy2_extended
+@requires("latex2sympy2_extended")
 def parse_latex_with_timeout(latex: str, timeout_seconds: int):
     from latex2sympy2_extended.latex2sympy2 import latex2sympy
 
@@ -428,7 +428,7 @@ def convert_to_pct(number: Number):
     return sympy.Mul(number, sympy.Rational(1, 100), evaluate=False)
 
 
-@requires_latex2sympy2_extended
+@requires("latex2sympy2_extended")
 @lru_cache(maxsize=20)
 def extract_latex(
     match: re.Match, latex_config: LatexExtractionConfig, timeout_seconds: int

src/lighteval/metrics/utils/linguistic_tokenizers.py

@@ -18,10 +18,8 @@
 from typing import Callable, Iterator
 
 from lighteval.utils.imports import (
-    NO_SPACY_TOKENIZER_ERROR_MSG,
-    NO_STANZA_TOKENIZER_ERROR_MSG,
-    can_load_spacy_tokenizer,
-    can_load_stanza_tokenizer,
+    Extra,
+    requires,
 )
 from lighteval.utils.language import Language
 
@@ -99,11 +97,10 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
         return list(self.tokenizer.span_tokenize(text))
 
 
+@requires(Extra.MULTILINGUAL)
 class SpaCyTokenizer(WordTokenizer):
     def __init__(self, spacy_language: str, config=None):
         super().__init__()
-        if not can_load_spacy_tokenizer(spacy_language):
-            raise ImportError(NO_SPACY_TOKENIZER_ERROR_MSG)
         self.spacy_language = spacy_language
         self.config = config
         self._tokenizer = None
@@ -137,11 +134,10 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
         ]
 
 
+@requires("stanza")
 class StanzaTokenizer(WordTokenizer):
     def __init__(self, stanza_language: str, **stanza_kwargs):
         super().__init__()
-        if not can_load_stanza_tokenizer():
-            raise ImportError(NO_STANZA_TOKENIZER_ERROR_MSG)
         self.stanza_language = stanza_language
         self.stanza_kwargs = stanza_kwargs
         self._tokenizer = None

src/lighteval/metrics/utils/llm_as_judge.py

@@ -34,7 +34,7 @@
 from tqdm import tqdm
 from tqdm.asyncio import tqdm_asyncio
 
-from lighteval.utils.imports import is_litellm_available, is_openai_available, is_vllm_available
+from lighteval.utils.imports import raise_if_package_not_available
 from lighteval.utils.utils import as_list
 
 
@@ -151,8 +151,7 @@ def __lazy_load_client(self):  # noqa: C901
             # Both "openai" and "tgi" backends use the OpenAI-compatible API
             # They are handled separately to allow for backend-specific validation and setup
             case "openai" | "tgi":
-                if not is_openai_available():
-                    raise RuntimeError("OpenAI backend is not available.")
+                raise_if_package_not_available("openai")
                 if self.client is None:
                     from openai import OpenAI
 
@@ -162,13 +161,11 @@ def __lazy_load_client(self):  # noqa: C901
                 return self.__call_api_parallel
 
             case "litellm":
-                if not is_litellm_available():
-                    raise RuntimeError("litellm is not available.")
+                raise_if_package_not_available("litellm")
                 return self.__call_litellm
 
             case "vllm":
-                if not is_vllm_available():
-                    raise RuntimeError("vllm is not available.")
+                raise_if_package_not_available("vllm")
                 if self.pipe is None:
                     from vllm import LLM, SamplingParams
                     from vllm.transformers_utils.tokenizer import get_tokenizer