diff --git a/README.md b/README.md index d4a0dcb18..c7712631c 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ hf auth login Lighteval offers the following entry points for model evaluation: -- `lighteval eval`: Evaluation models using [inspect-ai](https://inspect.aisi.org.uk/) as a backend (prefered). +- `lighteval eval`: Evaluation models using [inspect-ai](https://inspect.aisi.org.uk/) as a backend (preferred). - `lighteval accelerate`: Evaluate models on CPU or one or more GPUs using [🤗 Accelerate](https://github.com/huggingface/accelerate) - `lighteval nanotron`: Evaluate models in distributed settings using [⚡️ diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx index c68149a1e..0fee1b4ca 100644 --- a/docs/source/adding-a-custom-task.mdx +++ b/docs/source/adding-a-custom-task.mdx @@ -12,7 +12,7 @@ Lighteval provides a flexible framework for creating custom evaluation tasks. Th ### Step 1: Create the Task File First, create a Python file or directory under the `src/lighteval/tasks/tasks` directory. -A directory is helpfull if you need to split your file into multiple ones, just make sure to have one of the file named `main.py`. +A directory is helpful if you need to split your file into multiple ones, just make sure to have one of the file named `main.py`. ### Step 2: Define the Prompt Function diff --git a/docs/source/index.mdx b/docs/source/index.mdx index 9eef93f5b..b2d3dae0b 100644 --- a/docs/source/index.mdx +++ b/docs/source/index.mdx @@ -12,7 +12,7 @@ and see how your models stack up. ### 🚀 **Multi-Backend Support** Evaluate your models using the most popular and efficient inference backends: -- `eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models! (prefered way) +- `eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models! (preferred way) - `transformers`: Evaluate models on CPU or one or more GPUs using [🤗 Accelerate](https://github.com/huggingface/transformers) - `nanotron`: Evaluate models in distributed settings using [⚡️ diff --git a/docs/source/package_reference/models_outputs.mdx b/docs/source/package_reference/models_outputs.mdx index 8bc0737a1..23003612b 100644 --- a/docs/source/package_reference/models_outputs.mdx +++ b/docs/source/package_reference/models_outputs.mdx @@ -1,5 +1,5 @@ # Model's Output -All models will generate an ouput per Doc supplied to the `generation` or `loglikelihood` fuctions. +All models will generate an output per Doc supplied to the `generation` or `loglikelihood` functions. [[autodoc]] lighteval.models.model_output.ModelResponse diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 919c8a1a1..4fa1e6d61 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -21,7 +21,7 @@ Lighteval can be used with several different commands, each optimized for differ ### Evaluation Backends -- `lighteval eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models ! (prefered way) +- `lighteval eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models ! (preferred way) - `lighteval accelerate`: Evaluate models on CPU or one or more GPUs using [🤗 Accelerate](https://github.com/huggingface/accelerate) - `lighteval nanotron`: Evaluate models in distributed settings using [⚡️ diff --git a/src/lighteval/metrics/dynamic_metrics.py b/src/lighteval/metrics/dynamic_metrics.py index 66ed91c3a..b297f7c92 100644 --- a/src/lighteval/metrics/dynamic_metrics.py +++ b/src/lighteval/metrics/dynamic_metrics.py @@ -185,7 +185,7 @@ def __init__( fallback_mode: Literal["no_fallback", "first_match"] How to perform extraction. Defaults to "first_match". - "no_fallback": Only use first successfully parsed matches - - "first_match": Use the first successfully parsed match + first match irregardless the parsing success + - "first_match": Use the first successfully parsed match + first match regardless of the parsing success extraction_mode: Literal["first_match", "any_match"] - "first_match": Only tries to extract the first regex match if it fails no other matches are tried - "any_match": Tries to extract any regex match diff --git a/src/lighteval/metrics/utils/extractive_match_utils.py b/src/lighteval/metrics/utils/extractive_match_utils.py index da0d718d7..058b9e7ce 100644 --- a/src/lighteval/metrics/utils/extractive_match_utils.py +++ b/src/lighteval/metrics/utils/extractive_match_utils.py @@ -131,7 +131,7 @@ def lazy_expr_regex(expr_config: ExprExtractionConfig, language: Language) -> li currency_units = re.escape("$€£¥₹₽₪₩₫฿₡₢₣₤₥₦₧₨₩₪₫₭₮₯₰₱₲₳₴₵₶₷₸₹₺₻₼₽₾₿") expr_prefix_re = rf"(?:^|{space_re}|\=)(?:\*\*)?" expr_suffix_re = rf"(?:\*\*)?(?:{full_stop_re}|{comma_re}|\s?{colon_re}|{space_re}|\)|\$|$)" - # Expressions must be prefixed and suffixed while, digits don't need suffix and can have currency units preceeded, this is to ensure + # Expressions must be prefixed and suffixed while, digits don't need suffix and can have currency units preceded, this is to ensure # That we can extract stuff like $100 or 100m2, while we don't extract XDY2K as 2 expr_with_anchors = rf"(?:{expr_prefix_re}{expr_re}{expr_suffix_re})" number_with_anchors = rf"(?:{expr_prefix_re}[{currency_units}]?{number_re})" @@ -274,7 +274,7 @@ def lazy_latex_regex(latex_config: LatexExtractionConfig, language: Language) -> latex_re_boxed = rf"{latex_re_boxed}{next_groups}" regexes.append((latex_re_boxed, latex_config.boxed_match_priority)) # Match plain boxed, the issue with plain boxed is that it's impossible to know where it stops, so if there are - # till last }. We do the actuall extraction in the normalization step. + # till last }. We do the actual extraction in the normalization step. regexes.append((r"(?P\\boxed{.+})", latex_config.boxed_match_priority)) return [(re.compile(pattern, re.DOTALL), priority) for pattern, priority in regexes] @@ -402,7 +402,7 @@ def get_target_type_order(target_type: ExtractionTarget) -> int: return extraction_regexes -# Small cache, to catche repeated calls invalid parsing +# Small cache, to catch repeated calls invalid parsing @lru_cache(maxsize=20) @requires("latex2sympy2_extended") def parse_latex_with_timeout(latex: str, timeout_seconds: int): @@ -575,7 +575,7 @@ def extract_target_from_pred( target_res (list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]): List of regex patterns and their priorities for each target type fallback_mode (Literal["no_fallback", "first_match"], optional): How to handle extraction failures. Defaults to "no_fallback". - "no_fallback": Return only successfully parsed match - - "first_match": Additionaly Include the first string match no matter how parsing finished + - "first_match": Additionally include the first string match no matter how parsing finished extraction_mode (Literal["first_match", "any_match"], optional): How to handle extraction failures. Defaults to "any_match". - "first_match": Only tries to extract the first match - "any_match": Tries to extract any match diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py index 989588029..5010a5295 100644 --- a/src/lighteval/metrics/utils/llm_as_judge.py +++ b/src/lighteval/metrics/utils/llm_as_judge.py @@ -384,7 +384,7 @@ async def run_all() -> list[str]: try: loop = asyncio.get_running_loop() - logger.debug("Exting event loop is found, using loop.create_task") + logger.debug("Existing event loop is found, using loop.create_task") result = loop.run_until_complete(run_all()) except RuntimeError: logger.debug("No running event loop found, using asyncio.run") diff --git a/src/lighteval/metrics/utils/math_comparison.py b/src/lighteval/metrics/utils/math_comparison.py index 974d6d2cc..80cacc82c 100644 --- a/src/lighteval/metrics/utils/math_comparison.py +++ b/src/lighteval/metrics/utils/math_comparison.py @@ -57,7 +57,7 @@ def safe_sympy_doit(a: Basic | MatrixBase): """Safely execute doit() on a sympy expression, catching exceptions. - Doit in sympy will evaluate expressions it will pass the expression tree and evluate nodes. + Doit in sympy will evaluate expressions it will pass the expression tree and evaluate nodes. For example for 1+1+1 it will evaluate the additions and return 3. One issue with it is that it maybe evaluates too much as integrals will also be evaluated. diff --git a/src/lighteval/models/endpoints/endpoint_model.py b/src/lighteval/models/endpoints/endpoint_model.py index 6b08be575..7b5500b36 100644 --- a/src/lighteval/models/endpoints/endpoint_model.py +++ b/src/lighteval/models/endpoints/endpoint_model.py @@ -405,7 +405,7 @@ def get_larger_hardware_suggestion(cur_instance_type: str = None, cur_instance_s ) except IndexError: raise Exception( - "To avoid accidental costs, we do not upgrade the current endpoint above 4 a100 automatically, please request it explicitely." + "To avoid accidental costs, we do not upgrade the current endpoint above 4 a100 automatically, please request it explicitly." ) @staticmethod diff --git a/src/lighteval/models/nanotron/nanotron_model.py b/src/lighteval/models/nanotron/nanotron_model.py index 7ed6d35eb..2c78e0d82 100644 --- a/src/lighteval/models/nanotron/nanotron_model.py +++ b/src/lighteval/models/nanotron/nanotron_model.py @@ -397,7 +397,7 @@ def forward_batch(batch_size): logger.warning(f"Testing batch size {batch_size}") test_batch = torch.ones( (batch_size + int(0.1 * batch_size), max_input_length), device=self.device - ).long() # We add 10% for marging :) + ).long() # We add 10% for margin :) F.log_softmax(self._model_call(test_batch).float(), dim=-1).cpu() return batch_size @@ -1025,7 +1025,7 @@ def greedy_until( sample_index, batch = zip(*indexed_batch) # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily + # stop_tokens and max_tokens generated) which is not necessarily # the case! Because of that we should only use batch size of 1 # Since items are sorted by inverse length, the first one always has diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py index 64e790a2f..a425510e0 100644 --- a/src/lighteval/models/transformers/transformers_model.py +++ b/src/lighteval/models/transformers/transformers_model.py @@ -260,7 +260,7 @@ def from_model( if config is None: raise ValueError("Config must be provided to initialize the TransformersModel via `from_model` method.") - # Instanciate the object without using __init__ + # Instantiate the object without using __init__ self = cls.__new__(cls) self.transformers_config = model.config @@ -482,7 +482,7 @@ def _init_max_length(self) -> int: logger.warning( "No max_length attribute found in the model config. Using the default max sequence length setting `2048`. " - "It is recommended to set max_length trough the model args: max_length=..." + "It is recommended to set max_length through the model args: max_length=..." ) return 2048 @@ -556,7 +556,7 @@ def _continuous_greedy_until( stop_tokens = [] else: # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily + # stop_tokens and max_tokens generated) which is not necessarily # the case! Because of that we only use batch size of 1 stop_tokens = split[0].stop_sequence @@ -695,7 +695,7 @@ def _padded_greedy_until( stop_tokens = [self.tokenizer.eos_token] else: # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily + # stop_tokens and max_tokens generated) which is not necessarily # the case! Because of that we only use batch size of 1 stop_tokens = [self.tokenizer.eos_token] + batch[0].stop_sequences diff --git a/src/lighteval/models/transformers/vlm_transformers_model.py b/src/lighteval/models/transformers/vlm_transformers_model.py index 0697ab729..f79e9068e 100644 --- a/src/lighteval/models/transformers/vlm_transformers_model.py +++ b/src/lighteval/models/transformers/vlm_transformers_model.py @@ -333,7 +333,7 @@ def _init_max_length(self) -> int: logger.warning( "No max_length attribute found in the model config. Using the default max sequence length setting `2048`. " - "It is recommended to set max_length trough the model args: max_length=..." + "It is recommended to set max_length through the model args: max_length=..." ) return 2048 diff --git a/src/lighteval/models/utils.py b/src/lighteval/models/utils.py index f615019ea..13a62617e 100644 --- a/src/lighteval/models/utils.py +++ b/src/lighteval/models/utils.py @@ -45,7 +45,7 @@ def _get_dtype(dtype: Union[str, torch.dtype, None], config: Optional[AutoConfig torch.dtype: The torch dtype based on the input arguments. """ if config is not None and hasattr(config, "quantization_config"): - # must be infered + # must be inferred return None if dtype is not None: diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py index 0b719885f..9fd1bd5f4 100644 --- a/src/lighteval/models/vllm/vllm_model.py +++ b/src/lighteval/models/vllm/vllm_model.py @@ -176,7 +176,7 @@ class VLLMModelConfig(ModelConfig): gpu_memory_utilization: NonNegativeFloat = 0.9 # lower this if you are running out of memory enable_prefix_caching: bool = None # whether to enable prefix caching to speed up generation. May use more memory. Should be disabled for LFM2 max_model_length: PositiveInt | None = ( - None # maximum length of the model, ussually infered automatically. reduce this if you encouter OOM issues, 4096 is usually enough + None # maximum length of the model, usually inferred automatically. reduce this if you encounter OOM issues, 4096 is usually enough ) quantization: str | None = None load_format: str | None = None @@ -188,7 +188,7 @@ class VLLMModelConfig(ModelConfig): True # whether to add a space at the start of each continuation in multichoice generation ) pairwise_tokenization: bool = False # whether to tokenize the context and continuation separately or together. - max_num_seqs: PositiveInt = 128 # maximum number of sequences per iteration; This variable and `max_num_batched_tokens` effectively control the batch size at prefill stage. See https://github.com/vllm-project/vllm/issues/2492 for detailed explaination. + max_num_seqs: PositiveInt = 128 # maximum number of sequences per iteration; This variable and `max_num_batched_tokens` effectively control the batch size at prefill stage. See https://github.com/vllm-project/vllm/issues/2492 for detailed explanation. max_num_batched_tokens: PositiveInt = 2048 # maximum number of tokens per batch subfolder: str | None = None is_async: bool = False # Whether to use the async version or sync version of the model @@ -293,7 +293,7 @@ def _create_auto_model(self, config: VLLMModelConfig) -> Optional[LLM]: logger.warning( "The model max_length was not set in the model arguments. Since the model is using data parallelism, it is created later " " with `ray`, so we can't infer the max_length automatically atm. It might raise issues later on: if it does, relaunch your " - "run, but set `max_model_length` explicitely in the model args." + "run, but set `max_model_length` explicitly in the model args." ) return None @@ -352,7 +352,7 @@ def _greedy_until( stop_tokens = [] else: # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily + # stop_tokens and max_tokens generated) which is not necessarily # the case! Because of that we only use batch size of 1 stop_tokens = split[0].stop_sequences or [] diff --git a/src/lighteval/tasks/multilingual/adapters.py b/src/lighteval/tasks/multilingual/adapters.py index d4e2dfc41..afdff4f53 100644 --- a/src/lighteval/tasks/multilingual/adapters.py +++ b/src/lighteval/tasks/multilingual/adapters.py @@ -177,7 +177,7 @@ def agieval_adapter(lang: Language, formulation: Formulation, line: dict) -> MCQ # Here is the most tricky part. In some subsets (e.g. gaokai-history) the answers can be the chinese digits only. # This would break the CF formulation and we thus try to extract the full answers from the question. # Example - # Quesiton: 问题:在中美关系的发展中,台湾问题是一大障碍,在扫除这一障碍的过程中,取得突破性进展的事件包括(  )①中国恢复联合国席位 ②尼克松总统访华③中美两国正式建交 ④邓小平访问美国。 选项:(A)①② (B)①③ (C)②③ (D)③④ 答案:从A到D, 我们应选择 + # Question: 问题:在中美关系的发展中,台湾问题是一大障碍,在扫除这一障碍的过程中,取得突破性进展的事件包括(  )①中国恢复联合国席位 ②尼克松总统访华③中美两国正式建交 ④邓小平访问美国。 选项:(A)①② (B)①③ (C)②③ (D)③④ 答案:从A到D, 我们应选择 # Answers: [ "(A)①②", "(B)①③", "(C)②③", "(D)③④" ] answer_prefixes = [answer.replace("和", "").strip() for answer in cleaned_choices] @@ -204,7 +204,7 @@ def agieval_adapter(lang: Language, formulation: Formulation, line: dict) -> MCQ ) question = question.strip() - # If the answers still only contian the chinese numbers or we have just single choice we discard this sample + # If the answers still only contain the chinese numbers or we have just single choice we discard this sample if ( set("".join(cleaned_choices).replace("和", "").strip()).issubset("①②③④⑤⑥") or len(cleaned_choices) <= 1 diff --git a/src/lighteval/tasks/multilingual/tasks/serbian_eval.py b/src/lighteval/tasks/multilingual/tasks/serbian_eval.py index cb3f54177..b2534b6d2 100644 --- a/src/lighteval/tasks/multilingual/tasks/serbian_eval.py +++ b/src/lighteval/tasks/multilingual/tasks/serbian_eval.py @@ -201,7 +201,7 @@ def serbian_eval_prompt(line: dict, task_name: Optional[str] = None) -> Doc: else: gold_index = int(line["answer"]) - # Show all choises + # Show all choices for i, choice in enumerate(choices): query += f"{i}. {choice}\n" diff --git a/src/lighteval/utils/cache_management.py b/src/lighteval/utils/cache_management.py index 178154613..7f2303a28 100644 --- a/src/lighteval/utils/cache_management.py +++ b/src/lighteval/utils/cache_management.py @@ -194,7 +194,7 @@ def get_cache_path(self, task_id: TaskID) -> Path: return self.cache_dir / task_id.task_name / task_id.task_hash / f"{task_id.sampling_method.name}.parquet" def get_task_id(self, task_name: str, sampling_method: SamplingMethod) -> TaskID: - """Returns a unique task indentifier. Depends on the task name, + """Returns a unique task identifier. Depends on the task name, task version and parameters (from which a hash is derived), and current sampling method (current metric we look at).