Skip to content

Commit 7caa598

Browse files
ZiyiXiaZiyiXia
committed
eval abc
1 parent db20af3 commit 7caa598

8 files changed

Lines changed: 91 additions & 24 deletions

File tree

‎FlagEmbedding/abc/evaluation/data_loader.py‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ class AbsEvalDataLoader(ABC):
1717
1818
Args:
1919
eval_name (str): The experiment name of current evaluation.
20-
dataset_dir (str, optional): path to the datasets. Defaults to :data:`None`.
21-
cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to :data:`None`.
22-
token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to :data:`None`.
23-
force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to :data:`False`.
20+
dataset_dir (str, optional): path to the datasets. Defaults to ``None``.
21+
cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to ``None``.
22+
token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to ``None``.
23+
force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to ``False``.
2424
"""
2525
def __init__(
2626
self,
@@ -78,7 +78,7 @@ def check_splits(self, splits: Union[str, List[str]], dataset_name: Optional[str
7878
7979
Args:
8080
splits (Union[str, List[str]]): Splits to check.
81-
dataset_name (Optional[str], optional): Name of dataset to check. Defaults to None.
81+
dataset_name (Optional[str], optional): Name of dataset to check. Defaults to ``None``.
8282
8383
Returns:
8484
List[str]: The available splits.
@@ -98,7 +98,7 @@ def load_corpus(self, dataset_name: Optional[str] = None) -> datasets.DatasetDic
9898
"""Load the corpus from the dataset.
9999
100100
Args:
101-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
101+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
102102
103103
Returns:
104104
datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
@@ -116,8 +116,8 @@ def load_qrels(self, dataset_name: Optional[str] = None, split: str = 'test') ->
116116
"""Load the qrels from the dataset.
117117
118118
Args:
119-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
120-
split (str, optional): The split to load relevance from. Defaults to :data:'test'.
119+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
120+
split (str, optional): The split to load relevance from. Defaults to ``'test'``.
121121
122122
Raises:
123123
ValueError
@@ -144,8 +144,8 @@ def load_queries(self, dataset_name: Optional[str] = None, split: str = 'test')
144144
"""Load the queries from the dataset.
145145
146146
Args:
147-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
148-
split (str, optional): The split to load queries from. Defaults to :data:`'test'`.
147+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
148+
split (str, optional): The split to load queries from. Defaults to ``'test'``.
149149
150150
Raises:
151151
ValueError
@@ -176,8 +176,8 @@ def _load_remote_corpus(
176176
"""Abstract method to load corpus from remote dataset, to be overrode in child class.
177177
178178
Args:
179-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
180-
save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to :data:`None`.
179+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
180+
save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to ``None``.
181181
182182
Raises:
183183
NotImplementedError: Loading remote corpus is not implemented.
@@ -196,9 +196,9 @@ def _load_remote_qrels(
196196
"""Abstract method to load relevance from remote dataset, to be overrode in child class.
197197
198198
Args:
199-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
200-
split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`.
201-
save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to :data:`None`.
199+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
200+
split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``.
201+
save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to ``None``.
202202
203203
Raises:
204204
NotImplementedError: Loading remote qrels is not implemented.
@@ -217,9 +217,9 @@ def _load_remote_queries(
217217
"""Abstract method to load queries from remote dataset, to be overrode in child class.
218218
219219
Args:
220-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
221-
split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`.
222-
save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to :data:`None`.
220+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
221+
split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``.
222+
save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to ``None``.
223223
224224
Raises:
225225
NotImplementedError
@@ -234,7 +234,7 @@ def _load_local_corpus(self, save_dir: str, dataset_name: Optional[str] = None)
234234
235235
Args:
236236
save_dir (str): Path to save the loaded corpus.
237-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
237+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
238238
239239
Returns:
240240
datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
@@ -257,8 +257,8 @@ def _load_local_qrels(self, save_dir: str, dataset_name: Optional[str] = None, s
257257
258258
Args:
259259
save_dir (str): Path to save the loaded relevance.
260-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
261-
split (str, optional): Split to load from the local dataset. Defaults to :data:`'test'`.
260+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
261+
split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
262262
263263
Raises:
264264
ValueError
@@ -292,8 +292,8 @@ def _load_local_queries(self, save_dir: str, dataset_name: Optional[str] = None,
292292
293293
Args:
294294
save_dir (str): Path to save the loaded queries.
295-
dataset_name (Optional[str], optional): Name of the dataset. Defaults to None.
296-
split (str, optional): Split to load from the local dataset. Defaults to 'test'.
295+
dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
296+
split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
297297
298298
Raises:
299299
ValueError

‎FlagEmbedding/abc/evaluation/evaluator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ def evaluate_results(
357357
k_values (List[int], optional): Cutoffs. Defaults to :data:`[1, 3, 5, 10, 100, 1000]`.
358358
359359
Returns:
360-
_type_: _description_
360+
dict: Evaluation results.
361361
"""
362362
eval_results_dict = {}
363363

‎docs/source/API/abc/evaluation.rst‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Evaluation
2+
==========
3+
4+
.. toctree::
5+
evaluation/arguments
6+
evaluation/data_loader
7+
evaluation/searcher
8+
evaluation/evaluator
9+
evaluation/runner

‎docs/source/API/abc/evaluation/arguments.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Arguments
2+
=========
3+
4+
.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalArgs
5+
6+
7+
.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalModelArgs

‎docs/source/API/abc/evaluation/data_loader.rst‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
dataset loader
2+
==============
3+
4+
.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader
5+
6+
Methods
7+
-------
8+
9+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_dataset_names
10+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_splits
11+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_dataset_names
12+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_splits
13+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_corpus
14+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_qrels
15+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_queries
16+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_corpus
17+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_qrels
18+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_queries
19+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_corpus
20+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_qrels
21+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_queries
22+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_file
23+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._get_fpath_size
24+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_gz_file
25+
.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_zip_file

‎docs/source/API/abc/evaluation/evaluator.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Evaluator
2+
=========
3+
4+
.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvaluator

‎docs/source/API/abc/evaluation/runner.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
runner
2+
======
3+
4+
.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalRunner

‎docs/source/API/abc/evaluation/searcher.rst‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
========
2+
searcher
3+
========
4+
5+
EvalRetriever
6+
=============
7+
8+
.. autoclass:: FlagEmbedding.abc.evaluation.EvalRetriever
9+
10+
EvalDenseRetriever
11+
==================
12+
13+
.. autoclass:: FlagEmbedding.abc.evaluation.EvalDenseRetriever
14+
15+
EvalReranker
16+
============
17+
18+
.. autoclass:: FlagEmbedding.abc.evaluation.EvalReranker

0 commit comments

Comments
 (0)