diff --git a/mteb/tasks/classification/fra/__init__.py b/mteb/tasks/classification/fra/__init__.py index 774668a735..d11a09e02b 100644 --- a/mteb/tasks/classification/fra/__init__.py +++ b/mteb/tasks/classification/fra/__init__.py @@ -1,10 +1,26 @@ +from .abg_cosqa_fr_classification import FrAbgCosQA +from .air_dialogue_fr_classification import FrAirDialogueClassification +from .clinc_fr_classification import FrClincIntentClassification +from .daily_dialog_fr_classification import ( + FrDailyDialogClassificationAct, + FrDailyDialogClassificationEmotion, +) from .french_book_reviews import FrenchBookReviews, FrenchBookReviewsV2 +from .mantis_fr_classification import FrMantisClassification from .movie_review_sentiment_classification import ( MovieReviewSentimentClassification, MovieReviewSentimentClassificationV2, ) +from .vira_intent_fr_classification import FrViraIntentClassification __all__ = [ + "FrAbgCosQA", + "FrAirDialogueClassification", + "FrClincIntentClassification", + "FrDailyDialogClassificationAct", + "FrDailyDialogClassificationEmotion", + "FrMantisClassification", + "FrViraIntentClassification", "FrenchBookReviews", "FrenchBookReviewsV2", "MovieReviewSentimentClassification", diff --git a/mteb/tasks/classification/fra/abg_cosqa_fr_classification.py b/mteb/tasks/classification/fra/abg_cosqa_fr_classification.py new file mode 100644 index 0000000000..7258f46032 --- /dev/null +++ b/mteb/tasks/classification/fra/abg_cosqa_fr_classification.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import Any + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrAbgCosQA(AbsTaskClassification): + metadata = TaskMetadata( + name="FrAbgCosQA", + description="AbgCosQA", + dataset={ + "path": "DeepPavlov/coqa_abg_fr", + "revision": "c4e23fb92e0f4a6e6d85ca0fcb35c1e96abfa024", + }, + reference="https://huggingface.co/datasets/DeepPavlov/coqa_abg_fr", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="f1", + date=("2021-01-01", "2021-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["AbgCosQA"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None: + def process_history(row: dict[str, Any]) -> dict[str, Any]: + full_text = row["story"] + " " + for turn in row["history_turns"]: + full_text += ( + "User: " + turn["question"] + " Assistant: " + turn["answer"] + " " + ) + full_text += ( + "User: " + + row["target_turn"]["question"] + + " Assistant: " + + row["target_turn"]["answer"] + ) + row["text"] = full_text + row["label"] = row["ambiguity"] == "ambiguous" + return row + + for subset in self.dataset: + self.dataset[subset] = ( + self.dataset[subset] + .map( + process_history, + num_proc=num_proc, + ) + .select_columns(["text", "label"]) + ) diff --git a/mteb/tasks/classification/fra/air_dialogue_fr_classification.py b/mteb/tasks/classification/fra/air_dialogue_fr_classification.py new file mode 100644 index 0000000000..d6e0c4afe3 --- /dev/null +++ b/mteb/tasks/classification/fra/air_dialogue_fr_classification.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import Any + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrAirDialogueClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="FrAirDialogueClassification", + description="AirDialogue is a dataset of goal-oriented customer-agent conversations focused on booking flights under various travel restrictions.", + dataset={ + "path": "DeepPavlov/air_dialog_fr", + "revision": "414ee3c46fd2abc4df245f43a7ac0a33439f97f7", + }, + reference="https://huggingface.co/datasets/google/air_dialogue", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="f1", + date=("2018-01-01", "2022-06-07"), + domains=[], + task_subtypes=["Intent classification"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["AirDialogueClassification"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None: + def process_history(row: dict[str, Any]) -> dict[str, Any]: + history = row["text"] + text = "" + if len(history) > 0: + for entry in history: + if entry["role"] == "user": + text += f"User: {entry['content']}\n" + else: + text += f"Assistant: {entry['content']}\n" + row["text"] = text + return row + + for subset in self.dataset: + self.dataset[subset] = ( + self.dataset[subset] + .map( + process_history, + num_proc=num_proc, + ) + .select_columns(["text", "label"]) + ) diff --git a/mteb/tasks/classification/fra/clinc_fr_classification.py b/mteb/tasks/classification/fra/clinc_fr_classification.py new file mode 100644 index 0000000000..f111b36782 --- /dev/null +++ b/mteb/tasks/classification/fra/clinc_fr_classification.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrClincIntentClassification(AbsTaskClassification): + input_column_name = "utterance" + + metadata = TaskMetadata( + name="FrClincIntentClassification", + description="Task-oriented dialog systems need to know when a query falls outside their range of supported intents, but current text classification corpora only define label sets that cover every example. This is the single-config ('plus'-sized) packaging of CLINC150, DeepPavlov/clinc150_fr, rather than the small/plus/imbalanced multi-config packaging used by ClincIntentClassification.", + dataset={ + "path": "DeepPavlov/clinc150_fr", + "revision": "b5c3e44dc6d605bafb9585ce125b187dcc14e6c9", + }, + reference="https://huggingface.co/datasets/clinc/clinc_oos", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["validation", "test"], + eval_langs=["fra-Latn"], + main_score="accuracy", + date=("2019-01-01", "2019-01-01"), + domains=["Financial", "Web", "Social"], + task_subtypes=["Intent classification"], + license="cc-by-3.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["ClincIntentClassification"], + ) diff --git a/mteb/tasks/classification/fra/daily_dialog_fr_classification.py b/mteb/tasks/classification/fra/daily_dialog_fr_classification.py new file mode 100644 index 0000000000..142bce485f --- /dev/null +++ b/mteb/tasks/classification/fra/daily_dialog_fr_classification.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from typing import Any + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +def combine_dialogs(row: dict) -> dict: + row["dialog"] = "\n".join(row["dialog"]) + return row + + +class FrDailyDialogClassificationAct(AbsTaskClassification): + metadata = TaskMetadata( + name="FrDailyDialogClassificationAct", + description="", + dataset={ + "path": "DeepPavlov/daily_dialog_fr", + "revision": "4e1c4b0878f1bf5d6f41386e08aa9ad1ae787c4d", + }, + reference="https://huggingface.co/datasets/li2017dailydialog/daily_dialog", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["test", "validation"], + eval_langs=["fra-Latn"], + main_score="accuracy", + date=("2017-07-11", "2017-07-11"), + domains=["Social"], + task_subtypes=["Intent classification"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["DailyDialogClassificationAct"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any): + self.dataset = self.dataset.map(combine_dialogs) + self.dataset = self.dataset.rename_columns( + {"act_label": "label", "dialog": "text"} + ) + + +class FrDailyDialogClassificationEmotion(AbsTaskClassification): + metadata = TaskMetadata( + name="FrDailyDialogClassificationEmotion", + description="", + dataset={ + "path": "DeepPavlov/daily_dialog_fr", + "revision": "4e1c4b0878f1bf5d6f41386e08aa9ad1ae787c4d", + }, + reference="https://huggingface.co/datasets/li2017dailydialog/daily_dialog", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["test", "validation"], + eval_langs=["fra-Latn"], + main_score="accuracy", + date=("2017-07-11", "2017-07-11"), + domains=["Social"], + task_subtypes=["Intent classification"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["DailyDialogClassificationEmotion"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any): + self.dataset = self.dataset.map(combine_dialogs) + self.dataset = self.dataset.rename_columns( + {"emotion_label": "label", "dialog": "text"} + ) diff --git a/mteb/tasks/classification/fra/mantis_fr_classification.py b/mteb/tasks/classification/fra/mantis_fr_classification.py new file mode 100644 index 0000000000..eef56d561a --- /dev/null +++ b/mteb/tasks/classification/fra/mantis_fr_classification.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Any + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrMantisClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="FrMantisClassification", + description="Mantis", + dataset={ + "path": "DeepPavlov/mantis_fr", + "revision": "ca6ab868022ff5cee661d6d5f8466dd1f27d83fc", + }, + reference="https://huggingface.co/datasets/DeepPavlov/mantis_fr", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="f1", + date=("2019-01-01", "2019-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["MantisClassification"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None: + def process_history(row: dict[str, Any]) -> dict[str, Any]: + history = row["dialog"] + text = "" + if len(history) > 0: + for entry in history: + if entry["role"] == "user": + text += f"User: {entry['message']}\n" + else: + text += f"Assistant: {entry['message']}\n" + row["text"] = text + return row + + for subset in self.dataset: + self.dataset[subset] = ( + self.dataset[subset] + .map( + process_history, + ) + .rename_column("category", "label") + ) diff --git a/mteb/tasks/classification/fra/vira_intent_fr_classification.py b/mteb/tasks/classification/fra/vira_intent_fr_classification.py new file mode 100644 index 0000000000..3941a2af43 --- /dev/null +++ b/mteb/tasks/classification/fra/vira_intent_fr_classification.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from mteb.abstasks.classification import AbsTaskClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrViraIntentClassification(AbsTaskClassification): + metadata = TaskMetadata( + name="FrViraIntentClassification", + description="Chatbot-delivered COVID-19 vaccine communication message preferences of young adults and public health workers in urban American communities: qualitative study", + dataset={ + "path": "DeepPavlov/vira-intents-live_fr", + "revision": "430391e1240047b603e2e33aab4a446c25769140", + }, + reference="https://huggingface.co/datasets/ibm-research/vira-intents-live", + type="Classification", + category="t2c", + modalities=["text"], + eval_splits=["val", "test"], + eval_langs=["fra-Latn"], + main_score="accuracy", + date=("2020-01-01", "2022-07-06"), + domains=["Medical"], + task_subtypes=["Intent classification"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["ViraIntentClassification"], + ) diff --git a/mteb/tasks/pair_classification/__init__.py b/mteb/tasks/pair_classification/__init__.py index e219bce7af..c613cc809f 100644 --- a/mteb/tasks/pair_classification/__init__.py +++ b/mteb/tasks/pair_classification/__init__.py @@ -4,6 +4,7 @@ from .deu import * from .eng import * from .fas import * +from .fra import * from .hye import * from .ind import * from .ita import * diff --git a/mteb/tasks/pair_classification/fra/__init__.py b/mteb/tasks/pair_classification/fra/__init__.py new file mode 100644 index 0000000000..4b726788ab --- /dev/null +++ b/mteb/tasks/pair_classification/fra/__init__.py @@ -0,0 +1,5 @@ +from .clarqa_fr import FrClarQA + +__all__ = [ + "FrClarQA", +] diff --git a/mteb/tasks/pair_classification/fra/clarqa_fr.py b/mteb/tasks/pair_classification/fra/clarqa_fr.py new file mode 100644 index 0000000000..11f28d4735 --- /dev/null +++ b/mteb/tasks/pair_classification/fra/clarqa_fr.py @@ -0,0 +1,36 @@ +from mteb.abstasks.pair_classification import AbsTaskPairClassification +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrClarQA(AbsTaskPairClassification): + input1_column_name = "entity1" + input2_column_name = "entity2" + label_column_name = "label" + + metadata = TaskMetadata( + name="FrClarQA", + description="ClarQA.", + reference="https://huggingface.co/datasets/DeepPavlov/clarqa_fr", + dataset={ + "path": "DeepPavlov/clarqa_fr", + "revision": "fe5d77ba4762df41c8004619c9af7a7a5d2926ad", + }, + type="PairClassification", + category="t2t", + modalities=["text"], + eval_splits=["test"], + eval_langs={ + "single_turn": ["fra-Latn"], + "multi_turn": ["fra-Latn"], + }, + main_score="max_ap", + date=("2019-01-01", "2019-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["ClarQA"], + ) diff --git a/mteb/tasks/reranking/fra/__init__.py b/mteb/tasks/reranking/fra/__init__.py index 378dcc8730..ef98adbef3 100644 --- a/mteb/tasks/reranking/fra/__init__.py +++ b/mteb/tasks/reranking/fra/__init__.py @@ -1,4 +1,9 @@ from .alloprof_reranking import AlloprofReranking from .syntec_reranking import SyntecReranking +from .web_linx_fr_candidates_reranking import FrWebLINXCandidatesReranking -__all__ = ["AlloprofReranking", "SyntecReranking"] +__all__ = [ + "AlloprofReranking", + "FrWebLINXCandidatesReranking", + "SyntecReranking", +] diff --git a/mteb/tasks/reranking/fra/web_linx_fr_candidates_reranking.py b/mteb/tasks/reranking/fra/web_linx_fr_candidates_reranking.py new file mode 100644 index 0000000000..758fd4bf6c --- /dev/null +++ b/mteb/tasks/reranking/fra/web_linx_fr_candidates_reranking.py @@ -0,0 +1,36 @@ +from mteb.abstasks import AbsTaskReranking +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrWebLINXCandidatesReranking(AbsTaskReranking): + metadata = TaskMetadata( + name="FrWebLINXCandidatesReranking", + description="WebLINX is a large-scale benchmark of 100K interactions across 2300 expert demonstrations of conversational web navigation. The reranking task focuses on finding relevant elements at every given step in the trajectory.", + reference="https://huggingface.co/datasets/DeepPavlov/weblinx_fr", + dataset={ + "path": "DeepPavlov/weblinx_fr", + "revision": "352cf5e4cd129145d0b6a3c0188947647ff6da2c", + }, + type="Reranking", + category="t2t", + modalities=["text"], + eval_splits=[ + "validation", + "test_iid", + "test_cat", + "test_geo", + "test_vis", + "test_web", + ], + eval_langs=["fra-Latn"], + main_score="mrr_at_10", + date=("2023-03-01", "2023-10-30"), + domains=["Academic", "Web", "Written"], + task_subtypes=["Code retrieval", "Conversational retrieval"], + license="cc-by-nc-sa-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["WebLINXCandidatesReranking"], + ) diff --git a/mteb/tasks/retrieval/fra/__init__.py b/mteb/tasks/retrieval/fra/__init__.py index 007c13be88..b77821020d 100644 --- a/mteb/tasks/retrieval/fra/__init__.py +++ b/mteb/tasks/retrieval/fra/__init__.py @@ -1,15 +1,25 @@ from .alloprof_retrieval import AlloprofRetrieval from .bsard_retrieval import BSARDRetrieval, BSARDRetrievalv2 +from .canard_fr_retrieval import FrCanard +from .coral_fr_retrieval import FrCoral from .f_qu_ad_retrieval import FQuADRetrieval +from .faith_dial_fr_retrieval import FrFaithDialRetrieval from .french1_retrieval import French1Retrieval from .french_legal1_retrieval import FrenchLegal1Retrieval from .syntec_retrieval import SyntecRetrieval +from .topiocqa_fr_retrieval import FrTopiOCQARetrieval +from .wizard_of_wikipedia_fr_retrieval import FrWizardOfWikipedia __all__ = [ "AlloprofRetrieval", "BSARDRetrieval", "BSARDRetrievalv2", "FQuADRetrieval", + "FrCanard", + "FrCoral", + "FrFaithDialRetrieval", + "FrTopiOCQARetrieval", + "FrWizardOfWikipedia", "French1Retrieval", "FrenchLegal1Retrieval", "SyntecRetrieval", diff --git a/mteb/tasks/retrieval/fra/canard_fr_retrieval.py b/mteb/tasks/retrieval/fra/canard_fr_retrieval.py new file mode 100644 index 0000000000..41db57f045 --- /dev/null +++ b/mteb/tasks/retrieval/fra/canard_fr_retrieval.py @@ -0,0 +1,29 @@ +from mteb.abstasks.retrieval import AbsTaskRetrieval +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrCanard(AbsTaskRetrieval): + metadata = TaskMetadata( + name="FrCanard", + description="canard", + reference="https://huggingface.co/datasets/DeepPavlov/canard_fr", + dataset={ + "path": "DeepPavlov/canard_fr", + "revision": "e1ea7582a3e62c18262a28935a6a579f45a303e1", + }, + type="Retrieval", + category="t2t", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="ndcg_at_10", + date=("2019-01-01", "2019-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["Canard"], + ) diff --git a/mteb/tasks/retrieval/fra/coral_fr_retrieval.py b/mteb/tasks/retrieval/fra/coral_fr_retrieval.py new file mode 100644 index 0000000000..768f78a459 --- /dev/null +++ b/mteb/tasks/retrieval/fra/coral_fr_retrieval.py @@ -0,0 +1,40 @@ +from typing import Any + +from datasets import Value + +from mteb.abstasks.retrieval import AbsTaskRetrieval +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrCoral(AbsTaskRetrieval): + metadata = TaskMetadata( + name="FrCoral", + description="coral", + reference="https://huggingface.co/datasets/DeepPavlov/coral_fr", + dataset={ + "path": "DeepPavlov/coral_fr", + "revision": "ba6a23690ebf49b5509601d9ff6a0955aa09fc90", + }, + type="Retrieval", + category="t2t", + modalities=["text"], + eval_splits=["test", "train"], + eval_langs=["fra-Latn"], + main_score="ndcg_at_10", + date=("2024-01-01", "2024-12-31"), + domains=[], + task_subtypes=[], + license="mit", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["Coral"], + ) + + def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None: + for subset in self.dataset: + for split in self.dataset[subset]: + self.dataset[subset][split]["corpus"] = self.dataset[subset][split][ + "corpus" + ].cast_column("id", Value("string")) diff --git a/mteb/tasks/retrieval/fra/faith_dial_fr_retrieval.py b/mteb/tasks/retrieval/fra/faith_dial_fr_retrieval.py new file mode 100644 index 0000000000..cb910b53de --- /dev/null +++ b/mteb/tasks/retrieval/fra/faith_dial_fr_retrieval.py @@ -0,0 +1,72 @@ +from datasets import load_dataset + +from mteb.abstasks import AbsTaskRetrieval +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrFaithDialRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="FrFaithDialRetrieval", + dataset={ + "path": "DeepPavlov/faithdial_fr", + "revision": "172f3344378327b5798c7902526a2412bad5e8a4", + }, + reference="https://mcgill-nlp.github.io/FaithDial", + description=( + "FaithDial is a faithful knowledge-grounded dialogue benchmark." + + "It was curated by asking annotators to amend hallucinated utterances in Wizard of Wikipedia (WoW). " + + "It consists of conversation histories along with manually labelled relevant passage. " + + "For the purpose of retrieval, we only consider the instances marked as 'Edification' in the VRM field, " + + "as the gold passage associated with these instances is non-ambiguous." + ), + type="Retrieval", + category="t2t", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="ndcg_at_10", + date=("2022-01-01", "2022-03-31"), + domains=["Encyclopaedic", "Written"], + task_subtypes=["Conversational retrieval"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["FaithDial"], + ) + + # Mirrors RuFaithDialRetrieval's construction: DeepPavlov/faithdial_fr is the raw + # dialogue format (not a pre-built corpus/queries/qrels retrieval dataset), so we + # build the retrieval triples ourselves here, using the "_fr" (translated) columns. + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + for split in kwargs.get("eval_splits", self.metadata.eval_splits): + corpus, queries, qrels = self._load_data_for_split(split) + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def _load_data_for_split(self, split): + ds = load_dataset(split=split, **self.metadata.dataset) + queries, corpus, qrels = {}, {}, {} + for i, sample in enumerate(ds): + # document is added to corpus for all samples + doc_id = "doc:" + str(i) + corpus[doc_id] = { + "title": "", # title is not available + "text": sample["knowledge_fr"], + } + if "Edification" in sample["VRM"]: + query_id = "query:" + str(i) + query = sample["history_fr"] + queries[query_id] = query + qrels[query_id] = {doc_id: 1} + + return corpus, queries, qrels diff --git a/mteb/tasks/retrieval/fra/topiocqa_fr_retrieval.py b/mteb/tasks/retrieval/fra/topiocqa_fr_retrieval.py new file mode 100644 index 0000000000..5aabdc90e8 --- /dev/null +++ b/mteb/tasks/retrieval/fra/topiocqa_fr_retrieval.py @@ -0,0 +1,86 @@ +from datasets import load_dataset + +from mteb.abstasks import AbsTaskRetrieval +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrTopiOCQARetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="FrTopiOCQARetrieval", + dataset={ + "path": "DeepPavlov/topiocqa_fr", + "revision": "3cb6238bcd5876f79a25ef482ac61c63c3fbaf09", + }, + reference="https://mcgill-nlp.github.io/topiocqa", + description=( + "TopiOCQA (Human-in-the-loop Attributable Generative Retrieval for Information-seeking Dataset) " + + "is information-seeking conversational dataset with challenging topic switching phenomena. " + + "It consists of conversation histories along with manually labelled relevant/gold passage. " + + "Unlike the pre-built mteb/TopiOCQA corpus/queries/qrels split, DeepPavlov/topiocqa_fr ships the " + + "raw per-turn QA rows (Conversation_no, Turn_no, Question, Answer, Context, ...), so this task " + + "builds the retrieval triples itself: for each turn, the query is the conversation history up to " + + "and including that turn's Question, and the corpus document is that turn's gold Context " + + "passage(s), one document per turn, following the same construction pattern used for " + + "FrFaithDialRetrieval." + ), + type="Retrieval", + category="t2t", + modalities=["text"], + eval_splits=["validation"], + eval_langs=["fra-Latn"], + main_score="ndcg_at_10", + date=("2021-03-01", "2021-07-31"), + domains=["Encyclopaedic", "Written"], + task_subtypes=["Conversational retrieval"], + license="cc-by-nc-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["TopiOCQA"], + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + self.corpus, self.queries, self.relevant_docs = {}, {}, {} + for split in kwargs.get("eval_splits", self.metadata.eval_splits): + corpus, queries, qrels = self._load_data_for_split(split) + self.corpus[split], self.queries[split], self.relevant_docs[split] = ( + corpus, + queries, + qrels, + ) + + self.data_loaded = True + + def _load_data_for_split(self, split): + ds = load_dataset(split=split, **self.metadata.dataset) + ds = ds.sort(["Conversation_no", "Turn_no"]) + + queries, corpus, qrels = {}, {}, {} + history_by_conversation: dict[int, str] = {} + for i, sample in enumerate(ds): + conv_no = sample["Conversation_no"] + history = history_by_conversation.get(conv_no, "") + query_text = ( + f"{history}\nUser: {sample['Question']}" + if history + else f"User: {sample['Question']}" + ) + + doc_id = "doc:" + str(i) + corpus[doc_id] = { + "title": sample["Topic"] or "", + "text": " ".join(sample["Context"]), + } + + query_id = "query:" + str(i) + queries[query_id] = query_text + qrels[query_id] = {doc_id: 1} + + history_by_conversation[conv_no] = ( + f"{query_text}\nAssistant: {sample['Answer']}" + ) + + return corpus, queries, qrels diff --git a/mteb/tasks/retrieval/fra/wizard_of_wikipedia_fr_retrieval.py b/mteb/tasks/retrieval/fra/wizard_of_wikipedia_fr_retrieval.py new file mode 100644 index 0000000000..ee3bee0626 --- /dev/null +++ b/mteb/tasks/retrieval/fra/wizard_of_wikipedia_fr_retrieval.py @@ -0,0 +1,29 @@ +from mteb.abstasks.retrieval import AbsTaskRetrieval +from mteb.abstasks.task_metadata import TaskMetadata + + +class FrWizardOfWikipedia(AbsTaskRetrieval): + metadata = TaskMetadata( + name="FrWizardOfWikipedia", + description="WizardOfWikipedia", + reference="https://huggingface.co/datasets/DeepPavlov/wizard_of_wikipedia_fr", + dataset={ + "path": "DeepPavlov/wizard_of_wikipedia_fr", + "revision": "0c37af2e1d0e776a8d63d86b42fc4f90b19a2811", + }, + type="Retrieval", + category="t2t", + modalities=["text"], + eval_splits=["test"], + eval_langs=["fra-Latn"], + main_score="ndcg_at_10", + date=("2019-01-01", "2019-12-31"), + domains=[], + task_subtypes=[], + license="not specified", + annotations_creators="derived", + dialect=[], + sample_creation="machine-translated and verified", + bibtex_citation="", + adapted_from=["WiardOfWikipedia"], + )