add fra tasks

mteb/tasks/classification/fra/__init__.py

@@ -1,10 +1,26 @@
+from .abg_cosqa_fr_classification import FrAbgCosQA
+from .air_dialogue_fr_classification import FrAirDialogueClassification
+from .clinc_fr_classification import FrClincIntentClassification
+from .daily_dialog_fr_classification import (
+    FrDailyDialogClassificationAct,
+    FrDailyDialogClassificationEmotion,
+)
 from .french_book_reviews import FrenchBookReviews, FrenchBookReviewsV2
+from .mantis_fr_classification import FrMantisClassification
 from .movie_review_sentiment_classification import (
     MovieReviewSentimentClassification,
     MovieReviewSentimentClassificationV2,
 )
+from .vira_intent_fr_classification import FrViraIntentClassification
 
 __all__ = [
+    "FrAbgCosQA",
+    "FrAirDialogueClassification",
+    "FrClincIntentClassification",
+    "FrDailyDialogClassificationAct",
+    "FrDailyDialogClassificationEmotion",
+    "FrMantisClassification",
+    "FrViraIntentClassification",
     "FrenchBookReviews",
     "FrenchBookReviewsV2",
     "MovieReviewSentimentClassification",

mteb/tasks/classification/fra/abg_cosqa_fr_classification.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import Any
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrAbgCosQA(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrAbgCosQA",
+        description="AbgCosQA",
+        dataset={
+            "path": "DeepPavlov/coqa_abg_fr",
+            "revision": "c4e23fb92e0f4a6e6d85ca0fcb35c1e96abfa024",
+        },
+        reference="https://huggingface.co/datasets/DeepPavlov/coqa_abg_fr",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["fra-Latn"],
+        main_score="f1",
+        date=("2021-01-01", "2021-12-31"),
+        domains=[],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["AbgCosQA"],
+    )
+
+    def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None:
+        def process_history(row: dict[str, Any]) -> dict[str, Any]:
+            full_text = row["story"] + " "
+            for turn in row["history_turns"]:
+                full_text += (
+                    "User: " + turn["question"] + " Assistant: " + turn["answer"] + " "
+                )
+            full_text += (
+                "User: "
+                + row["target_turn"]["question"]
+                + " Assistant: "
+                + row["target_turn"]["answer"]
+            )
+            row["text"] = full_text
+            row["label"] = row["ambiguity"] == "ambiguous"
+            return row
+
+        for subset in self.dataset:
+            self.dataset[subset] = (
+                self.dataset[subset]
+                .map(
+                    process_history,
+                    num_proc=num_proc,
+                )
+                .select_columns(["text", "label"])
+            )

mteb/tasks/classification/fra/air_dialogue_fr_classification.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from typing import Any
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrAirDialogueClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrAirDialogueClassification",
+        description="AirDialogue is a dataset of goal-oriented customer-agent conversations focused on booking flights under various travel restrictions.",
+        dataset={
+            "path": "DeepPavlov/air_dialog_fr",
+            "revision": "414ee3c46fd2abc4df245f43a7ac0a33439f97f7",
+        },
+        reference="https://huggingface.co/datasets/google/air_dialogue",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["fra-Latn"],
+        main_score="f1",
+        date=("2018-01-01", "2022-06-07"),
+        domains=[],
+        task_subtypes=["Intent classification"],
+        license="not specified",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["AirDialogueClassification"],
+    )
+
+    def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None:
+        def process_history(row: dict[str, Any]) -> dict[str, Any]:
+            history = row["text"]
+            text = ""
+            if len(history) > 0:
+                for entry in history:
+                    if entry["role"] == "user":
+                        text += f"User: {entry['content']}\n"
+                    else:
+                        text += f"Assistant: {entry['content']}\n"
+            row["text"] = text
+            return row
+
+        for subset in self.dataset:
+            self.dataset[subset] = (
+                self.dataset[subset]
+                .map(
+                    process_history,
+                    num_proc=num_proc,
+                )
+                .select_columns(["text", "label"])
+            )

mteb/tasks/classification/fra/clinc_fr_classification.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrClincIntentClassification(AbsTaskClassification):
+    input_column_name = "utterance"
+
+    metadata = TaskMetadata(
+        name="FrClincIntentClassification",
+        description="Task-oriented dialog systems need to know when a query falls outside their range of supported intents, but current text classification corpora only define label sets that cover every example. This is the single-config ('plus'-sized) packaging of CLINC150, DeepPavlov/clinc150_fr, rather than the small/plus/imbalanced multi-config packaging used by ClincIntentClassification.",
+        dataset={
+            "path": "DeepPavlov/clinc150_fr",
+            "revision": "b5c3e44dc6d605bafb9585ce125b187dcc14e6c9",
+        },
+        reference="https://huggingface.co/datasets/clinc/clinc_oos",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["validation", "test"],
+        eval_langs=["fra-Latn"],
+        main_score="accuracy",
+        date=("2019-01-01", "2019-01-01"),
+        domains=["Financial", "Web", "Social"],
+        task_subtypes=["Intent classification"],
+        license="cc-by-3.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["ClincIntentClassification"],
+    )

mteb/tasks/classification/fra/daily_dialog_fr_classification.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from typing import Any
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+def combine_dialogs(row: dict) -> dict:
+    row["dialog"] = "\n".join(row["dialog"])
+    return row
+
+
+class FrDailyDialogClassificationAct(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrDailyDialogClassificationAct",
+        description="",
+        dataset={
+            "path": "DeepPavlov/daily_dialog_fr",
+            "revision": "4e1c4b0878f1bf5d6f41386e08aa9ad1ae787c4d",
+        },
+        reference="https://huggingface.co/datasets/li2017dailydialog/daily_dialog",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test", "validation"],
+        eval_langs=["fra-Latn"],
+        main_score="accuracy",
+        date=("2017-07-11", "2017-07-11"),
+        domains=["Social"],
+        task_subtypes=["Intent classification"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["DailyDialogClassificationAct"],
+    )
+
+    def dataset_transform(self, num_proc: int | None = None, **kwargs: Any):
+        self.dataset = self.dataset.map(combine_dialogs)
+        self.dataset = self.dataset.rename_columns(
+            {"act_label": "label", "dialog": "text"}
+        )
+
+
+class FrDailyDialogClassificationEmotion(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrDailyDialogClassificationEmotion",
+        description="",
+        dataset={
+            "path": "DeepPavlov/daily_dialog_fr",
+            "revision": "4e1c4b0878f1bf5d6f41386e08aa9ad1ae787c4d",
+        },
+        reference="https://huggingface.co/datasets/li2017dailydialog/daily_dialog",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test", "validation"],
+        eval_langs=["fra-Latn"],
+        main_score="accuracy",
+        date=("2017-07-11", "2017-07-11"),
+        domains=["Social"],
+        task_subtypes=["Intent classification"],
+        license="cc-by-nc-sa-4.0",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["DailyDialogClassificationEmotion"],
+    )
+
+    def dataset_transform(self, num_proc: int | None = None, **kwargs: Any):
+        self.dataset = self.dataset.map(combine_dialogs)
+        self.dataset = self.dataset.rename_columns(
+            {"emotion_label": "label", "dialog": "text"}
+        )

mteb/tasks/classification/fra/mantis_fr_classification.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import Any
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrMantisClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrMantisClassification",
+        description="Mantis",
+        dataset={
+            "path": "DeepPavlov/mantis_fr",
+            "revision": "ca6ab868022ff5cee661d6d5f8466dd1f27d83fc",
+        },
+        reference="https://huggingface.co/datasets/DeepPavlov/mantis_fr",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs=["fra-Latn"],
+        main_score="f1",
+        date=("2019-01-01", "2019-12-31"),
+        domains=[],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["MantisClassification"],
+    )
+
+    def dataset_transform(self, num_proc: int | None = None, **kwargs: Any) -> None:
+        def process_history(row: dict[str, Any]) -> dict[str, Any]:
+            history = row["dialog"]
+            text = ""
+            if len(history) > 0:
+                for entry in history:
+                    if entry["role"] == "user":
+                        text += f"User: {entry['message']}\n"
+                    else:
+                        text += f"Assistant: {entry['message']}\n"
+            row["text"] = text
+            return row
+
+        for subset in self.dataset:
+            self.dataset[subset] = (
+                self.dataset[subset]
+                .map(
+                    process_history,
+                )
+                .rename_column("category", "label")
+            )

mteb/tasks/classification/fra/vira_intent_fr_classification.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from mteb.abstasks.classification import AbsTaskClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrViraIntentClassification(AbsTaskClassification):
+    metadata = TaskMetadata(
+        name="FrViraIntentClassification",
+        description="Chatbot-delivered COVID-19 vaccine communication message preferences of young adults and public health workers in urban American communities: qualitative study",
+        dataset={
+            "path": "DeepPavlov/vira-intents-live_fr",
+            "revision": "430391e1240047b603e2e33aab4a446c25769140",
+        },
+        reference="https://huggingface.co/datasets/ibm-research/vira-intents-live",
+        type="Classification",
+        category="t2c",
+        modalities=["text"],
+        eval_splits=["val", "test"],
+        eval_langs=["fra-Latn"],
+        main_score="accuracy",
+        date=("2020-01-01", "2022-07-06"),
+        domains=["Medical"],
+        task_subtypes=["Intent classification"],
+        license="not specified",
+        annotations_creators="human-annotated",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["ViraIntentClassification"],
+    )

mteb/tasks/pair_classification/__init__.py

@@ -4,6 +4,7 @@
 from .deu import *
 from .eng import *
 from .fas import *
+from .fra import *
 from .hye import *
 from .ind import *
 from .ita import *

mteb/tasks/pair_classification/fra/__init__.py

@@ -0,0 +1,5 @@
+from .clarqa_fr import FrClarQA
+
+__all__ = [
+    "FrClarQA",
+]

mteb/tasks/pair_classification/fra/clarqa_fr.py

@@ -0,0 +1,36 @@
+from mteb.abstasks.pair_classification import AbsTaskPairClassification
+from mteb.abstasks.task_metadata import TaskMetadata
+
+
+class FrClarQA(AbsTaskPairClassification):
+    input1_column_name = "entity1"
+    input2_column_name = "entity2"
+    label_column_name = "label"
+
+    metadata = TaskMetadata(
+        name="FrClarQA",
+        description="ClarQA.",
+        reference="https://huggingface.co/datasets/DeepPavlov/clarqa_fr",
+        dataset={
+            "path": "DeepPavlov/clarqa_fr",
+            "revision": "fe5d77ba4762df41c8004619c9af7a7a5d2926ad",
+        },
+        type="PairClassification",
+        category="t2t",
+        modalities=["text"],
+        eval_splits=["test"],
+        eval_langs={
+            "single_turn": ["fra-Latn"],
+            "multi_turn": ["fra-Latn"],
+        },
+        main_score="max_ap",
+        date=("2019-01-01", "2019-12-31"),
+        domains=[],
+        task_subtypes=[],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        sample_creation="machine-translated and verified",
+        bibtex_citation="",
+        adapted_from=["ClarQA"],
+    )