From 0fb9768bbcc308e50e95cfe86e0b1a8f353d28f7 Mon Sep 17 00:00:00 2001 From: W3en2g Date: Tue, 1 Apr 2025 20:29:32 +0800 Subject: [PATCH 1/2] feat: new benchmark OlympiadBench --- eval_anything/benchmarks/__init__.py | 1 + .../olympiadbench/configs.yaml | 78 +++++++++ .../text_image_to_text/olympiadbench/eval.py | 59 +++++++ eval_anything/configs/evaluate.yaml | 9 +- eval_anything/dataloader/format_mm_dataset.py | 148 ++++++++++++++++++ eval_anything/evaluate_tools/t2t_tools.py | 2 +- eval_anything/utils/utils.py | 1 + 7 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml create mode 100644 eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py diff --git a/eval_anything/benchmarks/__init__.py b/eval_anything/benchmarks/__init__.py index 31d60f2..d726693 100644 --- a/eval_anything/benchmarks/__init__.py +++ b/eval_anything/benchmarks/__init__.py @@ -12,6 +12,7 @@ "benchmarks.text_to_text.CEval.eval", "benchmarks.text_image_to_text.mmmu.eval", "benchmarks.text_image_to_text.mathvision.eval", + "benchmarks.text_image_to_text.olympiadbench.eval", ] for module in benchmark_modules: diff --git a/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml b/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml new file mode 100644 index 0000000..ee2270f --- /dev/null +++ b/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml @@ -0,0 +1,78 @@ +# ============================================ +# 评测配置： +# - 数据集hf路径 +# - 数据集split +# - 数据集size +# - 数据集模态 +# - 评测方式（multiple choice / generation / ...） +# - 评测指标（accuracy / ...） +# - 是否支持few_shot +# - 选项编号形式（ABCD或1234...），判断对错时的标识(true/false或者1/0...) +# ============================================ +dataset: + name: olympiadbench + path: Hothan/OlympiadBench + split: train + size: + modality: text-image-to-text + fewshot_data_path: null + fewshot_data_name: null + fewshot_data_split: null + cot_fewshot_data_path: null + cot_fewshot_data_name: null + cot_fewshot_data_split: null + max_shot: 0 + default_task_list: ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE", "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE",] +task_defaults: &task_defaults + type: MultiChoice + question_key: question + answer_key: options + ground_truth_key: answer + candidate_labels: ["A", "B", "C", "D", "E"] + avalable_evaluate_tools: ["match_multi-choice_and_open-ended"] +task: + - name: OE_MM_maths_en_COMP + data_files: null + <<: *task_defaults + - name: OE_MM_maths_zh_CEE + data_files: null + <<: *task_defaults + - name: OE_MM_maths_zh_COMP + data_files: null + <<: *task_defaults + - name: OE_MM_physics_en_COMP + data_files: null + <<: *task_defaults + - name: OE_MM_physics_zh_CEE + data_files: null + <<: *task_defaults + - name: OE_TO_maths_en_COMP + data_files: null + <<: *task_defaults + - name: OE_TO_maths_zh_CEE + data_files: null + <<: *task_defaults + - name: OE_TO_maths_zh_COMP + data_files: null + <<: *task_defaults + - name: OE_TO_physics_en_COMP + data_files: null + <<: *task_defaults + - name: OE_TO_physics_zh_CEE + data_files: null + <<: *task_defaults +answer_extractor: + - name: match_multi-choice_and_open-ended + function: regex_match_latex_math + args: + match_index: -1 # Index for multiple choice +metrics: + - name: accuracy + function: accuracy + args: +overall_metrics: + - name: average + function: average_across_tasks + args: + null + \ No newline at end of file diff --git a/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py b/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py new file mode 100644 index 0000000..53785bd --- /dev/null +++ b/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py @@ -0,0 +1,59 @@ +""" +继承自eval-anything.pipeline.mm_und_benchmark +""" +from eval_anything.pipeline.mm_und_benchmark import MMUndBenchmark +from collections import namedtuple +from eval_anything.utils.data_type import EvaluationResult +from eval_anything.models.base_model import BaseModel +from eval_anything.utils.logger import EvalLogger +from eval_anything.utils.register import BenchmarkRegistry +from eval_anything.utils.cache_manager import CacheManager + +@BenchmarkRegistry.register('olympiadbench') +class olympiadbenchBenchmark(MMUndBenchmark): + def __init__(self, + model: BaseModel, + eval_cfgs: namedtuple, + model_cfgs: namedtuple, + infer_cfgs: namedtuple, + output_path: str, + cache_manager: CacheManager, + logger: EvalLogger): + super().__init__(model, eval_cfgs, model_cfgs, infer_cfgs, output_path, cache_manager, logger) + self.benchmark_name = "olympiadbench" + self.benchmark_cfgs = self.get_benchmark_cfgs(self.benchmark_name) + + def run(self, + task_list: list[str],) -> tuple[dict[str, list[EvaluationResult]], dict[str, dict[str, float]], dict[str, dict[str, float]]]: + """Run benchmark + Args: + task_list (list[str]): task list + + Returns: + evaluation_details (dict[str, list[EvaluationResult]]): evaluation details + evaluation_results (dict[str, dict[str, float]]): evaluation results + """ + + self.logger.log('info', f'Evaluating {self.benchmark_name}...') + + dataloader = self.init_dataloader(self.eval_cfgs, self.benchmark_cfgs) + input_data = dataloader.load_dataset(task_list) # Input_data: list[InferenceInput] + + inference_outputs = self.batch_inference(self.model, input_data) + ref_answers = {task: self.get_ref_answer(input_data[task], inference_outputs[task]) for task in task_list} + evaluation_details = {} + evaluation_results = {} + + for task in task_list: + evaluation_details[task], evaluation_results[task] = self.calculate_metrics(self.benchmark_name, inference_outputs[task], ref_answers[task], self.benchmark_cfgs.answer_extractor, self.benchmark_cfgs.metrics, judge_method="judge_equal_list") + + if len(task_list) > 1: + overall_result = self.calculate_overall_metrics(self.benchmark_cfgs.overall_metrics, result=evaluation_results) + else: + overall_result = {} + + self.display_benchmark_results(self.benchmark_name, evaluation_results) + if overall_result != {}: + self.display_benchmark_results(self.benchmark_name, overall_result) + self.save_benchmark_details(self.output_path, self.benchmark_name, input_data, evaluation_details) + return evaluation_details, evaluation_results, overall_result \ No newline at end of file diff --git a/eval_anything/configs/evaluate.yaml b/eval_anything/configs/evaluate.yaml index c67049f..08f2f32 100644 --- a/eval_anything/configs/evaluate.yaml +++ b/eval_anything/configs/evaluate.yaml @@ -19,6 +19,7 @@ "CEval": ['accountant', 'advanced_mathematics', 'art_studies', 'basic_medicine', 'business_administration', 'chinese_language_and_literature', 'civil_servant', 'clinical_medicine', 'college_chemistry', 'college_economics', 'college_physics', 'college_programming', 'computer_architecture', 'computer_network', 'discrete_mathematics', 'education_science', 'electrical_engineer', 'environmental_impact_assessment_engineer', 'fire_engineer', 'high_school_biology', 'high_school_chemistry', 'high_school_chinese', 'high_school_geography', 'high_school_history', 'high_school_mathematics', 'high_school_physics', 'high_school_politics', 'ideological_and_moral_cultivation', 'law', 'legal_professional', 'logic', 'mao_zedong_thought', 'marxism', 'metrology_engineer', 'middle_school_biology', 'middle_school_chemistry', 'middle_school_geography', 'middle_school_history', 'middle_school_mathematics', 'middle_school_physics', 'middle_school_politics', 'modern_chinese_history', 'operating_system', 'physician', 'plant_protection', 'probability_and_statistics', 'professional_tour_guide', 'sports_science', 'tax_accountant', 'teacher_qualification', 'urban_and_rural_planner', 'veterinary_medicine'], "mmmu": ["Accounting"], "mathvision": ["default"], + "olympiadbench": ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE", "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE"], } # Num shot n_shot: { @@ -33,6 +34,7 @@ "CEval": 5, "mmmu": 5, "mathvision": 0, + "olympiadbench": 0, } # Chain of thought cot: { @@ -47,6 +49,7 @@ "CEval": False, "mmmu": False, "mathvision": False, + "olympiadbench": False, } # Visualization visualization: @@ -56,13 +59,13 @@ # The model configurations model_cfgs: # Pretrained model unique identity - model_id: Llava-1.5-7B-HF + model_id: Qwen2-VL-7B-Instruct # Pretrained model name or path - model_name_or_path: /aifs4su/yaodong/models/llava-1.5-7b-hf + model_name_or_path: Qwen/Qwen2-VL-7B-Instruct # Model type ("LM" or "MM") model_type: "MM" # Chat template - chat_template: Llava + chat_template: Qwen2-VL infer_cfgs: # Inference backend infer_backend: "vllm" diff --git a/eval_anything/dataloader/format_mm_dataset.py b/eval_anything/dataloader/format_mm_dataset.py index 78c3cee..f589397 100644 --- a/eval_anything/dataloader/format_mm_dataset.py +++ b/eval_anything/dataloader/format_mm_dataset.py @@ -147,4 +147,152 @@ def _to_InferenceInput(self, dataset: Dataset) -> List["InferenceInput"]: ref_answer=item['answer'] ) ) + return inference_inputs + +@MMDatasetRegistry.register("olympiadbench") +class olympiadbenchDataset(BaseMMDataset): + def __init__(self, bench_cfgs: namedtuple, task: namedtuple, enable_cot: bool, num_shot: int): + super().__init__(bench_cfgs, task, enable_cot, num_shot) + + def set_few_shot_examples(self, few_shot_dataset: Dataset | None): + raise NotImplementedError("olympiadbench does not support few-shot learning.") + + def get_image_indice(self, text: str)->List[int]: + pattern = r'' + matches = re.findall(pattern, text) + return [int(num) for num in matches] + + + # refer: https://github.com/mathllm/MATH-V/blob/main/models/Qwen-VL.py#L19 + def _to_InferenceInput(self, dataset: Dataset) -> List["InferenceInput"]: + """ + Convert a dataset to a list of InferenceInput objects. + + Args: + dataset: Dataset object containing questions, options, and images + + Returns: + List of InferenceInput objects ready for model inference + """ + + # refer: https://github.com/OpenBMB/OlympiadBench/blob/main/inference/code/evaluators/evaluator.py + chinese_answer_type_dict = { + 'Numerical': '数值', + 'Expression': '表达式', + 'Equation': '方程', + 'Interval': '区间' + } + english_answer_type_dict = { + 'Numerical': 'a numerical value', + 'Expression': 'an expression', + 'Equation': 'an equation', + 'Interval': 'an interval' + } + + def get_single_answer_type_text(answer_type, is_chinese): + if '-' in answer_type: # No need now + answer_type = answer_type[:answer_type.find('-')] + for t in ['Numerical', 'Expression', 'Equation', 'Interval']: + if t in answer_type: + if is_chinese: + return chinese_answer_type_dict[t] + else: + return english_answer_type_dict[t] + exit(f'Error parsing answer type {answer_type}!') + + def get_answer_type_text(answer_type, is_chinese, multiple_answer): + if ('Need_human_evaluate' in answer_type) or ('Tuple' in answer_type): + full_answer_text = '' + else: + if not multiple_answer: + answer_text = get_single_answer_type_text(answer_type, is_chinese) + if is_chinese: + full_answer_text = f'，答案类型为{answer_text}' + else: + full_answer_text = f"The answer of The problem should be {answer_text}. " + else: + if ',' not in answer_type: # Same answer type for all answers + answer_text = get_single_answer_type_text(answer_type, is_chinese) + if is_chinese: + full_answer_text = f'，题目有多个答案，答案类型均为{answer_text}' + else: + full_answer_text = f'The problem has multiple answers, each of them should be {answer_text}. ' + else: + answer_types = answer_type.split(',') + answer_types = [get_single_answer_type_text(t, is_chinese) for t in answer_types] + if len(set(answer_types)) == 1: + answer_text = answer_types[0] + if is_chinese: + full_answer_text = f'，题目有多个答案，答案类型均为{answer_text}' + else: + full_answer_text = f'The problem has multiple answers, each of them should be {answer_text}. ' + else: + if is_chinese: + answer_text = '、'.join(answer_types) + full_answer_text = f'，题目有多个答案，答案类型分别为{answer_text}' + else: + answer_text = ', '.join(answer_types) + full_answer_text = f'The problem has multiple answers, with the answers in order being {answer_text}. ' + return full_answer_text + + def make_prompt(sample, is_chinese, is_math, is_theorem_proving): + if is_chinese: + subject_content = '数学' if is_math else '物理' + if is_theorem_proving: + prompt = f'以下是中国{subject_content}竞赛中的证明题。请根据题目的要求，运用逻辑推理及常用定理证明题目中的命题。证明过程中使用的变量和公式请使用LaTeX格式表示。' + else: + answer_type_text = get_answer_type_text(sample['answer_type'], is_chinese=True, multiple_answer=sample['is_multiple_answer']) + if sample['is_multiple_answer']: + multiple_answer_text = '\\boxed{用英文逗号连接的多个答案}' + else: + multiple_answer_text = '\\boxed{答案}' + unit_text = '' + if sample['unit']: + multiple_answer_text += '(单位)' + unit_text = '，注意答案的单位不要放在\\boxed{}中' + prompt = f'以下是中国{subject_content}竞赛中的解答题{answer_type_text}。请根据题目的要求和所提供的信息计算得出答案。解答过程和结果中使用的变量和公式请使用LaTeX格式表示。请在最后以"所以最终答案是{multiple_answer_text}。"显式给出结果{unit_text}。' + else: + subject_content = 'Math' if is_math else 'Physics' + if is_theorem_proving: + prompt = f'The following is a theorem proving problem from an International {subject_content} competition. Please use logical reasoning and common theorems to prove the proposition in the problem according to the given requirements. Please use LaTeX format to represent the variables and formulas used in the proof.' + else: + if sample['is_multiple_answer']: + multiple_answer_text = '\\boxed{multiple answers connected with commas}' + else: + multiple_answer_text = '\\boxed{answer}' + unit_text = '' + if sample['unit']: + multiple_answer_text += '(unit)' + unit_text = ', note that the unit of the answer should not be included in \\boxed{}' + answer_type_text = get_answer_type_text(sample['answer_type'], is_chinese=False, multiple_answer=sample['is_multiple_answer']) + prompt = f'The following is an open-ended problem from an International {subject_content} competition. {answer_type_text}Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with "So the final answer is {multiple_answer_text}." and give the result explicitly{unit_text}.' + return prompt + + inference_inputs = [] + for item in dataset: + is_chinese = item['language'] != 'English' + is_math = item['subject'] == 'Math' + is_theorem_proving = False # we won't test theorem proving in olympiadbench + if is_chinese: + subject = '数学' if is_math else '物理' + system_prompt = f"你是一个中文人工智能助手。请根据要求，完成下面的{subject}竞赛题目。" + else: + subject = 'Math' if is_math else 'Physics' + system_prompt = f'You are an AI assistant. Please answer the following {subject} competition problems as required.' + + formatted_prompt = make_prompt(item, is_chinese, is_math, is_theorem_proving) + item['question'] + # original image placeholder is , we need to transform it to for the function prompt_to_conversation + formatted_prompt = re.sub(r'', r'', formatted_prompt) + + image_ids = self.get_image_indice(formatted_prompt) + images = [item[f'image_{id}'] for id in image_ids] + conversation = ImageManager.prompt_to_conversation(user_prompt=formatted_prompt, system_prompt=system_prompt, images=images) + + inference_inputs.append( + InferenceInput( + task=self.task.name, + conversation=conversation, + ref_answer=str(item['final_answer'][0]) + ) + ) return inference_inputs \ No newline at end of file diff --git a/eval_anything/evaluate_tools/t2t_tools.py b/eval_anything/evaluate_tools/t2t_tools.py index 974918d..29dd992 100644 --- a/eval_anything/evaluate_tools/t2t_tools.py +++ b/eval_anything/evaluate_tools/t2t_tools.py @@ -149,7 +149,7 @@ def apply(self, data_1, data_2) -> bool: break return correct else: - return gold_answer in data_1 + return str(gold_answer) in data_1 def __call__(self, data_1, data_2) -> bool: return self.apply(data_1, data_2) diff --git a/eval_anything/utils/utils.py b/eval_anything/utils/utils.py index 1cc9113..b0e1594 100644 --- a/eval_anything/utils/utils.py +++ b/eval_anything/utils/utils.py @@ -37,6 +37,7 @@ 'beavertails': 'text_to_text', 'mmmu': 'text_image_to_text', 'mathvision': 'text_image_to_text', + 'olympiadbench': 'text_image_to_text', } class MultiChoicePromptBuilder(): From 0da2af5fdf5f4bcc918d99a43f055649997189bc Mon Sep 17 00:00:00 2001 From: W3en2g Date: Mon, 2 Jun 2025 17:24:41 +0800 Subject: [PATCH 2/2] fix: refine OlympiadBench evaluate settings --- .../olympiadbench/configs.yaml | 13 ++++--- .../text_image_to_text/olympiadbench/eval.py | 2 +- eval_anything/configs/evaluate.yaml | 2 +- eval_anything/dataloader/format_mm_dataset.py | 28 ++++++++------ eval_anything/evaluate_tools/t2t_tools.py | 37 +++++++++++++++++-- 5 files changed, 59 insertions(+), 23 deletions(-) diff --git a/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml b/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml index ee2270f..325dd34 100644 --- a/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml +++ b/eval_anything/benchmarks/text_image_to_text/olympiadbench/configs.yaml @@ -22,14 +22,14 @@ dataset: cot_fewshot_data_name: null cot_fewshot_data_split: null max_shot: 0 - default_task_list: ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE", "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE",] + default_task_list: ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE",] # "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE",] task_defaults: &task_defaults type: MultiChoice question_key: question answer_key: options - ground_truth_key: answer + ground_truth_key: final_answer candidate_labels: ["A", "B", "C", "D", "E"] - avalable_evaluate_tools: ["match_multi-choice_and_open-ended"] + avalable_evaluate_tools: ["match_origin_response"] task: - name: OE_MM_maths_en_COMP data_files: null @@ -62,10 +62,11 @@ task: data_files: null <<: *task_defaults answer_extractor: - - name: match_multi-choice_and_open-ended - function: regex_match_latex_math + - name: match_origin_response + function: null args: - match_index: -1 # Index for multiple choice + additional_pattern: "{original_pattern}" + match_index: 0 metrics: - name: accuracy function: accuracy diff --git a/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py b/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py index 53785bd..5d3905d 100644 --- a/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py +++ b/eval_anything/benchmarks/text_image_to_text/olympiadbench/eval.py @@ -45,7 +45,7 @@ def run(self, evaluation_results = {} for task in task_list: - evaluation_details[task], evaluation_results[task] = self.calculate_metrics(self.benchmark_name, inference_outputs[task], ref_answers[task], self.benchmark_cfgs.answer_extractor, self.benchmark_cfgs.metrics, judge_method="judge_equal_list") + evaluation_details[task], evaluation_results[task] = self.calculate_metrics(self.benchmark_name, inference_outputs[task], ref_answers[task], self.benchmark_cfgs.answer_extractor, self.benchmark_cfgs.metrics, judge_method="judge_latex_math_equal") if len(task_list) > 1: overall_result = self.calculate_overall_metrics(self.benchmark_cfgs.overall_metrics, result=evaluation_results) diff --git a/eval_anything/configs/evaluate.yaml b/eval_anything/configs/evaluate.yaml index d1508fe..ae9818f 100644 --- a/eval_anything/configs/evaluate.yaml +++ b/eval_anything/configs/evaluate.yaml @@ -19,7 +19,7 @@ "CEval": ['accountant', 'advanced_mathematics', 'art_studies', 'basic_medicine', 'business_administration', 'chinese_language_and_literature', 'civil_servant', 'clinical_medicine', 'college_chemistry', 'college_economics', 'college_physics', 'college_programming', 'computer_architecture', 'computer_network', 'discrete_mathematics', 'education_science', 'electrical_engineer', 'environmental_impact_assessment_engineer', 'fire_engineer', 'high_school_biology', 'high_school_chemistry', 'high_school_chinese', 'high_school_geography', 'high_school_history', 'high_school_mathematics', 'high_school_physics', 'high_school_politics', 'ideological_and_moral_cultivation', 'law', 'legal_professional', 'logic', 'mao_zedong_thought', 'marxism', 'metrology_engineer', 'middle_school_biology', 'middle_school_chemistry', 'middle_school_geography', 'middle_school_history', 'middle_school_mathematics', 'middle_school_physics', 'middle_school_politics', 'modern_chinese_history', 'operating_system', 'physician', 'plant_protection', 'probability_and_statistics', 'professional_tour_guide', 'sports_science', 'tax_accountant', 'teacher_qualification', 'urban_and_rural_planner', 'veterinary_medicine'], "mmmu": ["Accounting"], "mathvision": ["default"], - "olympiadbench": ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE", "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE"], + "olympiadbench": ["OE_MM_maths_en_COMP", "OE_MM_maths_zh_CEE", "OE_MM_maths_zh_COMP","OE_MM_physics_en_COMP", "OE_MM_physics_zh_CEE"], # "OE_TO_maths_en_COMP","OE_TO_maths_zh_CEE", "OE_TO_maths_zh_COMP", "OE_TO_physics_en_COMP","OE_TO_physics_zh_CEE"], "mmau": ["MMAU-mini"], "mmvu": ["default"], } diff --git a/eval_anything/dataloader/format_mm_dataset.py b/eval_anything/dataloader/format_mm_dataset.py index 8af9b35..d860b98 100644 --- a/eval_anything/dataloader/format_mm_dataset.py +++ b/eval_anything/dataloader/format_mm_dataset.py @@ -286,12 +286,11 @@ def set_few_shot_examples(self, few_shot_dataset: Dataset | None): raise NotImplementedError("olympiadbench does not support few-shot learning.") def get_image_indice(self, text: str)->List[int]: - pattern = r'' + pattern = r'' matches = re.findall(pattern, text) return [int(num) for num in matches] - # refer: https://github.com/mathllm/MATH-V/blob/main/models/Qwen-VL.py#L19 def _to_InferenceInput(self, dataset: Dataset) -> List["InferenceInput"]: """ Convert a dataset to a list of InferenceInput objects. @@ -408,23 +407,30 @@ def make_prompt(sample, is_chinese, is_math, is_theorem_proving): subject = 'Math' if is_math else 'Physics' system_prompt = f'You are an AI assistant. Please answer the following {subject} competition problems as required.' - formatted_prompt = make_prompt(item, is_chinese, is_math, is_theorem_proving) + item['question'] - # original image placeholder is , we need to transform it to for the function prompt_to_conversation - formatted_prompt = re.sub(r'', r'', formatted_prompt) - + formatted_prompt = make_prompt(item, is_chinese, is_math, is_theorem_proving) + if is_math: + formatted_prompt = formatted_prompt + '\n' + item['question'] + else: + if 'context' in item.keys() and item['context']: # cannot be null + formatted_prompt = formatted_prompt + '\n' + item['context']+'\n'+item['question'] + else: + formatted_prompt = formatted_prompt + '\n' + item['question'] image_ids = self.get_image_indice(formatted_prompt) images = [item[f'image_{id}'] for id in image_ids] - conversation = ImageManager.prompt_to_conversation(user_prompt=formatted_prompt, system_prompt=system_prompt, images=images) + # Some subset like OE_MM_physics_en_COMP subset, some questions have no images, here we just skip them + # ref https://github.com/OpenBMB/OlympiadBench/issues/16 + if images == []: + continue + image_manager = MMDataManagerRegistry.get_mm_data_manager("image") + conversation = image_manager.prompt_to_conversation(user_prompt=formatted_prompt, system_prompt=system_prompt, images=images) inference_inputs.append( InferenceInput( task=self.task.name, conversation=conversation, - ref_answer=str(item['final_answer'][0]) + ref_answer=str(item['final_answer'][0]), + metadata="image", ) ) - if len(inference_inputs) > 10: - break - return inference_inputs \ No newline at end of file diff --git a/eval_anything/evaluate_tools/t2t_tools.py b/eval_anything/evaluate_tools/t2t_tools.py index 29dd992..d6d4297 100644 --- a/eval_anything/evaluate_tools/t2t_tools.py +++ b/eval_anything/evaluate_tools/t2t_tools.py @@ -5,6 +5,7 @@ from eval_anything.evaluate_tools.base_tools import BaseTool from typing import Union, List, Iterable, Optional from latex2sympy2 import latex2sympy +import math_verify from math import * import numpy as np import re @@ -23,6 +24,7 @@ "judge_equal": "JudgeEqual", "judge_equal_list": "JudgeEqualList", "judge_latex_equal":"JudgeLatexEqual", + "judge_latex_math_equal":"JudgeLatexMathEqual", "judge_mc1": "JudgeMC1", "judge_mc2": "JudgeMC2", } @@ -58,7 +60,6 @@ def match_text(text): def __call__(self, data: Union[List, Iterable]) -> Union[List, None]: return self.apply(data) - class RegexMatchNumber(RegexMatch): def __init__(self, additional_pattern: str = None, match_index: int = None): pattern_match_number = r"(?:[+-]?(?:\d+/\d+|(?:\d*\.\d+)|\d+)|√(?:\([+-]?(?:\d+/\d+|(?:\d*\.\d+)|\d+)\)|[+-]?(?:\d+/\d+|(?:\d*\.\d+)|\d+)))" @@ -149,12 +150,11 @@ def apply(self, data_1, data_2) -> bool: break return correct else: - return str(gold_answer) in data_1 + return gold_answer in data_1 def __call__(self, data_1, data_2) -> bool: return self.apply(data_1, data_2) - # refer: https://github.com/mathllm/MATH-V/blob/main/evaluation/utils.py class JudgeLatexEqual(BaseTool): def __init__(self): @@ -244,6 +244,36 @@ def is_equal(asw: str, gt_asw: str) -> bool: def __call__(self, data_1, data_2) -> bool: return self.apply(data_1, data_2) +# Directly using math_verify for mathematical expression verification +class JudgeLatexMathEqual(BaseTool): + def __init__(self): + super().__init__() + + def apply(self, data_1, data_2) -> bool: + def is_equal(asw: str, gt_asw: str) -> bool: + """ + Judge if `asw` is equivalent to `gt_asw`. + + We use the `math_verify` library to parse and verify the mathematical expressions. + This function checks if the given answers are equivalent. + + Args: + asw (str): The answer string to be checked. + gt_asw (str): The ground truth answer string to be matched against. + + Returns: + bool: True if the answers are equivalent, otherwise False. + + """ + answer = math_verify.parse(asw) + gt_answer = math_verify.parse(gt_asw) + return math_verify.verify(answer, gt_answer) + + return is_equal(data_1, data_2) + + def __call__(self, data_1, data_2) -> bool: + return self.apply(data_1, data_2) + class RegexMatchLetter(RegexMatch): def __init__(self, additional_pattern: str = None, match_index: int = None): # Base pattern to match letters in parentheses (A-Z) @@ -654,7 +684,6 @@ def __call__(self, data: Union[List, Iterable]) -> Union[List, None]: return self.apply(data) - class JudgeMC1(BaseTool): def __init__(self): super().__init__()