Math dataset

eval_gsm.py

@@ -0,0 +1,39 @@
+# open the json file and read the data
+import json
+import re
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--file", type=str, default="experiment/gsm.jsonl")
+args = parser.parse_args()
+
+
+def extract_code(text):
+    match = re.search(r"```(.*?)```", text, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    return None
+
+
+correct = 0
+all_count = 0
+with open(args.file, "r") as f:
+    for line in f:
+        all_count += 1
+        data = json.loads(line)
+        answer = data["answer"]
+        code = data["output_pred"]
+        if code is None:
+            continue
+        predict_answer = None
+        try:
+            exec(code)
+            exec("predict_answer = solution()")
+            # exec("print(predict_answer, answer)")
+            # compute the accuracy
+        except Exception as e:
+            print(e)
+        if predict_answer == answer:
+            correct += 1
+print(correct, all_count)
+print("Accuracy:", correct / all_count)

gsm_evaluator_with_lora_soup.py

@@ -0,0 +1,38 @@
+# here, we train experts and we upload them to a local library (repository) of experts.
+
+import os
+from mttl.arguments import ExpertConfig
+from mttl.datamodule.base import get_datamodule
+from mttl.models.library.expert_library import ExpertLibrary
+from mttl.models.expert_model import (
+    ExpertModel,
+    MultiExpertModel,
+    MultiExpertModelConfig,
+    ExpertModelConfig,
+)
+from mttl.models.train_utils import train_model
+
+from mttl.evaluators.gsm_evaluator import GsmEvaluator
+from mttl.evaluators.rouge_evaluator import RougeEvaluator
+from mttl.models.containers.selectors.base import UniformSelectorConfig
+from mttl.arguments import EvaluationConfig, ExpertConfig
+from mttl.models.lightning.expert_module import ExpertModule
+import torch
+from mttl.logging import setup_logging
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+setup_logging()
+
+args = EvaluationConfig.parse()
+
+datamodule = get_datamodule(args, for_generation=True)
+evaluator = GsmEvaluator(datamodule)
+
+#
+module = ExpertModule(**vars(args)).to(device)
+
+if args.checkpoint is not None:
+    checkpoint = torch.load(args.checkpoint, weights_only=False)["state_dict"]
+    module.load_state_dict(checkpoint)
+## evaluate
+result = evaluator.evaluate(module.model, split="test")

mttl/arguments.py

@@ -590,6 +590,7 @@ class EvaluationConfig(MultiExpertConfig, TransformArgs):
     es_metric: str = "loss"
     n_ng_iterations: int = 30  # number of iterations for LoraHub
     recompute_prototypes: bool = False
+    gsm_template: str = "cot"
 
 
 @dataclass

mttl/dataloader/alpaca_dataset_readers.py

@@ -96,3 +96,19 @@ def read_all_instructions(self):
         for data in self.dataset:
             all_instructions.append(data["instruction"])
         return all_instructions
+
+
+class AlpacaCodeDataset(AlpacaDataset):
+    def __init__(self):
+        super().__init__()
+        self.dataset = DatasetLibrary.pull_dataset(
+            "zhan1993/code_alpaca_20k", split="train"
+        )
+
+
+class MathQaAlpacaCodeDataset(AlpacaDataset):
+    def __init__(self):
+        super().__init__()
+        self.dataset = DatasetLibrary.pull_dataset(
+            "zhan1993/metamath_code_alpaca_10k", split="train"
+        )

mttl/datamodule/alpaca_data_module.py

@@ -1,4 +1,8 @@
-from mttl.dataloader.alpaca_dataset_readers import AlpacaDataset
+from mttl.dataloader.alpaca_dataset_readers import (
+    AlpacaCodeDataset,
+    AlpacaDataset,
+    MathQaAlpacaCodeDataset,
+)
 from mttl.datamodule.base import DataModule, DatasetConfig
 
 
@@ -18,9 +22,46 @@ def setup_dataset(self):
         self.test_dataset = self.dev_dataset
 
 
-class AlpacaPretrainDataModule(AlpacaDataModule):
-    pass
+@DataModule.register("alpaca_code", config_cls=DatasetConfig)
+class AlpacaCodeDataModule(DataModule):
+    @property
+    def all_instructions(self):
+        return self.dataset.read_all_instructions()
+
+    def __init__(self, config, for_generation=False, val_mixin=None):
+        super().__init__(config, for_generation, val_mixin)
+
+    def setup_dataset(self):
+        dataset = AlpacaCodeDataset()
+
+        self.train_dataset, self.dev_dataset = self.create_train_valid_split(dataset)
+        self.test_dataset = self.dev_dataset
+
+
+@DataModule.register("mathqa_alpaca_code", config_cls=DatasetConfig)
+class MathQaAlpacaCodeDataModule(AlpacaDataModule):
+    def setup_dataset(self):
+        dataset = MathQaAlpacaCodeDataset()
+        self.train_dataset, self.dev_dataset = self.create_train_valid_split(dataset)
+        self.test_dataset = self.dev_dataset
 
 
 class AlpacaFinetuneDataModule(AlpacaDataModule):
     pass
+
+
+if __name__ == "__main__":
+    # alpaca_data_module = AlpacaDataModule(
+    #     DatasetConfig(model="meta-llama/Llama-2-7b-hf")
+    # )
+    # alpaca_data_module.setup_dataset()
+    # print(alpaca_data_module.train_dataset)
+
+    mathqa_alpaca_code_data_module = MathQaAlpacaCodeDataModule(
+        DatasetConfig(model="meta-llama/Llama-2-7b-hf")
+    )
+    mathqa_alpaca_code_data_module.setup_dataset()
+    val_dataloder = mathqa_alpaca_code_data_module.val_dataloader()
+    for batch in val_dataloder:
+        print(batch)
+        breakpoint()

mttl/datamodule/base.py

@@ -944,6 +944,12 @@ def get_datamodule(args, for_generation=False, dataset_override=None):
         HellaswagDataConfig,
         HellaswagMultiChoiceDataModule,
     )
+    from mttl.datamodule.mathqa_data_module import MathQADataConfig, MathQADataModule
+    from mttl.datamodule.gsm_data_module import GsmDataConfig, GsmDataModule
+    from mttl.datamodule.base import DatasetConfig
+    from mttl.datamodule.alpaca_data_module import (
+        AlpacaCodeDataModule,
+    )
     from mttl.datamodule.mmlu_data_module import MMLUDataConfig, MMLUDataModule
     from mttl.datamodule.mt_seq_to_seq_module import (
         FlanConfig,
@@ -1063,6 +1069,23 @@ def get_datamodule(args, for_generation=False, dataset_override=None):
             pack_sequences=args.pack_sequences,
         )
         dm = FlatMultiTaskModule(config, for_generation=for_generation)
+    elif "mathqa" in dataset:
+        config = MathQADataConfig(
+            **common_kwargs,
+        )
+        dm = MathQADataModule(config, for_generation=for_generation)
+    elif "gsm" in dataset:
+        config = GsmDataConfig(
+            **common_kwargs,
+            gsm_template=args.gsm_template,
+        )
+        dm = GsmDataModule(config, for_generation=for_generation)
+
+    elif "alpaca_code" in dataset:
+        config = DatasetConfig(
+            **common_kwargs,
+        )
+        dm = AlpacaCodeDataModule(config, for_generation=for_generation)
     elif "mmlu" in dataset:
         config = MMLUDataConfig(
             **common_kwargs,

mttl/datamodule/gsm_data_module.py

@@ -0,0 +1,96 @@
+import os
+from dataclasses import dataclass
+
+from mttl.datamodule.base import DataModule, DatasetConfig
+from mttl.models.library.dataset_library import DatasetLibrary
+import json
+
+
+@dataclass
+class GsmDataConfig(DatasetConfig):
+    gsm_template: str = (
+        "cot"  # the template we will use for the prompt, for code generation or chain of thought.
+    )
+
+
+# code refer to https://github.com/aksh555/LoRA-Soups/blob/main/evaluate.py#L208
+def generate_math_prompt_with_python(instruction, input=None):
+    with open("mttl/datamodule/math.json", "r") as f:
+        cot_data = json.load(f)
+    prompt = """Let's use Python to solve math problems step by step. Below are a few Instruction-Response pairs on how to do it."""
+    prompt += "\n\n"
+    for data in cot_data:
+        prompt += f"### Instruction:\n{data['instruction']}\n\n### Response:\n{data['output']}\n\n"
+    prompt += "Now write a function 'solution' encolsed in ``` in Python to solve this Instruction. Write only a code block. Write only valid Python code without using any units with the numerical values and any invalid symbols.\n\n"
+    prompt += f"### Instruction:\n{instruction}\n\n### Response:\n"
+    return prompt
+
+
+def instruct_template_python(example):
+    example["source"] = generate_math_prompt_with_python(example["input"])
+    example["target"] = str(example["answer"])
+    return example
+
+
+def instruct_template_cot(example):
+
+    PREAMBLE = """As an expert problem solver solve step by step the following mathematical questions."""
+    PROMPT = """Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?
+    A: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.
+
+    Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?
+    A: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.
+
+    Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?
+    A: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.
+
+    Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?
+    A: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.
+
+    Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?
+    A: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.
+
+    Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?
+    A: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.
+
+    Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?
+    A: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.
+
+    Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?
+    A: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8."""
+
+    TEMPLATE = """
+    Q: {question}
+    A:"""
+
+    full_prompt = (
+        PREAMBLE + "\n\n" + PROMPT + "\n" + TEMPLATE.format(question=example["input"])
+    )
+    example["source"] = full_prompt
+    example["target"] = str(example["answer"])
+    return example
+
+
+@DataModule.register("gsm", config_cls=GsmDataConfig)
+class GsmDataModule(DataModule):
+    def setup_dataset(self):
+        n_proc = int(os.environ.get("MTTL_NUM_PROC_DATASETS", 4))
+        dataset = DatasetLibrary.pull_dataset("reasoning-machines/gsm-hard")
+        dataset = dataset.rename_column("target", "answer")
+        if self.config.gsm_template == "cot":
+            dataset = dataset.map(instruct_template_cot, num_proc=n_proc)
+        elif self.config.gsm_template == "python":
+            dataset = dataset.map(instruct_template_python, num_proc=n_proc)
+        self.train_dataset = dataset["train"]
+        self.dev_dataset = self.test_dataset = dataset["train"]
+
+
+if __name__ == "__main__":
+    config = GsmDataConfig(model="microsoft/Phi-3-mini-4k-instruct", gsm_template="cot")
+
+    datamodule = GsmDataModule(config, for_generation=True)
+    train_dataloader = datamodule.train_dataloader()
+    val_dataloder = datamodule.val_dataloader()
+    for batch in val_dataloder:
+        print(batch)
+        breakpoint()

mttl/datamodule/math.json

@@ -0,0 +1,50 @@
+[
+    {
+        "instruction": "Olivia has $23. She bought five bagels for $3 each. How much money does she have left?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    money_initial = 23\n    bagels = 5\n    bagel_cost = 3\n    money_spent = bagels * bagel_cost\n    money_left = money_initial - money_spent\n    result = money_left\n    return result\n```",
+        "answer": "8"
+    },
+    {
+        "instruction": "Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    golf_balls_initial = 58\n    golf_balls_lost_tuesday = 23\n    golf_balls_lost_wednesday = 2\n    golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday\n    result = golf_balls_left\n    return result\n```",
+        "answer": "33"
+    },
+    {
+        "instruction": "There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    computers_initial = 9\n    computers_per_day = 5\n    num_days = 4  # 4 days between monday and thursday\n    computers_added = computers_per_day * num_days\n    computers_total = computers_initial + computers_added\n    result = computers_total\n    return result\n```",
+        "answer": "29"
+    },
+    {
+        "instruction": "Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    toys_initial = 5\n    mom_toys = 2\n    dad_toys = 2\n    total_received = mom_toys + dad_toys\n    total_toys = toys_initial + total_received\n    result = total_toys\n    return result\n```",
+        "answer": "9"
+    },
+    {
+        "instruction": "Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    jason_lollipops_initial = 20\n    jason_lollipops_after = 12\n    denny_lollipops = jason_lollipops_initial - jason_lollipops_after\n    result = denny_lollipops\n    return result\n```",
+        "answer": "8"
+    },
+    {
+        "instruction": "Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    leah_chocolates = 32\n    sister_chocolates = 42\n    total_chocolates = leah_chocolates + sister_chocolates\n    chocolates_eaten = 35\n    chocolates_left = total_chocolates - chocolates_eaten\n    result = chocolates_left\n    return result\n```",
+        "answer": "39"
+    },
+    {
+        "instruction": "If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?",
+        "input": "",
+        "output": "There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n\nLet\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    cars_initial = 3\n    cars_arrived = 2\n    total_cars = cars_initial + cars_arrived\n    result = total_cars\n    return result\n```",
+        "answer": "5"
+    },
+    {
+        "instruction": "There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?",
+        "input": "",
+        "output": "Let\u2019s go ahead and solve this using Python:\n```\ndef solution():\n    trees_initial = 15\n    trees_after = 21\n    trees_added = trees_after - trees_initial\n    result = trees_added\n    return result\n```",
+        "answer": "6"
+    }
+]