Noisy Data and Script

data/gsm/accuracy.csv

@@ -1,16 +1,28 @@
-flan_gsm.py,2.6615969581749046
-mistral_gsm.py,8.188021228203183
-mistral_instruct.py,8.41546626231994
-mistral_math.py,26.004548900682334
-flan_gsm.py,2.687296416938111
-mistral_gsm.py,8.188021228203183
-mistral_instruct.py,8.41546626231994
-mistral_math.py,26.004548900682334
-flan_gsm.py,2.631578947368421
-mistral_gsm.py,8.188021228203183
-mistral_instruct.py,8.41546626231994
-mistral_math.py,26.004548900682334
-flan_gsm.py,2.577710386656558
-mistral_gsm.py,8.188021228203183
-mistral_instruct.py,8.41546626231994
-mistral_math.py,26.004548900682334
+mistral_math.py,42.77777777777778
+mistral.py,7.777777777777778
+mistral_instruct.py,11.11111111111111
+flan.py,3.3333333333333335
+mistral_math.py,42.77777777777778
+mistral.py,7.777777777777778
+mistral_instruct.py,11.11111111111111
+flan.py,3.3333333333333335
+flan_gsm.py,2.6535253980288096
+mistral_gsm.py,7.429871114480667
+mistral_instruct.py,8.567096285064443
+mistral_math.py,22.062168309325248
+flan_gsm.py,1.6679302501895377
+mistral_gsm.py,7.202426080363912
+mistral_instruct.py,7.505686125852919
+mistral_math.py,19.711902956785444
+mistral_math.py,33.33333333333333
+mistral.py,9.444444444444445
+mistral_instruct.py,6.666666666666667
+flan.py,1.6666666666666667
+mistral_math.py,36.666666666666664
+mistral.py,3.888888888888889
+mistral_instruct.py,6.666666666666667
+flan.py,2.7777777777777777
+flan_gsm.py,1.1372251705837757
+mistral_gsm.py,5.913570887035633
+mistral_instruct.py,7.202426080363912
+mistral_math.py,21.37983320697498

requirements.txt

@@ -5,4 +5,7 @@ accelerate
 transformers
 pandas
 bitsandbytes
-jsonformer
+jsonformer
+protobuf
+sentencepiece
+setuptools

script/gsm/flan_gsm.py

@@ -8,10 +8,11 @@
 import random
 import sys, os, json
 
+from tqdm import tqdm
 from jsonformer import Jsonformer
 
 from config import access_token, DIR_PATH
-from utils import get_questions_and_answer_from_multiArith_dataset
+from utils import get_noisy_questions_and_answer_from_dataset
 
 tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")
 model = T5ForConditionalGeneration.from_pretrained(
@@ -25,13 +26,12 @@
         "answer": {"type": "string"},
     },
 }
-
-csv_file = f"{DIR_PATH}/data/multiArith/test_preprocessed.csv"
-questions, ground_truths = get_questions_and_answer_from_multiArith_dataset(csv_file)
+csv_file = f"{DIR_PATH}/data/noisy_datasets/gsm8k_test_noisy_punct_50.csv"
+questions, ground_truths = get_noisy_questions_and_answer_from_dataset(csv_file)
 
 
 output_file = (
-    f"{DIR_PATH}/data/multiArith/flan/flan_gsm_response.csv"
+    f"{DIR_PATH}/data/gsm/flan/flan_gsm_response_50.csv"
 )
 counter = 0
 with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
@@ -48,7 +48,7 @@
     json_format = {
         "answer": {"<contains the correct numerical answer>"},
     }
-    for question, ground_truth in zip(questions, ground_truths):
+    for question, ground_truth in tqdm(zip(questions, ground_truths), total=len(questions)):
 
         # prompt = f"""
         # [INST] 
@@ -87,15 +87,15 @@
             max_number_tokens=1000,
             max_array_length=1000,
             max_string_token_length=1000,
-            temperature = 0
+            temperature=0
         )
 
         generated_data = jsonformer()
         import pprint
 
-        pprint.pprint(prompt)
-        print("##RESPONSE##")
-        pprint.pprint(generated_data)
+        #pprint.pprint(prompt)
+        #print("##RESPONSE##")
+        #pprint.pprint(generated_data)
 
         writer.writerow(
             {
@@ -111,3 +111,10 @@
 
 
 print(f"Questions and answers saved to {output_file}")
+
+
+
+
+
+
+# github.com

script/gsm/mistral_gsm.py

@@ -12,11 +12,12 @@
 import random
 import sys, os, json
 import math
+from tqdm import tqdm
 
 from jsonformer import Jsonformer
 
 from config import access_token, DIR_PATH
-from utils import get_questions_and_answer_from_multiArith_dataset
+from utils import get_noisy_questions_and_answer_from_dataset
 
 access_token = access_token
 model_name = "mistralai/Mistral-7B-v0.1"
@@ -46,12 +47,11 @@
     },
 }
 
-csv_file = f"{DIR_PATH}/data/multiArith/test_preprocessed.csv"
-questions, ground_truths = get_questions_and_answer_from_multiArith_dataset(csv_file)
-
+csv_file = f"{DIR_PATH}/data/noisy_datasets/gsm8k_test_noisy_punct_50.csv"
+questions, ground_truths = get_noisy_questions_and_answer_from_dataset(csv_file)
 
 output_file = (
-    f"{DIR_PATH}/data/multiArith/mistral/mistral_multiArith_response.csv"
+    f"{DIR_PATH}/data/gsm/mistral/mistral_gsm_response_50.csv"
 )
 counter = 0
 with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
@@ -68,7 +68,7 @@
     json_format = {
         "answer": {"<contains the correct numerical answer>"},
     }
-    for question, ground_truth in zip(questions, ground_truths):
+    for question, ground_truth in tqdm(zip(questions, ground_truths), total=len(questions)):
 
         prompt = f"""
         [INST] 
@@ -105,9 +105,9 @@
         generated_data = jsonformer()
         import pprint
 
-        pprint.pprint(prompt)
-        print("##RESPONSE##")
-        pprint.pprint(generated_data)
+        #pprint.pprint(prompt)
+        #print("##RESPONSE##")
+        #pprint.pprint(generated_data)
 
         writer.writerow(
             {

script/gsm/mistral_instruct_gsm.py

@@ -12,10 +12,11 @@
 import random
 import sys, os, json
 
+from tqdm import tqdm
 from jsonformer import Jsonformer
 
 from config import access_token, DIR_PATH
-from utils import get_questions_and_answer_from_multiArith_dataset
+from utils import get_noisy_questions_and_answer_from_dataset
 
 
 access_token = access_token
@@ -46,11 +47,13 @@
     },
 }
 
-csv_file = f"{DIR_PATH}/data/multiArith/test_preprocessed.csv"
-questions, ground_truths = get_questions_and_answer_from_multiArith_dataset(csv_file)
+csv_file = f"{DIR_PATH}/data/noisy_datasets/gsm8k_test_noisy_punct_50.csv"
+questions, ground_truths = get_noisy_questions_and_answer_from_dataset(csv_file)
 
 
-output_file = f"{DIR_PATH}/data/multiArith/mistral_instruct/mistral_instruct_gsm_response.csv"
+
+
+output_file = f"{DIR_PATH}/data/gsm/mistral_instruct/mistral_instruct_gsm_response_50.csv"
 
 counter = 0
 with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
@@ -67,7 +70,7 @@
     json_format = {
         "answer": {},
     }
-    for question, ground_truth in zip(questions, ground_truths):
+    for question, ground_truth in tqdm(zip(questions, ground_truths), total=len(questions)):
 
         # prompt = f"""
         # [INST] 
@@ -112,9 +115,9 @@
         generated_data = jsonformer()
         import pprint
 
-        pprint.pprint(prompt)
-        print("##RESPONSE##")
-        pprint.pprint(generated_data)
+        #pprint.pprint(prompt)
+        #print("##RESPONSE##")
+        #pprint.pprint(generated_data)
 
         writer.writerow(
             {

script/gsm/mistral_math_gsm.py

@@ -12,10 +12,11 @@
 import random
 import sys, os, json
 
+from tqdm import tqdm
 from jsonformer import Jsonformer
 
 from config import access_token, DIR_PATH
-from utils import get_questions_and_answer_from_multiArith_dataset
+from utils import get_noisy_questions_and_answer_from_dataset
 
 
 access_token = access_token
@@ -45,11 +46,13 @@
     },
 }
 
-csv_file = f"{DIR_PATH}/data/multiArith/test_preprocessed.csv"
-questions, ground_truths = get_questions_and_answer_from_multiArith_dataset(csv_file)
+csv_file = f"{DIR_PATH}/data/noisy_datasets/gsm8k_test_noisy_punct_50.csv"
+questions, ground_truths = get_noisy_questions_and_answer_from_dataset(csv_file)
+
+
 
 #TODO: Change to relative path
-output_file = f"{DIR_PATH}/data/multiArith/mistral_math/mistral_math_gsm_response.csv"
+output_file = f"{DIR_PATH}/data/gsm/mistral_math/mistral_math_gsm_response_50.csv"
 counter = 0
 with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
     fieldnames = [
@@ -65,7 +68,7 @@
     json_format = {
         "answer": {},
     }
-    for question, ground_truth in zip(questions, ground_truths):
+    for question, ground_truth in tqdm(zip(questions, ground_truths), total=len(questions)):
 
         prompt = f"""
         [INST] 
@@ -89,9 +92,9 @@
         generated_data = jsonformer()
         import pprint
 
-        pprint.pprint(prompt)
-        print("##RESPONSE##")
-        pprint.pprint(generated_data)
+        #pprint.pprint(prompt)
+        #print("##RESPONSE##")
+        #pprint.pprint(generated_data)
 
         writer.writerow(
             {

script/gsm/script_accuracy.py

@@ -2,21 +2,21 @@
 
 from config import access_token, DIR_PATH
 
-# output_files = [
-#        (f"{DIR_PATH}/data/gsm/flan/flan_gsm_response.csv","flan_gsm"),
-#        (f"{DIR_PATH}/data/gsm/mistral/mistral_gsm_response.csv","mistral_gsm"),
-#        (f"{DIR_PATH}/data/gsm/mistral_instruct/mistral_instruct_gsm_response.csv","mistral_instruct"),
-#        (f"{DIR_PATH}/data/gsm/mistral_math/mistral_math_gsm_response.csv","mistral_math")
-
-#     ]
-
 output_files = [
-       (f"{DIR_PATH}/data/multiArith/mistral_math/mistral_math_multiArith_response.csv","mistral_math"),
-       (f"{DIR_PATH}/data/multiArith/mistral/mistral_multiArith_response.csv","mistral_gsm"),
-       (f"{DIR_PATH}/data/multiArith/mistral_instruct/mistral_instruct_multiArith_response.csv","mistral_instruct"),
-       (f"{DIR_PATH}/data/multiArith/flan/flan_multiArith_response.csv","flan_gsm"),
+       (f"{DIR_PATH}/data/gsm/flan/flan_gsm_response_50.csv","flan_gsm"),
+       (f"{DIR_PATH}/data/gsm/mistral/mistral_gsm_response_50.csv","mistral_gsm"),
+       (f"{DIR_PATH}/data/gsm/mistral_instruct/mistral_instruct_gsm_response_50.csv","mistral_instruct"),
+       (f"{DIR_PATH}/data/gsm/mistral_math/mistral_math_gsm_response_50.csv","mistral_math")
+
     ]
 
+# output_files = [
+#        (f"{DIR_PATH}/data/multiArith/mistral_math/mistral_math_multiArith_response_50.csv","mistral_math"),
+#        (f"{DIR_PATH}/data/multiArith/mistral/mistral_multiArith_response_50.csv","mistral"),
+#        (f"{DIR_PATH}/data/multiArith/mistral_instruct/mistral_instruct_multiArith_response_50.csv","mistral_instruct"),
+#        (f"{DIR_PATH}/data/multiArith/flan/flan_multiArith_response_50.csv","flan"),
+#     ]
+
     # Call the calculate_accuracy function for each output file
 for output_file in output_files:
 	print(output_file)

script/gsm/script_run.py

@@ -1,11 +1,12 @@
 import subprocess
+from config import DIR_PATH
 
 # List of scripts to run sequentially
 scripts = [
-    "python  mistral_gsm.py",
-    "python  mistral_instruct_gsm.py",
-    "python  mistral_math_gsm.py",
-    "python  flan_gsm.py"
+    f"python  {DIR_PATH}/script/gsm/mistral_gsm.py",
+    f"python  {DIR_PATH}/script/gsm/mistral_instruct_gsm.py",
+    f"python  {DIR_PATH}/script/gsm/mistral_math_gsm.py",
+    f"python  {DIR_PATH}/script/gsm/flan_gsm.py"
 ]
 
 # Run each script sequentially

script/gsm/utils.py

@@ -88,3 +88,18 @@ def safe_convert_llm_to_int(value):
     )
 
     print("Accuracy saved to accuracy.csv.")
+
+
+def get_noisy_questions_and_answer_from_dataset(csv_file_path):
+    # Load the specific CSV file
+    data = pd.read_csv(csv_file_path)
+    questions = data["noisy_questions"].tolist()
+    groundTruths = data["numeric_answer"].tolist()
+    return questions,groundTruths
+
+def get_noisy_questions_and_answer_from_multi_arith_dataset(csv_file_path):
+    # Load the specific CSV file
+    data = pd.read_csv(csv_file_path)
+    questions = data["noisy_questions"].tolist()
+    groundTruths = data["answer"].tolist()
+    return questions,groundTruths

script/multiArith/multiArith_preprocess.py

@@ -1,8 +1,10 @@
 import pandas as pd
 import json
 
+from config import DIR_PATH
+
 # Path to the JSON file
-json_file_path = "/home/stud/abedinz1/localDisk/nlplab/data/multiArith/questions.json"
+json_file_path = f"{DIR_PATH}/data/multiArith/questions.json"
 
 # Read JSON data from the file
 with open(json_file_path, 'r') as file:
@@ -19,7 +21,7 @@
 df = pd.DataFrame(data)
 
 # Save to CSV
-output_path = "/home/stud/abedinz1/localDisk/nlplab/data/multiArith/test_preprocessed.csv"
+output_path = f"{DIR_PATH}/data/multiArith/test_preprocessed.csv"
 df.to_csv(output_path, index=False)
 
 # Display the DataFrame

script/noise_creation/create_noise.py

@@ -5,6 +5,8 @@
 from tqdm import tqdm
 import pandas as pd
 
+from config import DIR_PATH
+
 tqdm.pandas()
 
 random.seed(0)
@@ -36,10 +38,10 @@ def insert_punctuation_marks(sentence, punc_ratio):
 
 
 def main(dataset):
-    data_df = pd.read_csv(dataset + '/train_preprocessed.csv')
+    data_df = pd.read_csv(DIR_PATH + '/data/multiArith/test_preprocessed.csv')
     for punct in tqdm(PUNC_RATIO):
         data_df[f'noisy_questions'] = data_df['question'].progress_apply(lambda x: insert_punctuation_marks(x,punct))
-        data_df.to_csv(f'../../data/noisy_datasets/gsm8k_noisy_punct_{int(punct*100)}.csv',index=False)
+        data_df.to_csv(f'{DIR_PATH}/data/noisy_datasets/multiArith_test_noisy_punct_{int(punct*100)}.csv',index=False)
 if __name__ == "__main__":
     #for dataset in DATASETS:
     main(DATASET)