From 80463f7017edbdd941a3540a406a7a47944cd600 Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 21 Oct 2023 14:48:04 +0900 Subject: [PATCH 1/6] need additional_special_tokens argument for HFLM initializer --- lm_eval/models/gpt2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py index c69b6142553..746eae6d01d 100644 --- a/lm_eval/models/gpt2.py +++ b/lm_eval/models/gpt2.py @@ -19,6 +19,7 @@ def __init__( load_in_8bit: Optional[bool] = False, trust_remote_code: Optional[bool] = False, use_fast: Optional[bool] = True, + additional_special_tokens: Optional[str] = None ): super().__init__() @@ -64,6 +65,7 @@ def __init__( revision=revision, trust_remote_code=trust_remote_code, use_fast=use_fast, + additional_special_tokens=additional_special_tokens ) self.vocab_size = self.tokenizer.vocab_size From edc0190602ff73bae9c2c220e8955dcb6fd4446b Mon Sep 17 00:00:00 2001 From: kumapo Date: Wed, 18 Oct 2023 23:45:54 +0900 Subject: [PATCH 2/6] need offload_folder argument for HFLM initializer --- lm_eval/models/gpt2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py index 746eae6d01d..e703342558b 100644 --- a/lm_eval/models/gpt2.py +++ b/lm_eval/models/gpt2.py @@ -13,6 +13,7 @@ def __init__( low_cpu_mem_usage=None, torch_dtype=None, device_map=None, + offload_folder=None, subfolder=None, tokenizer=None, batch_size=1, @@ -50,6 +51,7 @@ def __init__( low_cpu_mem_usage=low_cpu_mem_usage, torch_dtype=torch_dtype, device_map=device_map, + offload_folder=offload_folder, revision=revision, trust_remote_code=trust_remote_code, ).eval() From 08937786c89d3b5a5b62133a94310e33f31fb568 Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 28 Oct 2023 17:54:58 +0900 Subject: [PATCH 3/6] need JCommonsenseQAWithJAAlpacaPromptV32 and 4 prompts --- lm_eval/tasks/ja/jcommonsenseqa.py | 123 ++++++++++++++++++ .../cyberagent-open-calm-3b/harness.jcqa.sh | 3 + models/llama2/llama2-7b-chat/harness.jcqa.sh | 3 + .../harness.jcqa.sh | 3 + .../harness.jcqa.sh | 3 + .../harness.jcqa.sh | 13 ++ .../harness.jcqa.sh | 13 ++ 7 files changed, 161 insertions(+) create mode 100644 models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh create mode 100644 models/llama2/llama2-7b-chat/harness.jcqa.sh create mode 100644 models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh create mode 100644 models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh create mode 100644 models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh create mode 100644 models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py index c7d94dc95a2..0d6cd482928 100644 --- a/lm_eval/tasks/ja/jcommonsenseqa.py +++ b/lm_eval/tasks/ja/jcommonsenseqa.py @@ -166,6 +166,24 @@ def doc_to_text(self, doc): return input_text +class JCommonsenseQAWithFintanPromptV22(JCommonsenseQAWithFintanPromptV21): + PROMPT_VERSION = "0.2.2" + + def doc_to_text(self, doc): + """ + 与えられた選択肢の中から、最適な答えを選んでください。 + + 選択肢： + - {choice0} + - {choice4} + 質問：{question} + 回答： + """ + choices = "\n".join([f"- {choice}" for choice in doc["choices"]]) + input_text = f"選択肢：\n{choices}\n質問：{doc['goal']}\n回答：" # question last + return input_text + + class JCommonsenseQAWithJAAlpacaPrompt(JCommonsenseQA): """ This prompt format was inspired by the below data in fujiki/japanese_alpaca_data. @@ -205,6 +223,42 @@ def doc_to_text(self, doc): return f"### 指示:\n{instruction_text}\n\n### 入力:\n{input_text}\n\n### 応答:\n" +class JCommonsenseQAWithJAAlpacaPromptV32(JCommonsenseQAWithJAAlpacaPrompt): + """ + This prompt format was inspired by the below data in fujiki/japanese_alpaca_data. + ``` + { + 'instruction': 'この課題では、以下の選択肢から文の出典を特定する必要があります。\n\n出力は以下から選択してください：\n- 新聞\n- 教科書\n- オンライン記事\n- 百科事典', + 'input': '彼はローマの政治家であり哲学者であり、史上最も偉大な軍事指導者の一人と考えられています。', + 'output': '百科事典' + } + ``` + Reference: + - data: https://huggingface.co/datasets/fujiki/japanese_alpaca_data + - code: https://github.com/Stability-AI/gpt-neox/blob/c130a4edc1120dccec8f02a34eb60d3e8f484cd3/finetune/finetune_base_ja.py#LL118C23-L127C11 + """ + + PROMPT_VERSION = "0.3.2" + + def doc_to_text(self, doc): + """ + 以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。 + + ### 指示: + {instruction} + + ### 入力: + {input} + + ### 応答: + {response} + """ + instruction_text = self.INSTRUCTION + f"\n質問：{doc['goal']}" + choices = "\n".join([f"- {choice}" for choice in doc["choices"]]) + input_text = f"出力は以下から選択してください：\n{choices}" + return f"### 指示:\n{instruction_text}\n\n### 入力:\n{input_text}\n\n### 応答:\n" # question first + + class JCommonsenseQAWithRinnaInstructionSFT(JCommonsenseQA): """ Reference: @@ -223,6 +277,20 @@ def doc_to_text(self, doc): return f"ユーザー: {input_text}{self.SEP}システム: " +class JCommonsenseQAWithRinnaInstructionSFTV42(JCommonsenseQAWithRinnaInstructionSFT): + """ + Reference: + - HF Hub: https://huggingface.co/rinna/japanese-gpt-neox-3.6b-instruction-sft + """ + + PROMPT_VERSION = "0.4.2" + + def doc_to_text(self, doc): + choices = self.SEP.join([f"- {choice}" for choice in doc["choices"]]) + input_text = f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" # question last + return f"ユーザー: {input_text}{self.SEP}システム: " + + class JCommonsenseQAWithRinnaBilingualInstructionSFT( JCommonsenseQAWithRinnaInstructionSFT ): @@ -237,6 +305,22 @@ class JCommonsenseQAWithRinnaBilingualInstructionSFT( FEWSHOT_SEP = "\n" +class JCommonsenseQAWithRinnaBilingualInstructionSFTV52( + JCommonsenseQAWithRinnaBilingualInstructionSFT +): + """ + Reference: + - HF Hub: https://huggingface.co/rinna/bilingual-gpt-neox-4b-instruction-sft + """ + + PROMPT_VERSION = "0.5.2" + + def doc_to_text(self, doc): + choices = self.SEP.join([f"- {choice}" for choice in doc["choices"]]) + input_text = f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" # question last + return f"ユーザー: {input_text}{self.SEP}システム: " + + class JCommonsenseQAWithLlama2(JCommonsenseQA): """ This prompt version follows the Llama2-chat's prompt format: @@ -262,6 +346,7 @@ def doc_to_text(self, doc): Insert the following prompt into `{{ user_msg }}`, which is based on prompt version 0.3 ``` 与えられた選択肢の中から、最適な答えを選んでください。出力は以下から選択してください： + - choice0 ... - choice4 @@ -275,14 +360,52 @@ def doc_to_text(self, doc): return f"{instruction_text}\n\n{input_text} [/INST] " +class JCommonsenseQAWithLlama2V62(JCommonsenseQAWithLlama2): + """ + This prompt version follows the Llama2-chat's prompt format: + ``` + [INST] <> + {{ system_prompt }} + <> + + {{ user_msg_1 }} [/INST] {{ model_answer_1 }} [INST] {{ user_msg_2 }} [/INST] + ``` + reference: https://huggingface.co/blog/llama2#how-to-prompt-llama-2 + """ + + PROMPT_VERSION = "0.6.2" + + def doc_to_text(self, doc): + """ + Insert the following prompt into `{{ user_msg }}`, which is based on prompt version 0.3 + ``` + 与えられた選択肢の中から、最適な答えを選んでください。質問：... + + 出力は以下から選択してください： + - choice0 + ... + - choice4 [/INST] + ``` + """ + choices = "\n".join([f"- {choice}" for choice in doc["choices"]]) + input_text = f"質問：{doc['goal']}" + instruction_text = self.INSTRUCTION + input_text + choices = f"出力は以下から選択してください：\n{choices}" + return f"{instruction_text}\n\n{choices} [/INST] " # question first + + VERSIONS = [ JCommonsenseQA, JCommonsenseQAWithFintanPrompt, JCommonsenseQAWithFintanPromptV21, JCommonsenseQAWithJAAlpacaPrompt, + JCommonsenseQAWithJAAlpacaPromptV32, JCommonsenseQAWithRinnaInstructionSFT, + JCommonsenseQAWithRinnaInstructionSFTV42, JCommonsenseQAWithRinnaBilingualInstructionSFT, + JCommonsenseQAWithRinnaBilingualInstructionSFTV52, JCommonsenseQAWithLlama2, + JCommonsenseQAWithLlama2V62 ] diff --git a/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh b/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh new file mode 100644 index 00000000000..92bee44e9c7 --- /dev/null +++ b/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh @@ -0,0 +1,3 @@ +MODEL_ARGS="pretrained=cyberagent/open-calm-3b" +TASK="jcommonsenseqa-1.1-0.2.1" +python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/cyberagent/cyberagent-open-calm-3b/result.jcqa.json" diff --git a/models/llama2/llama2-7b-chat/harness.jcqa.sh b/models/llama2/llama2-7b-chat/harness.jcqa.sh new file mode 100644 index 00000000000..fea92c6818d --- /dev/null +++ b/models/llama2/llama2-7b-chat/harness.jcqa.sh @@ -0,0 +1,3 @@ +MODEL_ARGS="pretrained=meta-llama/Llama-2-7b-chat-hf,use_accelerate=True,dtype=auto" +TASK="jcommonsenseqa-1.1-0.3" +python main.py --model hf-causal-experimental --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/llama2/llama2-7b-chat/result.jcqa.json" --batch_size 2 diff --git a/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh b/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh new file mode 100644 index 00000000000..83e46d6602b --- /dev/null +++ b/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh @@ -0,0 +1,3 @@ +MODEL_ARGS="pretrained=rinna/bilingual-gpt-neox-4b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto" +TASK="jcommonsenseqa-1.1-0.5" +python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/rinna/rinna-bilingual-gpt-neox-4b/result.jcqa.json" diff --git a/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh b/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh new file mode 100644 index 00000000000..7adcd6017ca --- /dev/null +++ b/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh @@ -0,0 +1,3 @@ +MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto" +TASK="jcommonsenseqa-1.1-0.4" +python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.jcqa.json" diff --git a/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh b/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh new file mode 100644 index 00000000000..46d0770988c --- /dev/null +++ b/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -eu +MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,load_in_8bit=True,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']" +TASK="jcommonsenseqa-1.1-0.2.1" +NUM_FEW_SHOTS="3" +python main.py \ + --model hf-causal \ + --model_args $MODEL_ARGS \ + --tasks $TASK \ + --num_fewshot $NUM_FEW_SHOTS \ + --device "cuda" \ + --no_cache \ + --output_path "models/stablelm/stablelm-ja-base-alpha-7b/result.jcqa.json" diff --git a/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh b/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh new file mode 100644 index 00000000000..5cc18edce2f --- /dev/null +++ b/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh @@ -0,0 +1,13 @@ +#!/bin/bash +MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-instruct-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,load_in_8bit=True,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']" +TASK="jcommonsenseqa-1.1-0.3" +NUM_FEWSHOT="3" +OUTPUT_PATH="models/stabilityai/japanese-stablelm-instruct-alpha-7b/result.jcqa.json" +python main.py \ + --model hf-causal \ + --model_args $MODEL_ARGS \ + --tasks $TASK \ + --num_fewshot $NUM_FEWSHOT \ + --device "cuda" \ + --no_cache \ + --output_path $OUTPUT_PATH From 380cdcabb3b9ac4b908467affe9c000c749d88cf Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 4 Nov 2023 17:25:26 +0900 Subject: [PATCH 4/6] fix lint issues --- lm_eval/models/gpt2.py | 4 ++-- lm_eval/tasks/ja/jcommonsenseqa.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py index e703342558b..c79331b1df8 100644 --- a/lm_eval/models/gpt2.py +++ b/lm_eval/models/gpt2.py @@ -20,7 +20,7 @@ def __init__( load_in_8bit: Optional[bool] = False, trust_remote_code: Optional[bool] = False, use_fast: Optional[bool] = True, - additional_special_tokens: Optional[str] = None + additional_special_tokens: Optional[str] = None, ): super().__init__() @@ -67,7 +67,7 @@ def __init__( revision=revision, trust_remote_code=trust_remote_code, use_fast=use_fast, - additional_special_tokens=additional_special_tokens + additional_special_tokens=additional_special_tokens, ) self.vocab_size = self.tokenizer.vocab_size diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py index 0d6cd482928..b8dc7810be5 100644 --- a/lm_eval/tasks/ja/jcommonsenseqa.py +++ b/lm_eval/tasks/ja/jcommonsenseqa.py @@ -180,7 +180,7 @@ def doc_to_text(self, doc): 回答： """ choices = "\n".join([f"- {choice}" for choice in doc["choices"]]) - input_text = f"選択肢：\n{choices}\n質問：{doc['goal']}\n回答：" # question last + input_text = f"選択肢：\n{choices}\n質問：{doc['goal']}\n回答：" # question last return input_text @@ -256,7 +256,7 @@ def doc_to_text(self, doc): instruction_text = self.INSTRUCTION + f"\n質問：{doc['goal']}" choices = "\n".join([f"- {choice}" for choice in doc["choices"]]) input_text = f"出力は以下から選択してください：\n{choices}" - return f"### 指示:\n{instruction_text}\n\n### 入力:\n{input_text}\n\n### 応答:\n" # question first + return f"### 指示:\n{instruction_text}\n\n### 入力:\n{input_text}\n\n### 応答:\n" # question first class JCommonsenseQAWithRinnaInstructionSFT(JCommonsenseQA): @@ -287,7 +287,9 @@ class JCommonsenseQAWithRinnaInstructionSFTV42(JCommonsenseQAWithRinnaInstructio def doc_to_text(self, doc): choices = self.SEP.join([f"- {choice}" for choice in doc["choices"]]) - input_text = f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" # question last + input_text = ( + f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" + ) # question last return f"ユーザー: {input_text}{self.SEP}システム: " @@ -317,7 +319,9 @@ class JCommonsenseQAWithRinnaBilingualInstructionSFTV52( def doc_to_text(self, doc): choices = self.SEP.join([f"- {choice}" for choice in doc["choices"]]) - input_text = f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" # question last + input_text = ( + f"選択肢：{self.SEP}{choices}" + f"質問：{doc['goal']}{self.SEP}" + ) # question last return f"ユーザー: {input_text}{self.SEP}システム: " @@ -379,7 +383,7 @@ def doc_to_text(self, doc): """ Insert the following prompt into `{{ user_msg }}`, which is based on prompt version 0.3 ``` - 与えられた選択肢の中から、最適な答えを選んでください。質問：... + 与えられた選択肢の中から、最適な答えを選んでください。質問：... 出力は以下から選択してください： - choice0 @@ -391,7 +395,7 @@ def doc_to_text(self, doc): input_text = f"質問：{doc['goal']}" instruction_text = self.INSTRUCTION + input_text choices = f"出力は以下から選択してください：\n{choices}" - return f"{instruction_text}\n\n{choices} [/INST] " # question first + return f"{instruction_text}\n\n{choices} [/INST] " # question first VERSIONS = [ @@ -405,7 +409,7 @@ def doc_to_text(self, doc): JCommonsenseQAWithRinnaBilingualInstructionSFT, JCommonsenseQAWithRinnaBilingualInstructionSFTV52, JCommonsenseQAWithLlama2, - JCommonsenseQAWithLlama2V62 + JCommonsenseQAWithLlama2V62, ] From d25fec09597c687f032990326a71d93440dcdfc8 Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 4 Nov 2023 17:59:26 +0900 Subject: [PATCH 5/6] wont need harness.jcqa.sh --- .../cyberagent-open-calm-3b/harness.jcqa.sh | 3 --- models/llama2/llama2-7b-chat/harness.jcqa.sh | 3 --- .../harness.jcqa.sh | 3 --- .../harness.jcqa.sh | 3 --- .../harness.jcqa.sh | 13 ------------- .../harness.jcqa.sh | 13 ------------- 6 files changed, 38 deletions(-) delete mode 100644 models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh delete mode 100644 models/llama2/llama2-7b-chat/harness.jcqa.sh delete mode 100644 models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh delete mode 100644 models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh delete mode 100644 models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh delete mode 100644 models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh diff --git a/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh b/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh deleted file mode 100644 index 92bee44e9c7..00000000000 --- a/models/cyberagent/cyberagent-open-calm-3b/harness.jcqa.sh +++ /dev/null @@ -1,3 +0,0 @@ -MODEL_ARGS="pretrained=cyberagent/open-calm-3b" -TASK="jcommonsenseqa-1.1-0.2.1" -python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/cyberagent/cyberagent-open-calm-3b/result.jcqa.json" diff --git a/models/llama2/llama2-7b-chat/harness.jcqa.sh b/models/llama2/llama2-7b-chat/harness.jcqa.sh deleted file mode 100644 index fea92c6818d..00000000000 --- a/models/llama2/llama2-7b-chat/harness.jcqa.sh +++ /dev/null @@ -1,3 +0,0 @@ -MODEL_ARGS="pretrained=meta-llama/Llama-2-7b-chat-hf,use_accelerate=True,dtype=auto" -TASK="jcommonsenseqa-1.1-0.3" -python main.py --model hf-causal-experimental --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/llama2/llama2-7b-chat/result.jcqa.json" --batch_size 2 diff --git a/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh b/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh deleted file mode 100644 index 83e46d6602b..00000000000 --- a/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-ppo/harness.jcqa.sh +++ /dev/null @@ -1,3 +0,0 @@ -MODEL_ARGS="pretrained=rinna/bilingual-gpt-neox-4b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto" -TASK="jcommonsenseqa-1.1-0.5" -python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/rinna/rinna-bilingual-gpt-neox-4b/result.jcqa.json" diff --git a/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh b/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh deleted file mode 100644 index 7adcd6017ca..00000000000 --- a/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jcqa.sh +++ /dev/null @@ -1,3 +0,0 @@ -MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto" -TASK="jcommonsenseqa-1.1-0.4" -python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.jcqa.json" diff --git a/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh b/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh deleted file mode 100644 index 46d0770988c..00000000000 --- a/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -eu -MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,load_in_8bit=True,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']" -TASK="jcommonsenseqa-1.1-0.2.1" -NUM_FEW_SHOTS="3" -python main.py \ - --model hf-causal \ - --model_args $MODEL_ARGS \ - --tasks $TASK \ - --num_fewshot $NUM_FEW_SHOTS \ - --device "cuda" \ - --no_cache \ - --output_path "models/stablelm/stablelm-ja-base-alpha-7b/result.jcqa.json" diff --git a/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh b/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh deleted file mode 100644 index 5cc18edce2f..00000000000 --- a/models/stabilityai/stabilityai-japanese-stablelm-instruct-alpha-7b/harness.jcqa.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-instruct-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,load_in_8bit=True,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']" -TASK="jcommonsenseqa-1.1-0.3" -NUM_FEWSHOT="3" -OUTPUT_PATH="models/stabilityai/japanese-stablelm-instruct-alpha-7b/result.jcqa.json" -python main.py \ - --model hf-causal \ - --model_args $MODEL_ARGS \ - --tasks $TASK \ - --num_fewshot $NUM_FEWSHOT \ - --device "cuda" \ - --no_cache \ - --output_path $OUTPUT_PATH From e686ff090c2fdb17a0a30d368fc04cac6965ad08 Mon Sep 17 00:00:00 2001 From: kumapo Date: Sat, 4 Nov 2023 18:23:14 +0900 Subject: [PATCH 6/6] fix --- lm_eval/tasks/ja/jcommonsenseqa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py index b8dc7810be5..6629557d5cb 100644 --- a/lm_eval/tasks/ja/jcommonsenseqa.py +++ b/lm_eval/tasks/ja/jcommonsenseqa.py @@ -402,6 +402,7 @@ def doc_to_text(self, doc): JCommonsenseQA, JCommonsenseQAWithFintanPrompt, JCommonsenseQAWithFintanPromptV21, + JCommonsenseQAWithFintanPromptV22, JCommonsenseQAWithJAAlpacaPrompt, JCommonsenseQAWithJAAlpacaPromptV32, JCommonsenseQAWithRinnaInstructionSFT,