update security_sales_1

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.1.2.yaml

@@ -0,0 +1,4 @@
+include: security_sales_1.yaml
+task: security_sales_1-1.0-0.1.2
+description: "[問題]に対する[答え]を[選択肢]の中からアルファベットで選んでください。\n\n\n"
+doc_to_text: !function utils.doc_to_text_01_2

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.1.yaml

@@ -0,0 +1,6 @@
+include: security_sales_1.yaml
+task: security_sales_1-1.0-0.1
+description: "[問題]に対する[答え]を[選択肢]の中から選んでください。\n\n\n"
+doc_to_text: !function utils.doc_to_text_01
+doc_to_target: !function utils.doc_to_target_01
+doc_to_choice: !function utils.doc_to_choices_01

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.2.1.yaml

@@ -0,0 +1,4 @@
+include: security_sales_1-1.0-0.1.yaml
+task: security_sales_1-1.0-0.2.1
+description: "与えられた選択肢の中から、最適な答えを選んでください。\n\n\n"
+doc_to_text: !function utils.doc_to_text_02_1

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.2.yaml

@@ -0,0 +1,6 @@
+include: security_sales_1.yaml
+task: security_sales_1-1.0-0.2
+description: "質問と回答の選択肢を入力として受け取り、選択肢から回答を選択してください。なお、回答は選択肢の番号(例:0)でするものとします。\n\n\n"
+doc_to_text: !function utils.doc_to_text_02
+doc_to_target: !function utils.doc_to_target_02
+doc_to_choice: !function utils.doc_to_choices_02

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.3.yaml

@@ -0,0 +1,4 @@
+include: security_sales_1-1.0-0.1.yaml
+task: security_sales_1-1.0-0.3
+description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n### 指示:\n与えられた選択肢の中から、最適な答えを選んでください。\n\n\n"
+doc_to_text: !function utils.doc_to_text_03

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.4.yaml

@@ -0,0 +1,5 @@
+include: security_sales_1-1.0-0.1.yaml
+task: security_sales_1-1.0-0.4
+description: "ユーザー: 与えられた選択肢の中から、最適な答えを選んでください。<NL>システム: 分かりました。<NL>"
+fewshot_delimiter: "<NL>"
+doc_to_text: !function utils.doc_to_text_04

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.5.yaml

@@ -0,0 +1,5 @@
+include: security_sales_1-1.0-0.4.yaml
+task: security_sales_1-1.0-0.5
+description: "ユーザー: 与えられた選択肢の中から、最適な答えを選んでください。\nシステム: 分かりました。\n\n"
+fewshot_delimiter: "\n"
+doc_to_text: !function utils.doc_to_text_05

jlm_fin_eval/tasks/security_sales_1/security_sales_1-1.0-0.6.yaml

@@ -0,0 +1,5 @@
+include: security_sales_1-1.0-0.1.yaml
+task: security_sales_1-1.0-0.6
+description: "<s>[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\n<</SYS>>\n\n\n"
+fewshot_delimiter: " </s><s>[INST] "
+doc_to_text: !function utils.doc_to_text_06

jlm_fin_eval/tasks/security_sales_1/security_sales_1.yaml

@@ -0,0 +1,48 @@
+task: security_sales_1
+dataset_path: jlm_fin_eval/datasets/security_sales_1/security_sales_1.py
+dataset_name: security_sales_1
+output_type: multiple_choice
+training_split: null
+validation_split: null
+test_split: test
+description: "以下の問題の適切な答えを選択肢から選んでアルファベットで答えなさい。\n\n\n"
+target_delimiter: ""
+fewshot_delimiter: "\n\n"
+doc_to_text: !function utils.doc_to_text_00
+doc_to_target: !function utils.doc_to_target_alphabet
+doc_to_choice: !function utils.doc_to_choices_alphabet
+should_decontaminate: true
+doc_to_decontamination_query: sentence
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: map
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_2
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_3
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_4
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_2_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_3_norm
+    aggregation: mean
+    higher_is_better: true
+  - metric: map_4_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  - version: 1.0