diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py new file mode 100755 index 00000000..8f95c058 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py @@ -0,0 +1,64 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = 'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.\n' + return [ + { + "role": "system", + "content": "أنت خبير في تحليل و تصنيف التغريدات.", + }, + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt += "هنا بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية" + + out_prompt += ( + f"مثال {index}:\n" f"التغريدة: {example['input']}\n" f"التصنيف: {label}\n\n" + ) + + out_prompt += f"التغريدة: {input_sample}\nالتصنيف: \n" + + return out_prompt + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py new file mode 100755 index 00000000..bfae7aba --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py @@ -0,0 +1,80 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = ( + "Classify the 'tweet' as subjective or objective. Provide only label.\n" + ) + return [ + { + "role": "system", + "content": "You are an expert in tweet classification and analysis.", + }, + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "Here are some examples:\n\n" + for index, example in enumerate(examples): + label = "objective" if example["label"] == "OBJ" else "subjective" + + out_prompt = ( + out_prompt + + "Example " + + str(index) + + ":" + + "\n" + + "tweet: " + + example["input"] + + "\nlabel: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "tweet: " + input_sample + "\nlabel: \n" + + return out_prompt + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "obj" in label or "موضوعي" in label: + return "OBJ" + elif ( + "subj" in label + or "غير" in label + or "لا" in label + or "ذاتي" in label + or "ليس" in label + ): + return "SUBJ" + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py new file mode 100755 index 00000000..33c0f5f0 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py @@ -0,0 +1,69 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = 'صنف التغريدة الى "subjective" أو "objective". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.\n' + return [ + { + "role": "system", + "content": "أنت خبير في تحليل و تصنيف التغريدات.", + }, + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt += "هنا بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "objective" if example["label"] == "OBJ" else "subjective" + + out_prompt += ( + f"مثال {index}:\n" f"التغريدة: {example['input']}\n" f"التصنيف: {label}\n\n" + ) + + out_prompt += f"التغريدة: {input_sample}\nالتصنيف: \n" + + return out_prompt + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "obj" in label or "موضوعي" in label: + return "OBJ" + elif ( + "subj" in label + or "غير" in label + or "لا" in label + or "ذاتي" in label + or "ليس" in label + ): + return "SUBJ" + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py new file mode 100755 index 00000000..7b9cc990 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py @@ -0,0 +1,49 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + prompt_string = ( + f'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط دون أي تبرير إضافي.\n' + f"التغريدة: {input_sample}\n" + f"التصنيف: \n" + ) + return [ + { + "role": "system", + "content": "أنت خبير في تحليل و تصنيف التغريدات.", + }, + { + "role": "user", + "content": prompt_string, + }, + ] + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py new file mode 100755 index 00000000..2568590a --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py @@ -0,0 +1,58 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + prompt_string = ( + f'Classify the tweet as "objective" or "subjective". Provide only the label.\n\n' + f"tweet: {input_sample}\n" + f"label: \n" + ) + return [ + { + "role": "system", + "content": "You are an expert in tweet classification and analysis.", + }, + { + "role": "user", + "content": prompt_string, + }, + ] + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "obj" in label or "موضوعي" in label: + return "OBJ" + elif ( + "subj" in label + or "غير" in label + or "لا" in label + or "ذاتي" in label + or "ليس" in label + ): + return "SUBJ" + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py new file mode 100755 index 00000000..56d3a6d2 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py @@ -0,0 +1,54 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import OpenAIModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "GPT-4o-2024-05-22", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": OpenAIModel, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + prompt_string = ( + 'صنف التغريدة الى "objective" أو "subjective". قم بتقديم التصنيف دون أي تبرير إضافي.\n' + f"التغريدة: {input_sample}\n" + f"التصنيف: \n" + ) + return [ + { + "role": "system", + "content": "أنت خبير في تحليل و تصنيف التغريدات.", + }, + { + "role": "user", + "content": prompt_string, + }, + ] + + +def post_process(response): + label = response["choices"][0]["message"]["content"].strip().lower() + if "obj" in label or "موضوعي" in label: + return "OBJ" + elif ( + "subj" in label + or "غير" in label + or "لا" in label + or "ذاتي" in label + or "ليس" in label + ): + return "SUBJ" + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py new file mode 100755 index 00000000..0d49d7e8 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py @@ -0,0 +1,72 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.:\n' + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية" + + out_prompt = ( + out_prompt + + "مثال " + + str(index) + + ":" + + "\n" + + "التغريدة: " + + example["input"] + + "\التصنيف: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n" + + return out_prompt + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py new file mode 100755 index 00000000..fdd6d001 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py @@ -0,0 +1,71 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = 'Classify the tweet as "Objective" or "Subjective". Provide the classification for the last tweet only, do not provide any additional justification:\n' + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "Here are some examples:\n\n" + for index, example in enumerate(examples): + label = "Objective" if example["label"] == "OBJ" else "Subjective" + + out_prompt = ( + out_prompt + + "Example " + + str(index) + + ":\n" + + "Tweet: " + + example["input"] + + "\nClassification: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "Tweet: " + input_sample + "\nClassification: \n" + + return out_prompt + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py new file mode 100755 index 00000000..1b0e1a01 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py @@ -0,0 +1,71 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = "صنف الجملة على أنها subjective أو objective. قدم فقط التسمية.\n\nالنص: {input_sample}\التصنيف: " + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "objective" if example["label"] == "OBJ" else "subjective" + + out_prompt = ( + out_prompt + + "مثال " + + str(index) + + ":" + + "\n" + + "التغريدة: " + + example["input"] + + "\التصنيف: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n" + + return out_prompt + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + print(label) + label = label.lower() + if "subjective" in label or "sub" in label: + return "SUBJ" + return "OBJ" + return label diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py new file mode 100755 index 00000000..206cae81 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py @@ -0,0 +1,49 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + return [ + { + "role": "user", + "content": ( + 'Classify the tweet as "Objective" or "Subjective". Provide only the classification.\n\n' + + "Tweet: " + + input_sample + + "Classification: " + ), + } + ] + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + # print(label) + + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py new file mode 100755 index 00000000..1a3ea668 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py @@ -0,0 +1,42 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + base_prompt = f"صنف الجملة على أنها subjective أو objective. قدم فقط التسمية.\n\nالنص: {input_sample}\nالتصنيف: " + + return [ + { + "role": "user", + "content": base_prompt, + }, + ] + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + + if "subjective" in label or "subj" "ليس" in label or "غير" in label: + return "SUBJ" + return "OBJ" diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py new file mode 100755 index 00000000..8760822d --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py @@ -0,0 +1,44 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import FastChatModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "jais-13b-chat", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": FastChatModel, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + base_prompt = f'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط.\n\nالتغريدة: {input_sample}\nالتصنيف: ' + return [ + { + "role": "user", + "content": base_prompt, + }, + ] + + +def post_process(response): + label = ( + response["choices"][0]["message"]["content"].lower().replace(".", "").strip() + ) + + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py new file mode 100755 index 00000000..452ea072 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py @@ -0,0 +1,78 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.:\n' + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية" + + out_prompt = ( + out_prompt + + "مثال " + + str(index) + + ":" + + "\n" + + "التغريدة: " + + example["input"] + + "\التصنيف: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n" + + return out_prompt + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py new file mode 100755 index 00000000..e9d2778c --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py @@ -0,0 +1,77 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = 'Classify the tweet as "Objective" or "Subjective". Provide the classification for the last tweet only, do not provide any additional justification:\n' + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "Here are some examples:\n\n" + for index, example in enumerate(examples): + label = "Objective" if example["label"] == "OBJ" else "Subjective" + + out_prompt = ( + out_prompt + + "Example " + + str(index) + + ":\n" + + "Tweet: " + + example["input"] + + "\nClassification: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "Tweet: " + input_sample + "\nClassification: \n" + + return out_prompt + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py new file mode 100755 index 00000000..db6da233 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py @@ -0,0 +1,78 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 30, + }, + "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}}, + } + + +def prompt(input_sample, examples): + base_prompt = 'صنف التغريدة الى "subjective" أو "objective". قم بتقديم التصنيف فقط.' + return [ + { + "role": "user", + "content": few_shot_prompt(input_sample, base_prompt, examples), + }, + ] + + +def few_shot_prompt(input_sample, base_prompt, examples): + out_prompt = base_prompt + "\n" + out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n" + for index, example in enumerate(examples): + label = "objective" if example["label"] == "OBJ" else "subjective" + + out_prompt = ( + out_prompt + + "مثال " + + str(index) + + ":" + + "\n" + + "التغريدة: " + + example["input"] + + "التصنيف: " + + label + + "\n\n" + ) + + out_prompt = out_prompt + "التغريدة: " + input_sample + "التصنيف: \n" + + return out_prompt + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py new file mode 100755 index 00000000..cca523c3 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py @@ -0,0 +1,58 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 3, + }, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + return [ + { + "role": "user", + "content": ( + ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط.\n\n' + + "التغريدة: " + + input_sample + + "التصنيف: " + ), + } + ] + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py new file mode 100755 index 00000000..ada805a4 --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py @@ -0,0 +1,58 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 3, + }, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + return [ + { + "role": "user", + "content": ( + 'Classify the tweet as "Objective" or "Subjective". Provide only the classification.\n\n' + + "Tweet: " + + input_sample + + "Classification: " + ), + } + ] + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py new file mode 100755 index 00000000..3ded318b --- /dev/null +++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py @@ -0,0 +1,58 @@ +from llmebench.datasets import CT23SubjectivityDataset +from llmebench.models import AzureModel +from llmebench.tasks import SubjectivityTask + + +def metadata(): + return { + "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam", + "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)", + "model": "Llama-3.1-8B-Instruct", + "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).", + } + + +def config(): + return { + "dataset": CT23SubjectivityDataset, + "task": SubjectivityTask, + "model": AzureModel, + "model_args": { + "class_labels": ["SUBJ", "OBJ"], + "max_tries": 3, + }, + "general_args": {"test_split": "ar/dev"}, + } + + +def prompt(input_sample): + return [ + { + "role": "user", + "content": ( + 'صنف الجملة الى "subjective" أو "objective". قم بتقديم التصنيف فقط.\n\n' + + "التغريدة: " + + input_sample + + "\nالتصنيف: " + ), + } + ] + + +def post_process(response): + # print(response) + if "output" in response: + # if "content" in response["messages"]: + label = response["output"].strip() + label = label.replace("", "") + label = label.replace("", "") + label = label.lower() + else: + print("Response .. " + str(response)) + return None + if "غير" in label or "subj" in label or "not" in label or "ليس" in label: + return "SUBJ" + elif "موضوعية" in label or "obj" in label: + return "OBJ" + else: + return None