diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py
new file mode 100755
index 00000000..8f95c058
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Arabic.py
@@ -0,0 +1,64 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = 'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.\n'
+ return [
+ {
+ "role": "system",
+ "content": "أنت خبير في تحليل و تصنيف التغريدات.",
+ },
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt += "هنا بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية"
+
+ out_prompt += (
+ f"مثال {index}:\n" f"التغريدة: {example['input']}\n" f"التصنيف: {label}\n\n"
+ )
+
+ out_prompt += f"التغريدة: {input_sample}\nالتصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py
new file mode 100755
index 00000000..bfae7aba
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_English.py
@@ -0,0 +1,80 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = (
+ "Classify the 'tweet' as subjective or objective. Provide only label.\n"
+ )
+ return [
+ {
+ "role": "system",
+ "content": "You are an expert in tweet classification and analysis.",
+ },
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "Here are some examples:\n\n"
+ for index, example in enumerate(examples):
+ label = "objective" if example["label"] == "OBJ" else "subjective"
+
+ out_prompt = (
+ out_prompt
+ + "Example "
+ + str(index)
+ + ":"
+ + "\n"
+ + "tweet: "
+ + example["input"]
+ + "\nlabel: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "tweet: " + input_sample + "\nlabel: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "obj" in label or "موضوعي" in label:
+ return "OBJ"
+ elif (
+ "subj" in label
+ or "غير" in label
+ or "لا" in label
+ or "ذاتي" in label
+ or "ليس" in label
+ ):
+ return "SUBJ"
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py
new file mode 100755
index 00000000..33c0f5f0
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_FewShot_Mixed.py
@@ -0,0 +1,69 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = 'صنف التغريدة الى "subjective" أو "objective". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.\n'
+ return [
+ {
+ "role": "system",
+ "content": "أنت خبير في تحليل و تصنيف التغريدات.",
+ },
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt += "هنا بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "objective" if example["label"] == "OBJ" else "subjective"
+
+ out_prompt += (
+ f"مثال {index}:\n" f"التغريدة: {example['input']}\n" f"التصنيف: {label}\n\n"
+ )
+
+ out_prompt += f"التغريدة: {input_sample}\nالتصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "obj" in label or "موضوعي" in label:
+ return "OBJ"
+ elif (
+ "subj" in label
+ or "غير" in label
+ or "لا" in label
+ or "ذاتي" in label
+ or "ليس" in label
+ ):
+ return "SUBJ"
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py
new file mode 100755
index 00000000..7b9cc990
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Arabic.py
@@ -0,0 +1,49 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ prompt_string = (
+ f'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط دون أي تبرير إضافي.\n'
+ f"التغريدة: {input_sample}\n"
+ f"التصنيف: \n"
+ )
+ return [
+ {
+ "role": "system",
+ "content": "أنت خبير في تحليل و تصنيف التغريدات.",
+ },
+ {
+ "role": "user",
+ "content": prompt_string,
+ },
+ ]
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py
new file mode 100755
index 00000000..2568590a
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_English.py
@@ -0,0 +1,58 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ prompt_string = (
+ f'Classify the tweet as "objective" or "subjective". Provide only the label.\n\n'
+ f"tweet: {input_sample}\n"
+ f"label: \n"
+ )
+ return [
+ {
+ "role": "system",
+ "content": "You are an expert in tweet classification and analysis.",
+ },
+ {
+ "role": "user",
+ "content": prompt_string,
+ },
+ ]
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "obj" in label or "موضوعي" in label:
+ return "OBJ"
+ elif (
+ "subj" in label
+ or "غير" in label
+ or "لا" in label
+ or "ذاتي" in label
+ or "ليس" in label
+ ):
+ return "SUBJ"
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py
new file mode 100755
index 00000000..56d3a6d2
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_GPT4_ZeroShot_Mixed.py
@@ -0,0 +1,54 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import OpenAIModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "GPT-4o-2024-05-22",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": OpenAIModel,
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ prompt_string = (
+ 'صنف التغريدة الى "objective" أو "subjective". قم بتقديم التصنيف دون أي تبرير إضافي.\n'
+ f"التغريدة: {input_sample}\n"
+ f"التصنيف: \n"
+ )
+ return [
+ {
+ "role": "system",
+ "content": "أنت خبير في تحليل و تصنيف التغريدات.",
+ },
+ {
+ "role": "user",
+ "content": prompt_string,
+ },
+ ]
+
+
+def post_process(response):
+ label = response["choices"][0]["message"]["content"].strip().lower()
+ if "obj" in label or "موضوعي" in label:
+ return "OBJ"
+ elif (
+ "subj" in label
+ or "غير" in label
+ or "لا" in label
+ or "ذاتي" in label
+ or "ليس" in label
+ ):
+ return "SUBJ"
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py
new file mode 100755
index 00000000..0d49d7e8
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Arabic.py
@@ -0,0 +1,72 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.:\n'
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية"
+
+ out_prompt = (
+ out_prompt
+ + "مثال "
+ + str(index)
+ + ":"
+ + "\n"
+ + "التغريدة: "
+ + example["input"]
+ + "\التصنيف: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py
new file mode 100755
index 00000000..fdd6d001
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_English.py
@@ -0,0 +1,71 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = 'Classify the tweet as "Objective" or "Subjective". Provide the classification for the last tweet only, do not provide any additional justification:\n'
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "Here are some examples:\n\n"
+ for index, example in enumerate(examples):
+ label = "Objective" if example["label"] == "OBJ" else "Subjective"
+
+ out_prompt = (
+ out_prompt
+ + "Example "
+ + str(index)
+ + ":\n"
+ + "Tweet: "
+ + example["input"]
+ + "\nClassification: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "Tweet: " + input_sample + "\nClassification: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py
new file mode 100755
index 00000000..1b0e1a01
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_FewShot_Mixed.py
@@ -0,0 +1,71 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = "صنف الجملة على أنها subjective أو objective. قدم فقط التسمية.\n\nالنص: {input_sample}\التصنيف: "
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "objective" if example["label"] == "OBJ" else "subjective"
+
+ out_prompt = (
+ out_prompt
+ + "مثال "
+ + str(index)
+ + ":"
+ + "\n"
+ + "التغريدة: "
+ + example["input"]
+ + "\التصنيف: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+ print(label)
+ label = label.lower()
+ if "subjective" in label or "sub" in label:
+ return "SUBJ"
+ return "OBJ"
+ return label
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py
new file mode 100755
index 00000000..206cae81
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_English.py
@@ -0,0 +1,49 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ return [
+ {
+ "role": "user",
+ "content": (
+ 'Classify the tweet as "Objective" or "Subjective". Provide only the classification.\n\n'
+ + "Tweet: "
+ + input_sample
+ + "Classification: "
+ ),
+ }
+ ]
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+ # print(label)
+
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py
new file mode 100755
index 00000000..1a3ea668
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_JAIS13b_ZeroShot_Mixed.py
@@ -0,0 +1,42 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ base_prompt = f"صنف الجملة على أنها subjective أو objective. قدم فقط التسمية.\n\nالنص: {input_sample}\nالتصنيف: "
+
+ return [
+ {
+ "role": "user",
+ "content": base_prompt,
+ },
+ ]
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+
+ if "subjective" in label or "subj" "ليس" in label or "غير" in label:
+ return "SUBJ"
+ return "OBJ"
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py
new file mode 100755
index 00000000..8760822d
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_Jais13b_ZeroShot_Arabic.py
@@ -0,0 +1,44 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "jais-13b-chat",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": FastChatModel,
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ base_prompt = f'صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط.\n\nالتغريدة: {input_sample}\nالتصنيف: '
+ return [
+ {
+ "role": "user",
+ "content": base_prompt,
+ },
+ ]
+
+
+def post_process(response):
+ label = (
+ response["choices"][0]["message"]["content"].lower().replace(".", "").strip()
+ )
+
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py
new file mode 100755
index 00000000..452ea072
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Arabic.py
@@ -0,0 +1,78 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط لأخر تغريدة، لا تقدم أي تبرير إضافي.:\n'
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "موضوعية" if example["label"] == "OBJ" else "غير موضوعية"
+
+ out_prompt = (
+ out_prompt
+ + "مثال "
+ + str(index)
+ + ":"
+ + "\n"
+ + "التغريدة: "
+ + example["input"]
+ + "\التصنيف: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "التغريدة: " + input_sample + "\التصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py
new file mode 100755
index 00000000..e9d2778c
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_English.py
@@ -0,0 +1,77 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = 'Classify the tweet as "Objective" or "Subjective". Provide the classification for the last tweet only, do not provide any additional justification:\n'
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "Here are some examples:\n\n"
+ for index, example in enumerate(examples):
+ label = "Objective" if example["label"] == "OBJ" else "Subjective"
+
+ out_prompt = (
+ out_prompt
+ + "Example "
+ + str(index)
+ + ":\n"
+ + "Tweet: "
+ + example["input"]
+ + "\nClassification: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "Tweet: " + input_sample + "\nClassification: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py
new file mode 100755
index 00000000..db6da233
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_FewShot_Mixed.py
@@ -0,0 +1,78 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 30,
+ },
+ "general_args": {"test_split": "ar/dev", "fewshot": {"train_split": "ar"}},
+ }
+
+
+def prompt(input_sample, examples):
+ base_prompt = 'صنف التغريدة الى "subjective" أو "objective". قم بتقديم التصنيف فقط.'
+ return [
+ {
+ "role": "user",
+ "content": few_shot_prompt(input_sample, base_prompt, examples),
+ },
+ ]
+
+
+def few_shot_prompt(input_sample, base_prompt, examples):
+ out_prompt = base_prompt + "\n"
+ out_prompt = out_prompt + "هذه بعض الأمثلة:\n\n"
+ for index, example in enumerate(examples):
+ label = "objective" if example["label"] == "OBJ" else "subjective"
+
+ out_prompt = (
+ out_prompt
+ + "مثال "
+ + str(index)
+ + ":"
+ + "\n"
+ + "التغريدة: "
+ + example["input"]
+ + "التصنيف: "
+ + label
+ + "\n\n"
+ )
+
+ out_prompt = out_prompt + "التغريدة: " + input_sample + "التصنيف: \n"
+
+ return out_prompt
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py
new file mode 100755
index 00000000..cca523c3
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Arabic.py
@@ -0,0 +1,58 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 3,
+ },
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ return [
+ {
+ "role": "user",
+ "content": (
+ ' صنف التغريدة الى "موضوعية" أو "غير موضوعية". قم بتقديم التصنيف فقط.\n\n'
+ + "التغريدة: "
+ + input_sample
+ + "التصنيف: "
+ ),
+ }
+ ]
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py
new file mode 100755
index 00000000..ada805a4
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_English.py
@@ -0,0 +1,58 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 3,
+ },
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ return [
+ {
+ "role": "user",
+ "content": (
+ 'Classify the tweet as "Objective" or "Subjective". Provide only the classification.\n\n'
+ + "Tweet: "
+ + input_sample
+ + "Classification: "
+ ),
+ }
+ ]
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None
diff --git a/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py
new file mode 100755
index 00000000..3ded318b
--- /dev/null
+++ b/assets/ar/factuality_disinformation_harmful_content/subjectivity/CT23Subjectivity_LLama3-8b_ZeroShot_Mixed.py
@@ -0,0 +1,58 @@
+from llmebench.datasets import CT23SubjectivityDataset
+from llmebench.models import AzureModel
+from llmebench.tasks import SubjectivityTask
+
+
+def metadata():
+ return {
+ "author": "Mohamed Bayan Kmainasi, Rakif Khan, Ali Ezzat Shahroor, Boushra Bendou, Maram Hasanain, and Firoj Alam",
+ "affiliation": "Arabic Language Technologies, Qatar Computing Research Institute (QCRI), Hamad Bin Khalifa University (HBKU)",
+ "model": "Llama-3.1-8B-Instruct",
+ "description": "For a comprehensive analysis and results, refer to our peer-reviewed publication available at [Springer](https://doi.org/10.1007/978-981-96-0576-7_30) or explore the preprint version on [arXiv](https://arxiv.org/abs/2409.07054).",
+ }
+
+
+def config():
+ return {
+ "dataset": CT23SubjectivityDataset,
+ "task": SubjectivityTask,
+ "model": AzureModel,
+ "model_args": {
+ "class_labels": ["SUBJ", "OBJ"],
+ "max_tries": 3,
+ },
+ "general_args": {"test_split": "ar/dev"},
+ }
+
+
+def prompt(input_sample):
+ return [
+ {
+ "role": "user",
+ "content": (
+ 'صنف الجملة الى "subjective" أو "objective". قم بتقديم التصنيف فقط.\n\n'
+ + "التغريدة: "
+ + input_sample
+ + "\nالتصنيف: "
+ ),
+ }
+ ]
+
+
+def post_process(response):
+ # print(response)
+ if "output" in response:
+ # if "content" in response["messages"]:
+ label = response["output"].strip()
+ label = label.replace("", "")
+ label = label.replace("", "")
+ label = label.lower()
+ else:
+ print("Response .. " + str(response))
+ return None
+ if "غير" in label or "subj" in label or "not" in label or "ليس" in label:
+ return "SUBJ"
+ elif "موضوعية" in label or "obj" in label:
+ return "OBJ"
+ else:
+ return None